diff --git a/.gitignore b/.gitignore index f559dbea..41264a78 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,6 @@ dbdata/ __pycache__/ .idea/ .pytest_cache +test.py +todo.txt +test/ diff --git a/mdb.py b/mdb.py index a981e5be..261428df 100644 --- a/mdb.py +++ b/mdb.py @@ -5,19 +5,21 @@ import readline import traceback import shutil + sys.path.append('miniDB') from database import Database from table import Table +from misc import convert_to_RA # art font is "big" art = ''' - _ _ _____ ____ - (_) (_)| __ \ | _ \ + _ _ _____ ____ + (_) (_)| __ \ | _ \ _ __ ___ _ _ __ _ | | | || |_) | - | '_ ` _ \ | || '_ \ | || | | || _ < + | '_ ` _ \ | || '_ \ | || | | || _ < | | | | | || || | | || || |__| || |_) | - |_| |_| |_||_||_| |_||_||_____/ |____/ 2022 -''' + |_| |_| |_||_||_| |_||_||_____/ |____/ 2022 +''' def search_between(s, first, last): @@ -37,7 +39,6 @@ def in_paren(qsplit, ind): ''' return qsplit[:ind].count('(')>qsplit[:ind].count(')') - def create_query_plan(query, keywords, action): ''' Given a query, the set of keywords that we expect to pe present and the overall action, return the query plan for this query. @@ -53,31 +54,32 @@ def create_query_plan(query, keywords, action): kw_positions = [] i=0 while i=" + left + right = where_split[operator_idx[0]-1] + "<=" + right + where_split[operator_idx[0]+1] = left + where_split[operator_idx[0]+3] = right + del where_split[0] + del where_split[0] + + operator_idx_f = operator_idx[0] + where_dic = {} + left = where_split[:operator_idx_f] + right = where_split[operator_idx_f+1:] + + if (where_split[operator_idx[0]]) == "not": + left = None + elif has_parenth(left): + left = form_where_clause(left) + else: + left = ' '.join(left) + + if has_parenth(right) or operator_idx_f is not None: + right = form_where_clause(right) + else: + right = ' '.join(right) + + where_dic['left'] = left + where_dic['operator'] = ''.join(where_split[operator_idx_f]) + where_dic['right'] = right + + return where_dic + def interpret(query): ''' Interpret the query. @@ -175,14 +268,14 @@ def interpret(query): 'unlock table': ['unlock table', 'force'], 'delete from': ['delete from', 'where'], 'update table': ['update table', 'set', 'where'], - 'create index': ['create index', 'on', 'using'], + 'create index': ['create index', 'on', 'column', 'using'], 'drop index': ['drop index'], 'create view' : ['create view', 'as'] } if query[-1]!=';': query+=';' - + query = query.replace("(", " ( ").replace(")", " ) ").replace(";", " ;").strip() for kw in kw_per_action.keys(): @@ -195,13 +288,17 @@ def execute_dic(dic): ''' Execute the given dictionary ''' + for key in dic.keys(): - if isinstance(dic[key],dict): + # Skip the where key + if key != 'where' and isinstance(dic[key],dict): dic[key] = execute_dic(dic[key]) - + action = list(dic.keys())[0].replace(' ','_') + return getattr(db, action)(*dic.values()) + def interpret_meta(command): """ Interpret meta commands. These commands are used to handle DB stuff, something that can not be easily handled with mSQL given the current architecture. @@ -224,7 +321,7 @@ def interpret_meta(command): def list_databases(db_name): [print(fold.removesuffix('_db')) for fold in os.listdir('dbdata')] - + def list_tables(db_name): [print(pklf.removesuffix('.pkl')) for pklf in os.listdir(f'dbdata/{db_name}_db') if pklf.endswith('.pkl')\ and not pklf.startswith('meta')] @@ -232,7 +329,7 @@ def list_tables(db_name): def change_db(db_name): global db db = Database(db_name, load=True, verbose=verbose) - + def remove_db(db_name): shutil.rmtree(f'dbdata/{db_name}_db') @@ -252,25 +349,27 @@ def remove_db(db_name): db = Database(dbname, load=True) - - if fname is not None: for line in open(fname, 'r').read().splitlines(): if line.startswith('--'): continue if line.startswith('explain'): dic = interpret(line.removeprefix('explain ')) pprint(dic, sort_dicts=False) - else : + if line.startswith('convert'): + dic = convert_to_RA(interpret(line.removeprefix('convert '))) + pprint(dic, sort_dicts=False) + else: dic = interpret(line.lower()) result = execute_dic(dic) if isinstance(result,Table): result.show() - + from prompt_toolkit import PromptSession from prompt_toolkit.history import FileHistory from prompt_toolkit.auto_suggest import AutoSuggestFromHistory + print(art) session = PromptSession(history=FileHistory('.inp_history')) while 1: @@ -282,13 +381,16 @@ def remove_db(db_name): print('\nbye!') break try: - if line=='exit': + if line=='exit;': break if line.split(' ')[0].removesuffix(';') in ['lsdb', 'lstb', 'cdb', 'rmdb']: interpret_meta(line) elif line.startswith('explain'): dic = interpret(line.removeprefix('explain ')) pprint(dic, sort_dicts=False) + elif line.startswith('convert'): + RA_expression = convert_to_RA(interpret(line.removeprefix('convert '))) + print(RA_expression) else: dic = interpret(line) result = execute_dic(dic) @@ -296,3 +398,5 @@ def remove_db(db_name): result.show() except Exception: print(traceback.format_exc()) + + diff --git a/miniDB/database.py b/miniDB/database.py index a3ac6be7..f20222cd 100644 --- a/miniDB/database.py +++ b/miniDB/database.py @@ -13,6 +13,7 @@ from joins import Inlj, Smj from btree import Btree +from hash import Hash from misc import split_condition from table import Table @@ -54,7 +55,7 @@ def __init__(self, name, load=True, verbose = True): self.create_table('meta_length', 'table_name,no_of_rows', 'str,int') self.create_table('meta_locks', 'table_name,pid,mode', 'str,int,str') self.create_table('meta_insert_stack', 'table_name,indexes', 'str,list') - self.create_table('meta_indexes', 'table_name,index_name', 'str,str') + self.create_table('meta_indexes', 'table_name,table_column,index_name,index_type', 'str,str,str,str') self.save_database() def save_database(self): @@ -101,19 +102,24 @@ def _update(self): self._update_meta_insert_stack() - def create_table(self, name, column_names, column_types, primary_key=None, load=None): + def create_table(self, name, column_names, column_types, primary_key=None, unique_columns=None, load=None): ''' This method create a new table. This table is saved and can be accessed via db_object.tables['table_name'] or db_object.table_name Args: name: string. Name of table. - column_names: list. Names of columns. - column_types: list. Types of columns. + column_names: list?. Names of columns. + column_types: list?. Types of columns. primary_key: string. The primary key (if it exists). + unique_columns: string. Names of columns that will be unique (seperated by comma). load: boolean. Defines table object parameters as the name of the table and the column names. ''' + # print('here -> ', column_names.split(',')) - self.tables.update({name: Table(name=name, column_names=column_names.split(','), column_types=column_types.split(','), primary_key=primary_key, load=load)}) + if unique_columns is not None: + unique_columns = unique_columns.split(',') + + self.tables.update({name: Table(name=name, column_names=column_names.split(','), column_types=column_types.split(','), primary_key=primary_key, unique_columns=unique_columns, load=load)}) # self._name = Table(name=name, column_names=column_names, column_types=column_types, load=load) # check that new dynamic var doesnt exist already # self.no_of_tables += 1 @@ -269,8 +275,10 @@ def insert_into(self, table_name, row_str): # check the insert_stack meta table lock_ownership = self.lock_table(table_name, mode='x') insert_stack = self._get_insert_stack_for_table(table_name) + + indexes = self._find_idxs(table_name) try: - self.tables[table_name]._insert(row, insert_stack) + self.tables[table_name]._insert(row, insert_stack, indexes) except Exception as e: logging.info(e) logging.info('ABORTED') @@ -334,14 +342,20 @@ def delete_from(self, table_name, condition): def select(self, columns, table_name, condition, distinct=None, order_by=None, \ limit=True, desc=None, save_as=None, return_object=True): ''' - Selects and outputs a table's data where condtion is met. + Selects and outputs a table's data where condition is met. Args: table_name: string. Name of table (must be part of database). columns: list. The columns that will be part of the output table (use '*' to select all available columns) - condition: string. A condition using the following format: - 'column[<,<=,==,>=,>]value' or - 'value[<,<=,==,>=,>]column'. + condition: string or dict. + String is using the following format: + 'column[<,<=,==,>=,>]value' or + 'value[<,<=,==,>=,>]column'. + Dict is using the following format: + {'left': 'column[<,<=,==,>=,>]value', + 'operator': '[and,or]', + 'right': 'column[<,<=,==,>=,>]value', + } Operatores supported: (<,<=,==,>=,>) order_by: string. A column name that signals that the resulting table should be ordered based on it (no order if None). @@ -351,27 +365,29 @@ def select(self, columns, table_name, condition, distinct=None, order_by=None, \ return_object: boolean. If True, the result will be a table object (useful for internal use - the result will be printed by default). distinct: boolean. If True, the resulting table will contain only unique rows. ''' - + # self.lock_table(table_name, mode='x') + if self.is_locked(table_name): + return + # print(table_name) self.load_database() - if isinstance(table_name,Table): + if isinstance(table_name, Table): return table_name._select_where(columns, condition, distinct, order_by, desc, limit) - if condition is not None: - condition_column = split_condition(condition)[0] - else: - condition_column = '' - - # self.lock_table(table_name, mode='x') if self.is_locked(table_name): return - if self._has_index(table_name) and condition_column==self.tables[table_name].column_names[self.tables[table_name].pk_idx]: - index_name = self.select('*', 'meta_indexes', f'table_name={table_name}', return_object=True).column_by_name('index_name')[0] - bt = self._load_idx(index_name) - table = self.tables[table_name]._select_where_with_btree(columns, bt, condition, distinct, order_by, desc, limit) - else: - table = self.tables[table_name]._select_where(columns, condition, distinct, order_by, desc, limit) + + supported_indexes = None + if self._has_index(table_name): + index_rows = self.select('*', 'meta_indexes', f'table_name={table_name}', return_object=True).data + supported_indexes = {} + for index in index_rows: + supported_indexes[index[1]] = self._load_idx(index[2]) + + table = self.tables[table_name]._select_where(columns, condition, distinct, order_by, desc, limit, supported_indexes) + + # self.unlock_table(table_name) if save_as is not None: table._name = save_as @@ -650,47 +666,89 @@ def _update_meta_insert_stack_for_tb(self, table_name, new_stack): # indexes - def create_index(self, index_name, table_name, index_type='btree'): + def create_index(self, index_name, table_name, table_column=None, index_type=None): ''' Creates an index on a specified table with a given name. - Important: An index can only be created on a primary key (the user does not specify the column). + Important: An index can only be created on a primary key or unique column. Args: table_name: string. Table name (must be part of database). index_name: string. Name of the created index. + index_type: string. Hash or Btree ''' - if self.tables[table_name].pk_idx is None: # if no primary key, no index - raise Exception('Cannot create index. Table has no primary key.') + table_instance = self.tables[table_name] + + # Sets index to primary key by default + if table_column is None: + if table_instance.pk is not None: + table_column = table_instance.pk + else: + raise Exception(f'Cannot create index. {table_instance._name} does not have a primary key.') + + # Check if column exists + if(table_column not in table_instance.column_names): + raise Exception(f'Cannot create index. {table_column} column does not exist.') + + if (table_column is not table_instance.pk) and table_column not in table_instance.unique_columns: + raise Exception(f'Cannot create index. {table_column} is not a unique column.') + if index_name not in self.tables['meta_indexes'].column_by_name('index_name'): - # currently only btree is supported. This can be changed by adding another if. if index_type=='btree': logging.info('Creating Btree index.') # insert a record with the name of the index and the table on which it's created to the meta_indexes table - self.tables['meta_indexes']._insert([table_name, index_name]) - # crate the actual index - self._construct_index(table_name, index_name) + self.tables['meta_indexes']._insert([table_name, table_column, index_name, index_type]) + # create the actual index + self._construct_btree_index(table_name, table_column, index_name) + self.save_database() + elif index_type=='hash': + logging.info('Creating Hash index.') + # insert a record with the name of the index and the table on which it's created to the meta_indexes table + self.tables['meta_indexes']._insert([table_name, table_column, index_name, index_type]) + # create the actual index + self._construct_hash_index(table_name, table_column, index_name) self.save_database() - else: - raise Exception('Cannot create index. Another index with the same name already exists.') - def _construct_index(self, table_name, index_name): + def _construct_btree_index(self, table_name, table_column, index_name): ''' Construct a btree on a table and save. Args: table_name: string. Table name (must be part of database). index_name: string. Name of the created index. + table_column: string. Column name (must be unique). ''' bt = Btree(3) # 3 is arbitrary - - # for each record in the primary key of the table, insert its value and index to the btree - for idx, key in enumerate(self.tables[table_name].column_by_name(self.tables[table_name].pk)): + + # for each record in the table_column of the table, insert its value and index to the btree + for idx, key in enumerate(self.tables[table_name].column_by_name(table_column)): if key is None: continue bt.insert(key, idx) + # save the btree self._save_index(index_name, bt) + def _construct_hash_index(self, table_name, table_column, index_name): + ''' + Construct a hash index on a table column and save. + + Args: + table_name: string. Table name (must be part of database). + index_name: string. Name of the created index. + column_name: string. Name of the column to create the index on. + ''' + hash = Hash() + + # for each record in the column, insert its value and index to the hash table + for idx, value in enumerate(self.tables[table_name].column_by_name(table_column)): + if value is None: + continue + hash.insert(value, idx) + + # save the hash table + self._save_index(index_name, hash) + + def _has_index(self, table_name): ''' @@ -698,8 +756,15 @@ def _has_index(self, table_name): Args: table_name: string. Table name (must be part of database). + table_column: string. Table column. ''' - return table_name in self.tables['meta_indexes'].column_by_name('table_name') + rows = self.tables['meta_indexes'].data + for row_idx in range(len(rows)): + if rows[row_idx][0] == table_name: + return True + + return False + def _save_index(self, index_name, index): ''' @@ -717,6 +782,27 @@ def _save_index(self, index_name, index): with open(f'{self.savedir}/indexes/meta_{index_name}_index.pkl', 'wb') as f: pickle.dump(index, f) + def _find_idxs(self, table_name, table_column=None): + ''' + Find all the indexes for that table object. + + Args: + table_name: string. + ''' + if self._has_index(table_name): + supported_idxs = {} + if table_column is None: + index_rows = self.select('*', 'meta_indexes', f'table_name={table_name}', return_object=True).data + else: + index_rows = self.select('*', 'meta_indexes', f'table_name={table_name} and table_column={table_column}', return_object=True).data + + for row in index_rows: + supported_idxs[row[1]] = self._load_idx(row[2]) + else: + supported_idxs = None + + return supported_idxs + def _load_idx(self, index_name): ''' Load and return the specified index. @@ -736,6 +822,7 @@ def drop_index(self, index_name): Args: index_name: string. Name of index. ''' + if index_name in self.tables['meta_indexes'].column_by_name('index_name'): self.delete_from('meta_indexes', f'index_name = {index_name}') diff --git a/miniDB/hash.py b/miniDB/hash.py new file mode 100644 index 00000000..5f2468c7 --- /dev/null +++ b/miniDB/hash.py @@ -0,0 +1,47 @@ +class Hash: + def __init__(self): + self.capacity = 2 # Set the initial capacity of the hash table to 100. + self.size = 0 + self.buckets = [None] * self.capacity # Initialize the hash table with None values. + + def _hash(self, key): + # The hash function maps a key to a hash value. + # In this implementation, we're using the modulo operator "%". + hash_value = hash(key) % self.capacity + return hash_value + + def insert(self, key, value): + # Insert a key-value pair into the hash table. + # If the key already exists, update its value. + index = self._hash(key) + if self.buckets[index] is None: + # Create a new bucket if it's empty. + self.buckets[index] = [] + for item in self.buckets[index]: + if item[0] == key: + # Update the value if the key already exists. + item[1] = value + return + self.buckets[index].append([key, value]) + self.size += 1 + + def search(self, key): + # Search for a key in the hash table and return its value. + index = self._hash(key) + if self.buckets[index] is None: + return None + for item in self.buckets[index]: + if item[0] == key: + return item[1] + return None + + def delete(self, key): + # Delete a key from the hash table. + index = self._hash(key) + if self.buckets[index] is None: + return + for i in range(len(self.buckets[index])): + if self.buckets[index][i][0] == key: + self.buckets[index].pop(i) + self.size -= 1 + return diff --git a/miniDB/misc.py b/miniDB/misc.py index aefada74..e60ca329 100644 --- a/miniDB/misc.py +++ b/miniDB/misc.py @@ -24,6 +24,7 @@ def split_condition(condition): for op_key in ops.keys(): splt=condition.split(op_key) + if len(splt)>1: left, right = splt[0].strip(), splt[1].strip() @@ -48,3 +49,93 @@ def reverse_op(op): '<=' : '>=', '=' : '=' }.get(op) + +def logical_operator_on_rows(rows_len, left_rows, operator, right_rows): + ''' + Return the rows after the logical operation + ''' + + if(operator == 'and'): + row_idxs = list(set(left_rows).intersection(right_rows)) + elif(operator == 'or'): + row_idxs = list(set(left_rows).union(set(right_rows))) + elif(operator == 'not'): + row_idxs = [i for i in range(rows_len) if i not in right_rows] + else: + raise Exception('Not a valid logical operator.') + return row_idxs + +def convert_to_RA(dic): + ''' + Convert the given query to relational algebra string + ''' + RA_dic = convert_query_dic_to_RA_dic(dic) + projection = RA_dic['projection'] + selection = selection_to_string(RA_dic['selection']) + table = table_name_to_string(RA_dic['table']) + RA_expression = '' + if RA_dic['distinct']: + RA_expression += "δ \n " + if projection != '*': + RA_expression += "Π " + projection + "\n " + if selection is not None: + RA_expression += "σ " + selection + " (" + table +")" + else: + RA_expression += "σ(" + table +")" + return RA_expression + +def convert_query_dic_to_RA_dic(dic): + ''' + Convert a given query dictionary to a relational algebra dictionary + ''' + RA_expression = { + 'distinct': None, + 'projection': 'select', + 'selection': 'where', + 'table': 'from' + } + RA_expression['projection'] = dic['select'] + RA_expression['selection'] = dic['where'] + RA_expression['table'] = simplify_from(dic['from']) + if dic['distinct'] is not None: + RA_expression['distinct'] = True + return RA_expression + +def simplify_from(condition): + if (isinstance(condition, dict)) and (isinstance(condition['right'], dict)): + condition['right'] = simplify_from(condition['right']['from']) + return condition + elif (isinstance(condition, dict)) and (not (isinstance(condition['right'], dict))): + return condition + else: + return ''.join(condition) + +def selection_to_string(condition): + if condition is None: + return None + if isinstance(condition, dict): + if (condition['left'] == None): + temp_string = condition['operator'] + '(' + selection_to_string(condition['right']) + ')' + else: + temp_string = selection_to_string(condition['left']) + ' ' + condition['operator'] + ' ' + selection_to_string(condition['right']) + return temp_string + else: + return ''.join(condition) + +def table_name_to_string(table): + + if (isinstance(table, dict)) and (table['join'] is not None): + join_character = " ⋈ " + if table['join'] == 'left': + join_character = " ⋈ L " + elif table['join'] == 'right': + join_character = " ⋈ R " + elif table['join'] == 'full': + join_character = " ⋈ o " + + if (isinstance(table, dict)): + table_string = table_name_to_string(table['left']) + join_character + table['on'] +' '+ table_name_to_string(table['right']) + return table_string + else: + table_string = table + return ''.join(table_string) diff --git a/miniDB/table.py b/miniDB/table.py index f5c7d937..a30d43e7 100644 --- a/miniDB/table.py +++ b/miniDB/table.py @@ -6,8 +6,9 @@ sys.path.append(f'{os.path.dirname(os.path.dirname(os.path.abspath(__file__)))}/miniDB') -from misc import get_op, split_condition - +from misc import get_op, split_condition, logical_operator_on_rows +from btree import Btree +from hash import Hash class Table: ''' @@ -26,8 +27,8 @@ class Table: - a dictionary that includes the appropriate info (all the attributes in __init__) ''' - def __init__(self, name=None, column_names=None, column_types=None, primary_key=None, load=None): - + def __init__(self, name=None, column_names=None, column_types=None, primary_key=None, unique_columns=None, load=None): + if load is not None: # if load is a dict, replace the object dict with it (replaces the object with the specified one) if isinstance(load, dict): @@ -36,7 +37,7 @@ def __init__(self, name=None, column_names=None, column_types=None, primary_key= # if load is str, load from a file elif isinstance(load, str): self._load_from_file(load) - + # if name, columns_names and column types are not none elif (name is not None) and (column_names is not None) and (column_types is not None): @@ -46,9 +47,9 @@ def __init__(self, name=None, column_names=None, column_types=None, primary_key= raise ValueError('Need same number of column names and types.') self.column_names = column_names - + self.unique_columns = unique_columns self.columns = [] - + for col in self.column_names: if col not in self.__dir__(): # this is used in order to be able to call a column using its name as an attribute. @@ -102,7 +103,7 @@ def _cast_column(self, column_name, cast_type): # self._update() - def _insert(self, row, insert_stack=[]): + def _insert(self, row, insert_stack=[], indexes=None): ''' Insert row to table. @@ -110,6 +111,7 @@ def _insert(self, row, insert_stack=[]): row: list. A list of values to be inserted (will be casted to a predifined type automatically). insert_stack: list. The insert stack (empty by default). ''' + if len(row)!=len(self.column_names): raise ValueError(f'ERROR -> Cannot insert {len(row)} values. Only {len(self.column_names)} columns exist') @@ -123,12 +125,18 @@ def _insert(self, row, insert_stack=[]): except TypeError as exc: if row[i] != None: print(exc) - - # if value is to be appended to the primary_key column, check that it doesnt alrady exist (no duplicate primary keys) + + # if value is to be appended to the primary_key column, check that it doesnt already exist (no duplicate primary keys) if i==self.pk_idx and row[i] in self.column_by_name(self.pk): raise ValueError(f'## ERROR -> Value {row[i]} already exists in primary key column.') elif i==self.pk_idx and row[i] is None: raise ValueError(f'ERROR -> The value of the primary key cannot be None.') + + # if value is to be appended to the unique column, check that it doesnt already exist + if hasattr(self, 'unique_columns') and self.unique_columns is not None: + column_name = self.column_names[i] + if column_name in self.unique_columns and row[i] in self.column_by_name(column_name): + raise ValueError(f'## ERROR -> Value {row[i]} already exists in {column_name} unique column.') # if insert_stack is not empty, append to its last index if insert_stack != []: @@ -137,6 +145,13 @@ def _insert(self, row, insert_stack=[]): self.data.append(row) # self._update() + # add row values to the index + if indexes is not None: + for column_name, btree in indexes.items(): + column_pointer = self.column_names.index(column_name) + row_pointer = len(self.data) + btree.insert(row[column_pointer], row_pointer) + def _update_rows(self, set_value, set_column, condition): ''' Update where Condition is met. @@ -147,7 +162,7 @@ def _update_rows(self, set_value, set_column, condition): condition: string. A condition using the following format: 'column[<,<=,=,>=,>]value' or 'value[<,<=,=,>=,>]column'. - + Operatores supported: (<,<=,=,>=,>) ''' # parse the condition @@ -176,21 +191,21 @@ def _delete_where(self, condition): These rows are then appended to the insert_stack. Args: - condition: string. A condition using the following format: - 'column[<,<=,==,>=,>]value' or - 'value[<,<=,==,>=,>]column'. - - Operatores supported: (<,<=,==,>=,>) + condition: string or dict. + String is using the following format: + 'column[<,<=,==,>=,>]value' or + 'value[<,<=,==,>=,>]column'. + Dict is using the following format: + {'left': 'column[<,<=,==,>=,>]value', + 'operator': '[and,or]', + 'right': 'column[<,<=,==,>=,>]value', + } ''' - column_name, operator, value = self._parse_condition(condition) - - indexes_to_del = [] - - column = self.column_by_name(column_name) - for index, row_value in enumerate(column): - if get_op(operator, row_value, value): - indexes_to_del.append(index) - + if condition is not None: + indexes_to_del = self._find_rows(condition) + else: + indexes_to_del = [i for i in range(len(self.data))] + # we pop from highest to lowest index in order to avoid removing the wrong item # since we dont delete, we dont have to to pop in that order, but since delete is used # to delete from meta tables too, we still implement it. @@ -207,15 +222,21 @@ def _delete_where(self, condition): return indexes_to_del - def _select_where(self, return_columns, condition=None, distinct=False, order_by=None, desc=True, limit=None): + def _select_where(self, return_columns, condition=None, distinct=False, order_by=None, desc=True, limit=None, supported_indexes=None): ''' Select and return a table containing specified columns and rows where condition is met. Args: return_columns: list. The columns to be returned. - condition: string. A condition using the following format: - 'column[<,<=,==,>=,>]value' or - 'value[<,<=,==,>=,>]column'. + condition: string or dict. + String is using the following format: + 'column[<,<=,==,>=,>]value' or + 'value[<,<=,==,>=,>]column'. + Dict is using the following format: + {'left': 'column[<,<=,==,>=,>]value', + 'operator': '[and,or]', + 'right': 'column[<,<=,==,>=,>]value', + } Operatores supported: (<,<=,==,>=,>) distinct: boolean. If True, the resulting table will contain only unique rows (False by default). @@ -233,9 +254,7 @@ def _select_where(self, return_columns, condition=None, distinct=False, order_by # if condition is None, return all rows # if not, return the rows with values where condition is met for value if condition is not None: - column_name, operator, value = self._parse_condition(condition) - column = self.column_by_name(column_name) - rows = [ind for ind, x in enumerate(column) if get_op(operator, x, value)] + rows = self._find_rows(condition, supported_indexes) else: rows = [i for i in range(len(self.data))] @@ -259,9 +278,9 @@ def _select_where(self, return_columns, condition=None, distinct=False, order_by # k = int(limit) # except ValueError: # raise Exception("The value following 'top' in the query should be a number.") - + # # Remove from the table's data all the None-filled rows, as they are not shown by default - # # Then, show the first k rows + # # Then, show the first k rows # s_table.data.remove(len(s_table.column_names) * [None]) # s_table.data = s_table.data[:k] if isinstance(limit,str): @@ -269,60 +288,101 @@ def _select_where(self, return_columns, condition=None, distinct=False, order_by return s_table + def _find_rows(self, condition, supported_indexes=None): + ''' + Find and return all the rows where condition is met. - def _select_where_with_btree(self, return_columns, bt, condition, distinct=False, order_by=None, desc=True, limit=None): - - # if * return all columns, else find the column indexes for the columns specified - if return_columns == '*': - return_cols = [i for i in range(len(self.column_names))] - else: - return_cols = [self.column_names.index(colname) for colname in return_columns] - - - column_name, operator, value = self._parse_condition(condition) - - # if the column in condition is not a primary key, abort the select - if column_name != self.column_names[self.pk_idx]: - print('Column is not PK. Aborting') - - # here we run the same select twice, sequentially and using the btree. - # we then check the results match and compare performance (number of operation) - column = self.column_by_name(column_name) + Args: + condition: string or dict. + String is using the following format: + 'column[<,<=,==,>=,>]value' or + 'value[<,<=,==,>=,>]column'. + Dict is using the following format: + {'left': 'column[<,<=,==,>=,>]value', + 'operator': '[and,or]', + 'right': 'column[<,<=,==,>=,>]value', + } + supported_indexes: dict. + Dict is using the following format: + { + 'column': btree object, + } + ''' + + if isinstance(condition, str): + final_rows = self._in_depth(condition, supported_indexes) + elif isinstance(condition, dict): + left_part = condition['left'] + right_part = condition['right'] + operator = condition['operator'] + if(operator != 'not'): + left_rows = self._in_depth(left_part, supported_indexes) + else: + left_rows = None + right_rows = self._in_depth(right_part, supported_indexes) - # sequential - rows1 = [] - opsseq = 0 - for ind, x in enumerate(column): - opsseq+=1 - if get_op(operator, x, value): - rows1.append(ind) + final_rows = logical_operator_on_rows(rows_len=len(self.data), left_rows=left_rows, operator=operator, right_rows=right_rows) + + return final_rows - # btree find - rows = bt.find(operator, value) + def _in_depth(self, condition, supported_indexes): + ''' + This method is used for recursion for the nested dictionaries. - try: - k = int(limit) - except TypeError: - k = None - # same as simple select from now on - rows = rows[:k] - # TODO: this needs to be dumbed down - dict = {(key):([[self.data[i][j] for j in return_cols] for i in rows] if key=="data" else value) for key,value in self.__dict__.items()} + Args: + condition: string or dict. + String is using the following format: + 'column[<,<=,==,>=,>]value' or + 'value[<,<=,==,>=,>]column'. + Dict is using the following format: + {'left': 'column[<,<=,==,>=,>]value', + 'operator': '[and,or]', + 'right': 'column[<,<=,==,>=,>]value', + } + supported_indexes: dict. + Dict is using the following format: + { + 'column': btree object, + } + ''' - dict['column_names'] = [self.column_names[i] for i in return_cols] - dict['column_types'] = [self.column_types[i] for i in return_cols] + if isinstance(condition,str): + column_name, operator, value = self._parse_condition(condition) + if supported_indexes is not None and column_name in supported_indexes: + index = supported_indexes[column_name] + if isinstance(index, Hash) and operator == '=': + # Search using hash index + row = index.search(value) + row = [int(d) for d in str(row)] + return row + elif isinstance(supported_indexes[column_name], Btree): + # Search using btree + rows = index.find(operator, value) + return rows + + # Linear search if index was not used + column = self.column_by_name(column_name) + rows = [ind for ind, x in enumerate(column) if get_op(operator, x, value)] + return rows - s_table = Table(load=dict) + elif isinstance(condition,dict): - s_table.data = list(set(map(lambda x: tuple(x), s_table.data))) if distinct else s_table.data + left_part = condition['left'] + right_part = condition['right'] + operator = condition['operator'] - if order_by: - s_table.order_by(order_by, desc) + if(operator != 'not'): + left_rows = self._in_depth(left_part, supported_indexes) + else: + left_rows = None + right_rows = self._in_depth(right_part, supported_indexes) + + rows = logical_operator_on_rows(rows_len=len(self.data), left_rows=left_rows, operator=operator, right_rows=right_rows) - if isinstance(limit,str): - s_table.data = [row for row in s_table.data if row is not None][:int(limit)] + else: + raise Exception('Not a valid where type.') - return s_table + return rows def order_by(self, column_name, desc=True): ''' @@ -347,7 +407,7 @@ def _general_join_processing(self, table_right:Table, condition, join_type): condition: string. A condition using the following format: 'column[<,<=,==,>=,>]value' or 'value[<,<=,==,>=,>]column'. - + Operators supported: (<,<=,==,>=,>) ''' # get columns and operator @@ -391,7 +451,7 @@ def _inner_join(self, table_right: Table, condition): condition: string. A condition using the following format: 'column[<,<=,==,>=,>]value' or 'value[<,<=,==,>=,>]column'. - + Operators supported: (<,<=,==,>=,>) ''' join_table, column_index_left, column_index_right, operator = self._general_join_processing(table_right, condition, 'inner') @@ -411,7 +471,7 @@ def _inner_join(self, table_right: Table, condition): join_table._insert(row_left+row_right) return join_table - + def _left_join(self, table_right: Table, condition): ''' Perform a left join on the table with the supplied table (right). @@ -420,7 +480,7 @@ def _left_join(self, table_right: Table, condition): condition: string. A condition using the following format: 'column[<,<=,==,>=,>]value' or 'value[<,<=,==,>=,>]column'. - + Operators supported: (<,<=,==,>=,>) ''' join_table, column_index_left, column_index_right, operator = self._general_join_processing(table_right, condition, 'left') @@ -450,7 +510,7 @@ def _right_join(self, table_right: Table, condition): condition: string. A condition using the following format: 'column[<,<=,==,>=,>]value' or 'value[<,<=,==,>=,>]column'. - + Operators supported: (<,<=,==,>=,>) ''' join_table, column_index_left, column_index_right, operator = self._general_join_processing(table_right, condition, 'right') @@ -471,7 +531,7 @@ def _right_join(self, table_right: Table, condition): join_table._insert(row_left + row_right) return join_table - + def _full_join(self, table_right: Table, condition): ''' Perform a full join on the table with the supplied table (right). @@ -480,7 +540,7 @@ def _full_join(self, table_right: Table, condition): condition: string. A condition using the following format: 'column[<,<=,==,>=,>]value' or 'value[<,<=,==,>=,>]column'. - + Operators supported: (<,<=,==,>=,>) ''' join_table, column_index_left, column_index_right, operator = self._general_join_processing(table_right, condition, 'full') @@ -490,7 +550,7 @@ def _full_join(self, table_right: Table, condition): right_table_row_length = len(table_right.column_names) left_table_row_length = len(self.column_names) - + for row_left in self.data: left_value = row_left[column_index_left] if left_value is None: @@ -533,6 +593,11 @@ def show(self, no_of_rows=None, is_locked=False): if self.pk_idx is not None: # table has a primary key, add PK next to the appropriate column headers[self.pk_idx] = headers[self.pk_idx]+' #PK#' + # adds unique keyword next to the column name + if hasattr(self, 'unique_columns') and self.unique_columns is not None: + unique_indxs = [i for i, x in enumerate(self.column_names) if x in self.unique_columns] + for indx in unique_indxs: + headers[indx] = headers[indx] + " #UNIQUE#" # detect the rows that are no tfull of nones (these rows have been deleted) # if we dont skip these rows, the returning table has empty rows at the deleted positions non_none_rows = [row for row in self.data if any(row)] @@ -543,15 +608,15 @@ def show(self, no_of_rows=None, is_locked=False): def _parse_condition(self, condition, join=False): ''' Parse the single string condition and return the value of the column and the operator. - Args: condition: string. A condition using the following format: 'column[<,<=,==,>=,>]value' or 'value[<,<=,==,>=,>]column'. - + Operatores supported: (<,<=,==,>=,>) join: boolean. Whether to join or not (False by default). ''' + # if both_columns (used by the join function) return the names of the names of the columns (left first) if join: return split_condition(condition) @@ -559,12 +624,13 @@ def _parse_condition(self, condition, join=False): # cast the value with the specified column's type and return the column name, the operator and the casted value left, op, right = split_condition(condition) if left not in self.column_names: - raise ValueError(f'Condition is not valid (cant find column name)') + raise ValueError(f'Condition is not valid (cant find column {left})') coltype = self.column_types[self.column_names.index(left)] return left, op, coltype(right) + def _load_from_file(self, filename): ''' Load table from a pkl file (not used currently).