From f7f6ca7ad95be7e3ee9990fc7ae02d7d7ce44332 Mon Sep 17 00:00:00 2001 From: Antonyfrtz Date: Mon, 16 Jan 2023 14:59:22 +0200 Subject: [PATCH 01/25] Implementation of NOT Operator - Checks if NOT keyword is present by splicing the condition on whitespace and evaluating the number of items from the split. If there are more than one, not keyword is in the query and we trim the 'not ' part of the condition, and reverse the operator with a modification of the built-in function in the misc.py file - Fixed operators in the misc.py implementation of reverse_op - Added comments to sections of code --- .gitignore | 2 ++ mdb.py | 20 +++++++++++--------- miniDB/database.py | 4 ++-- miniDB/misc.py | 26 +++++++++++++++----------- miniDB/table.py | 9 ++++++++- 5 files changed, 38 insertions(+), 23 deletions(-) diff --git a/.gitignore b/.gitignore index f559dbea..55802537 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,5 @@ dbdata/ __pycache__/ .idea/ .pytest_cache +.vscode/launch.json +.vscode/settings.json diff --git a/mdb.py b/mdb.py index a981e5be..00668a02 100644 --- a/mdb.py +++ b/mdb.py @@ -45,14 +45,14 @@ def create_query_plan(query, keywords, action): This can and will be used recursively ''' - dic = {val: None for val in keywords if val!=';'} + dic = {val: None for val in keywords if val!=';'} # dict of query words - ql = [val for val in query.split(' ') if val !=''] + ql = [val for val in query.split(' ') if val !=''] # list of query words kw_in_query = [] kw_positions = [] i=0 - while i=': operator.ge, '<=': operator.le, - '=': operator.eq} + '=': operator.eq, + '<>': operator.ne + } try: return ops[op](a,b) @@ -20,12 +22,14 @@ def split_condition(condition): '<=': operator.le, '=': operator.eq, '>': operator.gt, - '<': operator.lt} + '<': operator.lt, + '<>': operator.ne + } for op_key in ops.keys(): - splt=condition.split(op_key) - if len(splt)>1: - left, right = splt[0].strip(), splt[1].strip() + splt=condition.split(op_key) # split string on operator + if len(splt)>1: # if split successful + left, right = splt[0].strip(), splt[1].strip() # values for condition if right[0] == '"' == right[-1]: # If the value has leading and trailing quotes, remove them. right = right.strip('"') @@ -39,12 +43,12 @@ def split_condition(condition): def reverse_op(op): ''' - Reverse the operator given + Reverse the operator given (REWORKED WITH NOT OPERATOR) ''' return { - '>' : '<', - '>=' : '<=', - '<' : '>', - '<=' : '>=', - '=' : '=' + '>' : '<=', + '>=' : '<', + '<' : '>=', + '<=' : '>', + '=' : '<>' }.get(op) diff --git a/miniDB/table.py b/miniDB/table.py index f5c7d937..e5ca9ed5 100644 --- a/miniDB/table.py +++ b/miniDB/table.py @@ -6,7 +6,7 @@ sys.path.append(f'{os.path.dirname(os.path.dirname(os.path.abspath(__file__)))}/miniDB') -from misc import get_op, split_condition +from misc import get_op, reverse_op, split_condition class Table: @@ -233,7 +233,14 @@ def _select_where(self, return_columns, condition=None, distinct=False, order_by # if condition is None, return all rows # if not, return the rows with values where condition is met for value if condition is not None: + # NOT case + containsNot=False # flag for not keyword + if len(condition.split(' '))>1: # split will have more than one item + condition=condition.replace('not ','') # remove not keyword + containsNot=True column_name, operator, value = self._parse_condition(condition) + if containsNot: # reverse operator if flag true + operator=reverse_op(operator) column = self.column_by_name(column_name) rows = [ind for ind, x in enumerate(column) if get_op(operator, x, value)] else: From 52fddcf91b40cda4e514fa4fa9f4f4f82ae0444b Mon Sep 17 00:00:00 2001 From: Vassiliki Karagianniki <78561945+vassilikikrg@users.noreply.github.com> Date: Mon, 16 Jan 2023 17:06:55 +0200 Subject: [PATCH 02/25] between: work in progress --- miniDB/database.py | 5 ++++- miniDB/table.py | 36 ++++++++++++++++++++++++++---------- 2 files changed, 30 insertions(+), 11 deletions(-) diff --git a/miniDB/database.py b/miniDB/database.py index f8633254..0b8a4e94 100644 --- a/miniDB/database.py +++ b/miniDB/database.py @@ -358,7 +358,10 @@ def select(self, columns, table_name, condition, distinct=None, order_by=None, \ return table_name._select_where(columns, condition, distinct, order_by, desc, limit) if condition is not None: # get column of where clause - condition_column = split_condition(condition)[0] + if "between" in condition: #if condition contains the keyword "between",then condition has the format of table.column between value1 and value2 + condition_column=condition.split(" ")[0] + else: + condition_column = split_condition(condition)[0] else: condition_column = '' diff --git a/miniDB/table.py b/miniDB/table.py index e5ca9ed5..2cac563e 100644 --- a/miniDB/table.py +++ b/miniDB/table.py @@ -233,16 +233,32 @@ def _select_where(self, return_columns, condition=None, distinct=False, order_by # if condition is None, return all rows # if not, return the rows with values where condition is met for value if condition is not None: - # NOT case - containsNot=False # flag for not keyword - if len(condition.split(' '))>1: # split will have more than one item - condition=condition.replace('not ','') # remove not keyword - containsNot=True - column_name, operator, value = self._parse_condition(condition) - if containsNot: # reverse operator if flag true - operator=reverse_op(operator) - column = self.column_by_name(column_name) - rows = [ind for ind, x in enumerate(column) if get_op(operator, x, value)] + + # BETWEEN case + if "between" in condition: + condition_list=condition.split(" ") + if len(condition_list)!=5 or condition_list[1]!="between" or condition_list[3]!="and": + raise Exception("condition containing between keyword must have the following format: column between value1 and value2") + else: + try: + column=self.column_by_name(condition_list[0]) + value1=int(condition_list[2]) + value2=int(condition_list[4]) + rows = [ind for ind, x in enumerate(column) if ( get_op('>=', x, value) and get_op('<=', x, value) )] + except: + print("You need to provide only arithmetical values in order to evaluate between clause") + else: + # NOT case + containsNot=False # flag for not keyword + if len(condition.split(' '))>1: # split will have more than one item + condition=condition.replace('not ','') # remove not keyword + containsNot=True + column_name, operator, value = self._parse_condition(condition) + if containsNot: # reverse operator if flag true + operator=reverse_op(operator) + column = self.column_by_name(column_name) + rows = [ind for ind, x in enumerate(column) if get_op(operator, x, value)] + else: rows = [i for i in range(len(self.data))] From ecf55738c0a35ac741edfe1a4e383a4510bc389f Mon Sep 17 00:00:00 2001 From: Vassiliki Karagianniki <78561945+vassilikikrg@users.noreply.github.com> Date: Mon, 16 Jan 2023 20:21:57 +0200 Subject: [PATCH 03/25] Implementation of BETWEEN Operator -In database.py file: we check if condition contains the keyword "between". If so the condition has the format of "column between value1 and value2" and thus we assign the column name to the condition_column value -In table.py file: if condition contains the keyword "between", condition is splited into a list using whitespace as seperator. If list lenght is different than 5 (correct format is:column between value1 and value2) or 'between' and 'and' aren't in the correct positions,exception is raised. Else, column values are grouped into a list, and then we define a new list(named "rows") in which each value is greater than or to equal value1 and less than or equal to value2 (value1 and value2 are provided by the user in between statement). In case of error,we raise ValueError because value1 and value2 are not valid. --- miniDB/table.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/miniDB/table.py b/miniDB/table.py index 2cac563e..a0e7de52 100644 --- a/miniDB/table.py +++ b/miniDB/table.py @@ -238,15 +238,16 @@ def _select_where(self, return_columns, condition=None, distinct=False, order_by if "between" in condition: condition_list=condition.split(" ") if len(condition_list)!=5 or condition_list[1]!="between" or condition_list[3]!="and": - raise Exception("condition containing between keyword must have the following format: column between value1 and value2") + raise Exception("Condition containing between keyword must have the following format: column between value1 and value2") else: try: column=self.column_by_name(condition_list[0]) value1=int(condition_list[2]) value2=int(condition_list[4]) - rows = [ind for ind, x in enumerate(column) if ( get_op('>=', x, value) and get_op('<=', x, value) )] + rows = [ind for ind, x in enumerate(column) if value1 <= x <= value2]#append index of row if x between value1 and value2 except: - print("You need to provide only arithmetical values in order to evaluate between clause") + raise ValueError('You need to provide numeric values inside a between statement') + #print("You need to provide only arithmetical values in order to evaluate between clause") else: # NOT case containsNot=False # flag for not keyword From 8f107533cf17074856b252d18ce4eb4fa55fa8e5 Mon Sep 17 00:00:00 2001 From: Vassiliki Karagianniki <78561945+vassilikikrg@users.noreply.github.com> Date: Mon, 16 Jan 2023 20:50:44 +0200 Subject: [PATCH 04/25] Changed ValueError to TypeError --- miniDB/table.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/miniDB/table.py b/miniDB/table.py index a0e7de52..5cc4d67d 100644 --- a/miniDB/table.py +++ b/miniDB/table.py @@ -246,7 +246,7 @@ def _select_where(self, return_columns, condition=None, distinct=False, order_by value2=int(condition_list[4]) rows = [ind for ind, x in enumerate(column) if value1 <= x <= value2]#append index of row if x between value1 and value2 except: - raise ValueError('You need to provide numeric values inside a between statement') + raise TypeError('You need to provide numeric values inside a between statement') #print("You need to provide only arithmetical values in order to evaluate between clause") else: # NOT case From 4ecc4f818d12cfd2462d8477e0a3c7107430f34e Mon Sep 17 00:00:00 2001 From: Jimsidi Date: Thu, 26 Jan 2023 18:38:47 +0200 Subject: [PATCH 05/25] Implementation of AND and OR operator + started working on unique constraint Other changes -reworked NOT operation -replaced removesuffix with replace function to be compatible with more py versions -wrote new reverse operator function so it doesn't mess with the one built for join -in table.py: added try-catch block for handling primary key exceptions --- mdb.py | 19 ++++++++++------ miniDB/database.py | 12 +++++++++- miniDB/misc.py | 14 +++++++++++- miniDB/table.py | 56 ++++++++++++++++++++++++++++++++++------------ 4 files changed, 78 insertions(+), 23 deletions(-) diff --git a/mdb.py b/mdb.py index 00668a02..08af2e56 100644 --- a/mdb.py +++ b/mdb.py @@ -88,7 +88,7 @@ def create_query_plan(query, keywords, action): dic['desc'] = True else: dic['desc'] = False - dic['order by'] = dic['order by'].removesuffix(' asc').removesuffix(' desc') + dic['order by'] = dic['order by'].replace(' asc','').replace(' desc','') else: dic['desc'] = None @@ -101,10 +101,15 @@ def create_query_plan(query, keywords, action): dic['column_names'] = ','.join([val[0] for val in arglist]) dic['column_types'] = ','.join([val[1] for val in arglist]) if 'primary key' in args: - arglist = args[1:-1].split(' ') - dic['primary key'] = arglist[arglist.index('primary')-2] + arglist = args[1:-1].split(' ') # remove () from create table arguments statement and split into keywords + + dic['primary key'] = arglist[arglist.index('primary')-2] # search for index of keyword primary, and get index of the PK (-2 because -1 is type) else: dic['primary key'] = None + if 'unique' in args: + arglist = args[1:-1].split(' ') + matched_indexes = [i for i, kw in enumerate(arglist) if kw == 'unique'] + dic['unique'] =[arglist[m-2] for m in matched_indexes] # list of columns with unique keyword if action=='import': dic = {'import table' if key=='import' else key: val for key, val in dic.items()} @@ -215,7 +220,7 @@ def interpret_meta(command): cdb - change/create database rmdb - delete database """ - action = command.split(' ')[0].removesuffix(';') + action = command.split(' ')[0].replace(';') db_name = db._name if search_between(command, action,';')=='' else search_between(command, action,';') @@ -225,10 +230,10 @@ def interpret_meta(command): verbose = False def list_databases(db_name): - [print(fold.removesuffix('_db')) for fold in os.listdir('dbdata')] + [print(fold.replace('_db','')) for fold in os.listdir('dbdata')] def list_tables(db_name): - [print(pklf.removesuffix('.pkl')) for pklf in os.listdir(f'dbdata/{db_name}_db') if pklf.endswith('.pkl')\ + [print(pklf.replace('.pkl','')) for pklf in os.listdir(f'dbdata/{db_name}_db') if pklf.endswith('.pkl')\ and not pklf.startswith('meta')] def change_db(db_name): @@ -286,7 +291,7 @@ def remove_db(db_name): try: if line=='exit': break - if line.split(' ')[0].removesuffix(';') in ['lsdb', 'lstb', 'cdb', 'rmdb']: + if line.split(' ')[0].replace(';','') in ['lsdb', 'lstb', 'cdb', 'rmdb']: interpret_meta(line) elif line.startswith('explain'): dic = interpret(line.removeprefix('explain ')) diff --git a/miniDB/database.py b/miniDB/database.py index 0b8a4e94..2258570e 100644 --- a/miniDB/database.py +++ b/miniDB/database.py @@ -357,9 +357,16 @@ def select(self, columns, table_name, condition, distinct=None, order_by=None, \ if isinstance(table_name,Table): # is table in database? return table_name._select_where(columns, condition, distinct, order_by, desc, limit) + condition_AND=False + condition_OR=False + if condition is not None: # get column of where clause if "between" in condition: #if condition contains the keyword "between",then condition has the format of table.column between value1 and value2 condition_column=condition.split(" ")[0] + elif "and" in condition: + condition_AND=True + elif "or" in condition: + condition_OR=True else: condition_column = split_condition(condition)[0] else: @@ -369,7 +376,10 @@ def select(self, columns, table_name, condition, distinct=None, order_by=None, \ # self.lock_table(table_name, mode='x') if self.is_locked(table_name): return - if self._has_index(table_name) and condition_column==self.tables[table_name].column_names[self.tables[table_name].pk_idx]: + + if condition_AND is True or condition_OR is True: + table = self.tables[table_name]._select_where(columns, condition, distinct, order_by, desc, limit) + elif self._has_index(table_name) and condition_column==self.tables[table_name].column_names[self.tables[table_name].pk_idx]: index_name = self.select('*', 'meta_indexes', f'table_name={table_name}', return_object=True).column_by_name('index_name')[0] bt = self._load_idx(index_name) table = self.tables[table_name]._select_where_with_btree(columns, bt, condition, distinct, order_by, desc, limit) diff --git a/miniDB/misc.py b/miniDB/misc.py index 5069624f..550969bc 100644 --- a/miniDB/misc.py +++ b/miniDB/misc.py @@ -43,7 +43,19 @@ def split_condition(condition): def reverse_op(op): ''' - Reverse the operator given (REWORKED WITH NOT OPERATOR) + Reverse the operator given + ''' + return { + '>' : '<', + '>=' : '<=', + '<' : '>', + '<=' : '>=', + '=' : '=' + }.get(op) + +def reverse_op_not(op): + ''' + Reverse the operator given for NOT keyword ''' return { '>' : '<=', diff --git a/miniDB/table.py b/miniDB/table.py index 5cc4d67d..dc11e357 100644 --- a/miniDB/table.py +++ b/miniDB/table.py @@ -6,7 +6,7 @@ sys.path.append(f'{os.path.dirname(os.path.dirname(os.path.abspath(__file__)))}/miniDB') -from misc import get_op, reverse_op, split_condition +from misc import get_op, reverse_op_not, split_condition class Table: @@ -26,7 +26,7 @@ class Table: - a dictionary that includes the appropriate info (all the attributes in __init__) ''' - def __init__(self, name=None, column_names=None, column_types=None, primary_key=None, load=None): + def __init__(self, name=None, column_names=None, column_types=None, primary_key=None,unique=None, load=None): if load is not None: # if load is a dict, replace the object dict with it (replaces the object with the specified one) @@ -70,6 +70,13 @@ def __init__(self, name=None, column_names=None, column_types=None, primary_key= self.pk = primary_key # self._update() + # if unique is set, keep its index as an attribute + if unique is not None: + self.unique_idx = self.column_names.index(unique) + else: + self.unique_idx = None + + self.unique = unique # if any of the name, columns_names and column types are none. return an empty table object def column_by_name(self, column_name): @@ -125,10 +132,13 @@ def _insert(self, row, insert_stack=[]): print(exc) # if value is to be appended to the primary_key column, check that it doesnt alrady exist (no duplicate primary keys) - if i==self.pk_idx and row[i] in self.column_by_name(self.pk): - raise ValueError(f'## ERROR -> Value {row[i]} already exists in primary key column.') - elif i==self.pk_idx and row[i] is None: - raise ValueError(f'ERROR -> The value of the primary key cannot be None.') + try: + if i==self.pk_idx and row[i] in self.column_by_name(self.pk): + raise ValueError(f'## ERROR -> Value {row[i]} already exists in primary key column "{self.pk}".') + elif i==self.pk_idx and row[i] is None: + raise ValueError(f'ERROR -> The value of the primary key cannot be None.') + except Exception as e: + print(str(e)) #added this to print error # if insert_stack is not empty, append to its last index if insert_stack != []: @@ -248,15 +258,33 @@ def _select_where(self, return_columns, condition=None, distinct=False, order_by except: raise TypeError('You need to provide numeric values inside a between statement') #print("You need to provide only arithmetical values in order to evaluate between clause") - else: + # AND case + elif "and" in condition: + all_rows=[] + seperated_conditions=condition.split(" and ") + for cond in seperated_conditions: #select all the rows that match each condition + column_name, operator, value = self._parse_condition(cond) + column = self.column_by_name(column_name) + all_rows+=[ind for ind, x in enumerate(column) if get_op(operator, x, value)]#add indexes to all_rows + rows=[i for i in all_rows if all_rows.count(i) > 1] #Remove unique values from all_rows list and keep only duplicates which match both conditions + # OR case + elif "or" in condition: + all_rows=[] + seperated_conditions=condition.split(" or ") + for cond in seperated_conditions: #select all the rows that match each condition + column_name, operator, value = self._parse_condition(cond) + column = self.column_by_name(column_name) + all_rows+=[ind for ind, x in enumerate(column) if get_op(operator, x, value)]#add indexes to all_rows + rows=[*set(all_rows)] #keep all rows that match either of the conditions but first remove the double values + elif "not" in condition: # NOT case - containsNot=False # flag for not keyword - if len(condition.split(' '))>1: # split will have more than one item - condition=condition.replace('not ','') # remove not keyword - containsNot=True + condition=condition.replace('not ','') # remove not keyword + column_name, operator, value = self._parse_condition(condition) + operator=reverse_op_not(operator) # reverse operator + column = self.column_by_name(column_name) + rows = [ind for ind, x in enumerate(column) if get_op(operator, x, value)] + else: column_name, operator, value = self._parse_condition(condition) - if containsNot: # reverse operator if flag true - operator=reverse_op(operator) column = self.column_by_name(column_name) rows = [ind for ind, x in enumerate(column) if get_op(operator, x, value)] @@ -292,7 +320,7 @@ def _select_where(self, return_columns, condition=None, distinct=False, order_by s_table.data = [row for row in s_table.data if any(row)][:int(limit)] return s_table - + def _select_where_with_btree(self, return_columns, bt, condition, distinct=False, order_by=None, desc=True, limit=None): From db521471b5ae5fb9f25f91e5c8e398a2f248a9a2 Mon Sep 17 00:00:00 2001 From: Antonyfrtz Date: Thu, 26 Jan 2023 19:24:24 +0200 Subject: [PATCH 06/25] Bug Fixes for unique and AND statement - AND statement now works for multiple ANDs and will not produce duplicate values - fixed bug where unique keys would only be found if they were the last argument - also changed removeprefix to replace --- mdb.py | 6 +++--- miniDB/table.py | 6 ++++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/mdb.py b/mdb.py index 08af2e56..895e610f 100644 --- a/mdb.py +++ b/mdb.py @@ -108,7 +108,7 @@ def create_query_plan(query, keywords, action): dic['primary key'] = None if 'unique' in args: arglist = args[1:-1].split(' ') - matched_indexes = [i for i, kw in enumerate(arglist) if kw == 'unique'] + matched_indexes = [i for i, kw in enumerate(arglist) if kw in ('unique', 'unique,')] dic['unique'] =[arglist[m-2] for m in matched_indexes] # list of columns with unique keyword if action=='import': @@ -265,7 +265,7 @@ def remove_db(db_name): for line in open(fname, 'r').read().splitlines(): if line.startswith('--'): continue if line.startswith('explain'): - dic = interpret(line.removeprefix('explain ')) + dic = interpret(line.replace('explain ','')) pprint(dic, sort_dicts=False) else : dic = interpret(line.lower()) @@ -294,7 +294,7 @@ def remove_db(db_name): if line.split(' ')[0].replace(';','') in ['lsdb', 'lstb', 'cdb', 'rmdb']: interpret_meta(line) elif line.startswith('explain'): - dic = interpret(line.removeprefix('explain ')) + dic = interpret(line.replace('explain ','')) pprint(dic, sort_dicts=False) else: dic = interpret(line) diff --git a/miniDB/table.py b/miniDB/table.py index dc11e357..87fbaacf 100644 --- a/miniDB/table.py +++ b/miniDB/table.py @@ -72,7 +72,8 @@ def __init__(self, name=None, column_names=None, column_types=None, primary_key= # if unique is set, keep its index as an attribute if unique is not None: - self.unique_idx = self.column_names.index(unique) + for u in unique: + self.unique_idx += self.column_names.index(u) else: self.unique_idx = None @@ -266,7 +267,8 @@ def _select_where(self, return_columns, condition=None, distinct=False, order_by column_name, operator, value = self._parse_condition(cond) column = self.column_by_name(column_name) all_rows+=[ind for ind, x in enumerate(column) if get_op(operator, x, value)]#add indexes to all_rows - rows=[i for i in all_rows if all_rows.count(i) > 1] #Remove unique values from all_rows list and keep only duplicates which match both conditions + # Keep the rows from the all_rows list that appear as many times as the number of conditions in the AND clauses + rows=[*set(i for i in all_rows if all_rows.count(i) > len(seperated_conditions)-1)] # OR case elif "or" in condition: all_rows=[] From a6775f22d3594d0cb495ef70a4a206e8c7c90af2 Mon Sep 17 00:00:00 2001 From: Vassiliki Karagianniki <78561945+vassilikikrg@users.noreply.github.com> Date: Mon, 30 Jan 2023 18:19:37 +0200 Subject: [PATCH 07/25] General bug fixes related to unique -added 'unique' keyword to create_table function of database.py -fixed non initialized list self.unique_idx in table.py -print '#UQ#' for unique columns ( function 'show' in table.py ) --- miniDB/database.py | 4 ++-- miniDB/table.py | 7 ++++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/miniDB/database.py b/miniDB/database.py index 2258570e..97ec0601 100644 --- a/miniDB/database.py +++ b/miniDB/database.py @@ -101,7 +101,7 @@ def _update(self): self._update_meta_insert_stack() - def create_table(self, name, column_names, column_types, primary_key=None, load=None): + def create_table(self, name, column_names, column_types, primary_key=None,unique=None, load=None): ''' This method create a new table. This table is saved and can be accessed via db_object.tables['table_name'] or db_object.table_name @@ -113,7 +113,7 @@ def create_table(self, name, column_names, column_types, primary_key=None, load= load: boolean. Defines table object parameters as the name of the table and the column names. ''' # print('here -> ', column_names.split(',')) - self.tables.update({name: Table(name=name, column_names=column_names.split(','), column_types=column_types.split(','), primary_key=primary_key, load=load)}) + self.tables.update({name: Table(name=name, column_names=column_names.split(','), column_types=column_types.split(','), primary_key=primary_key,unique=unique, load=load)}) # self._name = Table(name=name, column_names=column_names, column_types=column_types, load=load) # check that new dynamic var doesnt exist already # self.no_of_tables += 1 diff --git a/miniDB/table.py b/miniDB/table.py index 87fbaacf..40a0ff99 100644 --- a/miniDB/table.py +++ b/miniDB/table.py @@ -72,8 +72,9 @@ def __init__(self, name=None, column_names=None, column_types=None, primary_key= # if unique is set, keep its index as an attribute if unique is not None: + self.unique_idx=[] for u in unique: - self.unique_idx += self.column_names.index(u) + self.unique_idx.append(self.column_names.index(u)) else: self.unique_idx = None @@ -587,6 +588,10 @@ def show(self, no_of_rows=None, is_locked=False): if self.pk_idx is not None: # table has a primary key, add PK next to the appropriate column headers[self.pk_idx] = headers[self.pk_idx]+' #PK#' + if self.unique_idx is not None: + # table has unique columns, add UQ next to the appropriate column + for i in self.unique_idx: + headers[i] = headers[i]+' #UQ#' # detect the rows that are no tfull of nones (these rows have been deleted) # if we dont skip these rows, the returning table has empty rows at the deleted positions non_none_rows = [row for row in self.data if any(row)] From de103f3e48ed919e4be88c395789feefc82697f4 Mon Sep 17 00:00:00 2001 From: Vassiliki Karagianniki <78561945+vassilikikrg@users.noreply.github.com> Date: Tue, 31 Jan 2023 00:22:06 +0200 Subject: [PATCH 08/25] CREATE TABLE statement enriched with 'unique' kw -in table.py: check if duplicate key value violates unique constraint during the insertion of new values in any of the unique columns of a table --- mdb.py | 4 ++-- miniDB/table.py | 12 ++++++++++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/mdb.py b/mdb.py index 895e610f..b91e1b37 100644 --- a/mdb.py +++ b/mdb.py @@ -7,8 +7,8 @@ import shutil sys.path.append('miniDB') -from database import Database -from table import Table +from miniDB.database import Database +from miniDB.table import Table # art font is "big" art = ''' _ _ _____ ____ diff --git a/miniDB/table.py b/miniDB/table.py index 40a0ff99..b6131b4e 100644 --- a/miniDB/table.py +++ b/miniDB/table.py @@ -142,6 +142,18 @@ def _insert(self, row, insert_stack=[]): except Exception as e: print(str(e)) #added this to print error + # if value is to be appended to a unique column, check that it doesnt alrady exist (no duplicate values) + try: + if self.unique is not None: + for count,uq_idx in enumerate(self.unique_idx): + if i==uq_idx and row[i] in self.column_by_name(self.unique[count]): + raise ValueError(f'## ERROR -> Value {row[i]} already exists in unique column "{self.unique[count]}".') + elif i==uq_idx and row[i] is None: + raise ValueError(f'ERROR -> The value of a unique column cannot be None.') + except Exception as e: + print(str(e)) #added this to print error + + # if insert_stack is not empty, append to its last index if insert_stack != []: self.data[insert_stack[-1]] = row From a65f417a75ea7515dbf669f239c0f54751a88ac7 Mon Sep 17 00:00:00 2001 From: Vassiliki Karagianniki <78561945+vassilikikrg@users.noreply.github.com> Date: Wed, 1 Feb 2023 16:32:46 +0200 Subject: [PATCH 09/25] Btree indexing over unique columns -in mdb.py: added 'index_column' keyword in create index statement that handles both primary key and unique column cases -in database.py: 1.Alteration of meta_indexes table (init function): added index_colum column in order to save the column on which index is created 2.Alteration of the elif statement of select function that checks if database contains an index for the table and the column on which we perform the select query 3.Creation of btree index on both pk and unique columns -> in create_index function: if index_column is not specified by the user,we set pk as the index column(in case there is a pk), else we check that the provided index_column is a unique column. we insert a record that also contains index_column to the meta_indexes table and we construct the index for the specified column -> in _construct_index function: we create the nodes of the btree that contain all the values of the index_column -> in _has_index function: added index_column to arguments (to be continued) -in table.py:(_select_where_with_btree function) we find all unique column names and if the column in condition is not a primary key or a unique column,we abort the select --- mdb.py | 9 ++++++++- miniDB/database.py | 48 +++++++++++++++++++++++++++++++--------------- miniDB/table.py | 10 +++++++--- 3 files changed, 48 insertions(+), 19 deletions(-) diff --git a/mdb.py b/mdb.py index b91e1b37..755a5fb2 100644 --- a/mdb.py +++ b/mdb.py @@ -126,6 +126,13 @@ def create_query_plan(query, keywords, action): else: dic['force'] = False + if action=='create index': + if '(' and ')' in dic['on']: #if user has specified a column on which we will create the index + l=dic['on'].split(' ') + dic['index_column']=l[2] #key is the specified column + dic['on']=l[0] #now the key contains only the table name + else: + dic['index_column']=None #default case->index column is not specified return dic @@ -220,7 +227,7 @@ def interpret_meta(command): cdb - change/create database rmdb - delete database """ - action = command.split(' ')[0].replace(';') + action = command.split(' ')[0].replace(';','') db_name = db._name if search_between(command, action,';')=='' else search_between(command, action,';') diff --git a/miniDB/database.py b/miniDB/database.py index 97ec0601..e550a788 100644 --- a/miniDB/database.py +++ b/miniDB/database.py @@ -54,7 +54,7 @@ def __init__(self, name, load=True, verbose = True): self.create_table('meta_length', 'table_name,no_of_rows', 'str,int') self.create_table('meta_locks', 'table_name,pid,mode', 'str,int,str') self.create_table('meta_insert_stack', 'table_name,indexes', 'str,list') - self.create_table('meta_indexes', 'table_name,index_name', 'str,str') + self.create_table('meta_indexes', 'table_name,index_name,index_column', 'str,str,str')#added index_colum column in order to keep the column on which index is created self.save_database() def save_database(self): @@ -379,7 +379,10 @@ def select(self, columns, table_name, condition, distinct=None, order_by=None, \ if condition_AND is True or condition_OR is True: table = self.tables[table_name]._select_where(columns, condition, distinct, order_by, desc, limit) - elif self._has_index(table_name) and condition_column==self.tables[table_name].column_names[self.tables[table_name].pk_idx]: + #comment!!!!! + + #elif self._has_index(table_name) and (condition_column==self.tables[table_name].column_names[self.tables[table_name].pk_idx] or condition_column in self.tables[table_name].column_names[self.tables[table_name].unique_idx]): + elif self._has_index(table_name,condition_column): index_name = self.select('*', 'meta_indexes', f'table_name={table_name}', return_object=True).column_by_name('index_name')[0] bt = self._load_idx(index_name) table = self.tables[table_name]._select_where_with_btree(columns, bt, condition, distinct, order_by, desc, limit) @@ -663,7 +666,7 @@ def _update_meta_insert_stack_for_tb(self, table_name, new_stack): # indexes - def create_index(self, index_name, table_name, index_type='btree'): + def create_index(self, index_name, table_name, index_type='btree',index_column=None): ''' Creates an index on a specified table with a given name. Important: An index can only be created on a primary key (the user does not specify the column). @@ -671,33 +674,48 @@ def create_index(self, index_name, table_name, index_type='btree'): Args: table_name: string. Table name (must be part of database). index_name: string. Name of the created index. + index_column: string. Name of the column on which we create the index ''' - if self.tables[table_name].pk_idx is None: # if no primary key, no index - raise Exception('Cannot create index. Table has no primary key.') - if index_name not in self.tables['meta_indexes'].column_by_name('index_name'): + + #if index_column not specified by user,index creation on primary key + if index_column is None: + if self.tables[table_name].pk_idx is not None: + index_column=self.tables[table_name].pk + else: + raise Exception('Cannot create index. Table has no primary key.')# if no primary key, no index + else: + try: + col_names=self.tables[table_name].column_names #we find the column names of the table + index_column_idx=col_names.index(index_column) #the index of the given column(on which we create the index) in the list of column names + if index_column_idx not in self.tables[table_name].unique_idx: + raise Exception('Cannot create index on a column that is not unique.') + except: + raise ValueError('The requested column to index does not exist in the table.') + + if index_name not in self.tables['meta_indexes'].column_by_name('index_name'):#if there isn't already an index with the same name # currently only btree is supported. This can be changed by adding another if. if index_type=='btree': logging.info('Creating Btree index.') - # insert a record with the name of the index and the table on which it's created to the meta_indexes table - self.tables['meta_indexes']._insert([table_name, index_name]) + # insert a record with the name of the index,the table and the column on which it's created to the meta_indexes table + self.tables['meta_indexes']._insert([table_name, index_name,index_column]) # crate the actual index - self._construct_index(table_name, index_name) + self._construct_index(table_name, index_name,index_column) self.save_database() else: raise Exception('Cannot create index. Another index with the same name already exists.') - def _construct_index(self, table_name, index_name): + def _construct_index(self, table_name, index_name,index_column): ''' Construct a btree on a table and save. Args: table_name: string. Table name (must be part of database). index_name: string. Name of the created index. + index_column: string. Name of the column on which we create the index ''' bt = Btree(3) # 3 is arbitrary - - # for each record in the primary key of the table, insert its value and index to the btree - for idx, key in enumerate(self.tables[table_name].column_by_name(self.tables[table_name].pk)): + # for each record of the table, insert its column value and index to the btree + for idx, key in enumerate(self.tables[table_name].column_by_name(index_column)): if key is None: continue bt.insert(key, idx) @@ -705,14 +723,14 @@ def _construct_index(self, table_name, index_name): self._save_index(index_name, bt) - def _has_index(self, table_name): + def _has_index(self, table_name,index_column): ''' Check whether the specified table's primary key column is indexed. Args: table_name: string. Table name (must be part of database). ''' - return table_name in self.tables['meta_indexes'].column_by_name('table_name') + return table_name in self.tables['meta_indexes'].column_by_name('table_name') def _save_index(self, index_name, index): ''' diff --git a/miniDB/table.py b/miniDB/table.py index b6131b4e..04edd5b1 100644 --- a/miniDB/table.py +++ b/miniDB/table.py @@ -348,9 +348,13 @@ def _select_where_with_btree(self, return_columns, bt, condition, distinct=False column_name, operator, value = self._parse_condition(condition) - # if the column in condition is not a primary key, abort the select - if column_name != self.column_names[self.pk_idx]: - print('Column is not PK. Aborting') + unique_col_names=[] + for n in self.column_names[self.unique_idx]: + unique_col_names.append(self.column_names[n]) #find all unique column names + + # if the column in condition is not a primary key or a unique column, abort the select + if column_name != self.column_names[self.pk_idx] or column_name not in unique_col_names: + print('Column is neither a PK nor unique. Aborting') # here we run the same select twice, sequentially and using the btree. # we then check the results match and compare performance (number of operation) From 7da0b33a8a2942082b0f89a681df6c5d5ab63f8f Mon Sep 17 00:00:00 2001 From: Antonyfrtz Date: Wed, 1 Feb 2023 18:54:31 +0200 Subject: [PATCH 10/25] Worked on BTree Index checking - Code cannot be tested until bug fixes for BTree are implemented - TBD --- miniDB/database.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/miniDB/database.py b/miniDB/database.py index e550a788..6be2c414 100644 --- a/miniDB/database.py +++ b/miniDB/database.py @@ -725,12 +725,21 @@ def _construct_index(self, table_name, index_name,index_column): def _has_index(self, table_name,index_column): ''' - Check whether the specified table's primary key column is indexed. + Check whether the specified table's column is indexed. Args: table_name: string. Table name (must be part of database). + index_column: string. Column name to be checked for index. + + Returns: + The index column if it is indexed, None otherwise. ''' - return table_name in self.tables['meta_indexes'].column_by_name('table_name') + # This is probably wrong, should be tested after bug fixes in index creation + index_columns = self.tables['meta_indexes'] # load meta_indexes table as array + for row in index_columns: + if row['table_name']==table_name and row['index_column']==index_column: # check for our table and index + return table_name,row['index_column'] + return False def _save_index(self, index_name, index): ''' From e9c99962fc6febbb75982fc25f8eded0294d9a55 Mon Sep 17 00:00:00 2001 From: Antonyfrtz Date: Thu, 2 Feb 2023 02:46:34 +0200 Subject: [PATCH 11/25] hasIndex now operational and tested - Should check whether or not to use an index if select has no specified column - Also need to handle PK case better --- miniDB/database.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/miniDB/database.py b/miniDB/database.py index 6be2c414..60b82aba 100644 --- a/miniDB/database.py +++ b/miniDB/database.py @@ -734,11 +734,13 @@ def _has_index(self, table_name,index_column): Returns: The index column if it is indexed, None otherwise. ''' - # This is probably wrong, should be tested after bug fixes in index creation - index_columns = self.tables['meta_indexes'] # load meta_indexes table as array - for row in index_columns: - if row['table_name']==table_name and row['index_column']==index_column: # check for our table and index - return table_name,row['index_column'] + # Get all the data from meta_indexes - this is most likely a small table so this is not a problem + data=self.tables['meta_indexes'].data + for row in data: + # Check rows that have a corresponding index to the selected table and column combination + if row[0]==table_name and row[2]==index_column: + print(table_name,"has an index on",index_column) + return True return False def _save_index(self, index_name, index): From 9dcc7a955373c31a95caa1da53cca0433a455623 Mon Sep 17 00:00:00 2001 From: Vassiliki Karagianniki <78561945+vassilikikrg@users.noreply.github.com> Date: Tue, 7 Feb 2023 14:54:05 +0200 Subject: [PATCH 12/25] Unique constraint duplicates and testing -table.py:(_insert function):fixed violation of unique columns constraint (no duplicates), (function _select_where_with_btree): if the column in condition is not a primary key or a unique column, abort the btree select else continue searching using index over that column -smallRelationsInsertFile.sql: changed create table statement for classroom table in order to test the creation of index over the unique column 'capacity' --- miniDB/table.py | 103 +++++++++++-------------- sql_files/smallRelationsInsertFile.sql | 2 +- 2 files changed, 48 insertions(+), 57 deletions(-) diff --git a/miniDB/table.py b/miniDB/table.py index 04edd5b1..1dcb2c85 100644 --- a/miniDB/table.py +++ b/miniDB/table.py @@ -134,25 +134,18 @@ def _insert(self, row, insert_stack=[]): print(exc) # if value is to be appended to the primary_key column, check that it doesnt alrady exist (no duplicate primary keys) - try: - if i==self.pk_idx and row[i] in self.column_by_name(self.pk): - raise ValueError(f'## ERROR -> Value {row[i]} already exists in primary key column "{self.pk}".') - elif i==self.pk_idx and row[i] is None: - raise ValueError(f'ERROR -> The value of the primary key cannot be None.') - except Exception as e: - print(str(e)) #added this to print error - - # if value is to be appended to a unique column, check that it doesnt alrady exist (no duplicate values) - try: - if self.unique is not None: - for count,uq_idx in enumerate(self.unique_idx): - if i==uq_idx and row[i] in self.column_by_name(self.unique[count]): - raise ValueError(f'## ERROR -> Value {row[i]} already exists in unique column "{self.unique[count]}".') - elif i==uq_idx and row[i] is None: - raise ValueError(f'ERROR -> The value of a unique column cannot be None.') - except Exception as e: - print(str(e)) #added this to print error - + + if i==self.pk_idx and row[i] in self.column_by_name(self.pk): + raise ValueError(f'## ERROR -> Value {row[i]} already exists in primary key column "{self.pk}".') + elif i==self.pk_idx and row[i] is None: + raise ValueError(f'ERROR -> The value of the primary key cannot be None.') + + # if value is to be appended to a unique column, check that it doesnt alrady exist + if self.unique_idx is not None: + if i in self.unique_idx and row[i] in self.column_by_name(self.unique[self.unique_idx.index(i)]): + raise ValueError(f'## ERROR -> Value {row[i]} already exists in unique column.') + elif i in self.unique_idx and row[i] is None: + raise ValueError(f'ERROR -> The value of a unique field cannot be None.') # if insert_stack is not empty, append to its last index if insert_stack != []: @@ -348,52 +341,50 @@ def _select_where_with_btree(self, return_columns, bt, condition, distinct=False column_name, operator, value = self._parse_condition(condition) - unique_col_names=[] - for n in self.column_names[self.unique_idx]: - unique_col_names.append(self.column_names[n]) #find all unique column names + #if the column in condition is a primary key or a unique column,continue the select + if column_name in self.unique or (self.pk is not None and column_name==self.pk) : + # here we run the same select twice, sequentially and using the btree. + # we then check the results match and compare performance (number of operation) + column = self.column_by_name(column_name) - # if the column in condition is not a primary key or a unique column, abort the select - if column_name != self.column_names[self.pk_idx] or column_name not in unique_col_names: - print('Column is neither a PK nor unique. Aborting') + # sequential + rows1 = [] + opsseq = 0 + for ind, x in enumerate(column): + opsseq+=1 + if get_op(operator, x, value): + rows1.append(ind) - # here we run the same select twice, sequentially and using the btree. - # we then check the results match and compare performance (number of operation) - column = self.column_by_name(column_name) - - # sequential - rows1 = [] - opsseq = 0 - for ind, x in enumerate(column): - opsseq+=1 - if get_op(operator, x, value): - rows1.append(ind) - - # btree find - rows = bt.find(operator, value) + # btree find + bt.show() + rows = bt.find(operator, value) - try: - k = int(limit) - except TypeError: - k = None - # same as simple select from now on - rows = rows[:k] - # TODO: this needs to be dumbed down - dict = {(key):([[self.data[i][j] for j in return_cols] for i in rows] if key=="data" else value) for key,value in self.__dict__.items()} + try: + k = int(limit) + except TypeError: + k = None + # same as simple select from now on + rows = rows[:k] + # TODO: this needs to be dumbed down + dict = {(key):([[self.data[i][j] for j in return_cols] for i in rows] if key=="data" else value) for key,value in self.__dict__.items()} - dict['column_names'] = [self.column_names[i] for i in return_cols] - dict['column_types'] = [self.column_types[i] for i in return_cols] + dict['column_names'] = [self.column_names[i] for i in return_cols] + dict['column_types'] = [self.column_types[i] for i in return_cols] - s_table = Table(load=dict) + s_table = Table(load=dict) - s_table.data = list(set(map(lambda x: tuple(x), s_table.data))) if distinct else s_table.data + s_table.data = list(set(map(lambda x: tuple(x), s_table.data))) if distinct else s_table.data - if order_by: - s_table.order_by(order_by, desc) + if order_by: + s_table.order_by(order_by, desc) - if isinstance(limit,str): - s_table.data = [row for row in s_table.data if row is not None][:int(limit)] + if isinstance(limit,str): + s_table.data = [row for row in s_table.data if row is not None][:int(limit)] - return s_table + return s_table + else: + print('Column is neither a PK nor unique. Aborting')# if the column in condition is not a primary key or a unique column, abort the select + return def order_by(self, column_name, desc=True): ''' diff --git a/sql_files/smallRelationsInsertFile.sql b/sql_files/smallRelationsInsertFile.sql index d05d81b9..117cd8e2 100644 --- a/sql_files/smallRelationsInsertFile.sql +++ b/sql_files/smallRelationsInsertFile.sql @@ -1,4 +1,4 @@ -create table classroom (building str, room_number str, capacity int); +create table classroom (building str, room_number str, capacity int unique); create table department (dept_name str primary key, building str, budget int); create table course (course_id str primary key, title str, dept_name str, credits int); create table instructor (ID str primary key, name str, dept_name str, salary int); From b570a7d7744a1b2b5e06137daa9308fdbe26599f Mon Sep 17 00:00:00 2001 From: Antonyfrtz Date: Wed, 8 Feb 2023 15:27:47 +0200 Subject: [PATCH 13/25] Created base template for hash index - Created file hash.py - Added classes and methods required - Added h(key) code --- miniDB/hash.py | 49 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 miniDB/hash.py diff --git a/miniDB/hash.py b/miniDB/hash.py new file mode 100644 index 00000000..ee7b3680 --- /dev/null +++ b/miniDB/hash.py @@ -0,0 +1,49 @@ +''' +https://gunet2.cs.unipi.gr/modules/document/file.php/TMC110/Lectures/05-FilesIndexing.pdf Slide 20 (Extendible Hashing) +''' +import binascii # we will convert values to hex bytes and then take the modulo of the corresponding int with the key + +class Hash: + key=2047 # 1st mersenne prime + def __init__(self, b): + ''' + The tree abstraction. + ''' + self.b = b # blocking factor + self.buckets = [] # list of nodes. Every new node is appended here + + def insert(self, value, ptr): + ''' + Insert the value to the appropriate node. + + Args: + value: float. The input value. + ptr: float. The ptr of the inserted value (e.g. its index). + ''' + arr=["dimitris","vassiliki","antonis"] + # Converting String to integer + for val in arr: + num=int(binascii.hexlify(bytes(val,'utf-8')), 16) + print(num) + #convert back to string + print(binascii.unhexlify(format(num, "x").encode("utf-8")).decode("utf-8")) + + h_key=num%key #we will get the MSB from this value + + def _search(self, value, return_ops=False): + pass + + + def split(self, node_id): + # split buckets + pass + + def show(self): + # optionally, implement hash printing + pass + +class Bucket: + def __init__(self, r): + self.r = r # number of records held in block (blocking factor) + self.values = [] if values is None else values # Values (the data from the pk column) + self.ptrs = [] if ptrs is None else ptrs # ptrs (the indexes of each datapoint or the index of another bucket) \ No newline at end of file From 8e4dd48f52e596a8ecf635efe2e5cc1c166ca809 Mon Sep 17 00:00:00 2001 From: Antonyfrtz Date: Wed, 8 Feb 2023 20:09:45 +0200 Subject: [PATCH 14/25] Implementing hash indexing (without split) Currently able to construct a hash index that uses buckets which are assigned to by pointers in a hash_prefix dictionary The values in the buckets are also held in a dictionary - For database.py: - - Now supports index keyword of type hash in a similar fashion as for btrees - - A construct_hash_index function has been created to instantiate the hash class and save it to meta_indexes after inserting values - For hash.py: - - Init now takes the key as an argument, and hash attributes are instantiated here, alongside the required buckets - - Insert computes a binary value to put in the buckets - - Bucket class fields instantiated. Values passed into bucket as dictionary instead of list to keep val-pointer combo // TODO Bucket splitting has not yet been implemented and will cause overflow --- miniDB/database.py | 28 ++++++++++++++++++++-- miniDB/hash.py | 58 ++++++++++++++++++++++++++-------------------- 2 files changed, 59 insertions(+), 27 deletions(-) diff --git a/miniDB/database.py b/miniDB/database.py index 60b82aba..ca177922 100644 --- a/miniDB/database.py +++ b/miniDB/database.py @@ -13,6 +13,7 @@ from joins import Inlj, Smj from btree import Btree +from hash import Hash # file name might need change from misc import split_condition from table import Table @@ -693,14 +694,20 @@ def create_index(self, index_name, table_name, index_type='btree',index_column=N raise ValueError('The requested column to index does not exist in the table.') if index_name not in self.tables['meta_indexes'].column_by_name('index_name'):#if there isn't already an index with the same name - # currently only btree is supported. This can be changed by adding another if. if index_type=='btree': logging.info('Creating Btree index.') # insert a record with the name of the index,the table and the column on which it's created to the meta_indexes table self.tables['meta_indexes']._insert([table_name, index_name,index_column]) - # crate the actual index + # create the actual index self._construct_index(table_name, index_name,index_column) self.save_database() + if index_type=='hash': + logging.info('Creating Hash index.') + # insert a record with the name of the index,the table and the column on which it's created to the meta_indexes table + self.tables['meta_indexes']._insert([table_name, index_name,index_column]) + # create the actual index + self._construct_hash_index(table_name, index_name,index_column) + self.save_database() else: raise Exception('Cannot create index. Another index with the same name already exists.') @@ -722,6 +729,23 @@ def _construct_index(self, table_name, index_name,index_column): # save the btree self._save_index(index_name, bt) + def _construct_hash_index(self, table_name, index_name,index_column): + ''' + Construct a hash index on a table and save. + + Args: + table_name: string. Table name (must be part of database). + index_name: string. Name of the created index. + index_column: string. Name of the column on which we create the index + ''' + hash_table=Hash(15) # argument is blocking factor + # for each record of the table, insert its column value and index to the btree + for idx, key in enumerate(self.tables[table_name].column_by_name(index_column)): + if key is None: + continue + hash_table.insert(key, idx) + # save the hashtable + self._save_index(index_name, hash_table) def _has_index(self, table_name,index_column): ''' diff --git a/miniDB/hash.py b/miniDB/hash.py index ee7b3680..72ff5793 100644 --- a/miniDB/hash.py +++ b/miniDB/hash.py @@ -4,31 +4,34 @@ import binascii # we will convert values to hex bytes and then take the modulo of the corresponding int with the key class Hash: - key=2047 # 1st mersenne prime - def __init__(self, b): + + def __init__(self, b, key=2047): # 1st mersenne prime ''' The tree abstraction. ''' self.b = b # blocking factor - self.buckets = [] # list of nodes. Every new node is appended here + self.hash_prefix = {} # search dictionary + self.key = key # hash key + self.MSB = 1 # number of bits used for hash prefix + # create as many buckets as hash prefix keys + self.buckets = [Bucket(self.b) for i in range(2**self.MSB)] # list of buckets. Every value is placed by hashing function in one of these buckets + for i in range(2**self.MSB): + # dictionary keys are binary representations of MSB number range (0 to 2^msb-1) + self.hash_prefix[format(i,'0'+str(self.MSB)+'b')]=self.buckets[i] # map hash to equivalent bucket - def insert(self, value, ptr): - ''' - Insert the value to the appropriate node. - Args: - value: float. The input value. - ptr: float. The ptr of the inserted value (e.g. its index). - ''' - arr=["dimitris","vassiliki","antonis"] - # Converting String to integer - for val in arr: - num=int(binascii.hexlify(bytes(val,'utf-8')), 16) - print(num) - #convert back to string - print(binascii.unhexlify(format(num, "x").encode("utf-8")).decode("utf-8")) - - h_key=num%key #we will get the MSB from this value + def insert(self, value, ptr): + # key to be hashed + h_key=self.calc_hash(value) + # formatted as binary value + bin_val=format(h_key,'011b') + bits=bin_val[0:self.MSB] + selected=self.hash_prefix[bits] # selected bucket + # Check if record fits in the bucket + if (len(selected.data)<=selected.b): + selected.data[value]=ptr + else: + pass # split() def _search(self, value, return_ops=False): pass @@ -36,14 +39,19 @@ def _search(self, value, return_ops=False): def split(self, node_id): # split buckets - pass + pass # :( def show(self): # optionally, implement hash printing pass + + def calc_hash(self,value): + if isinstance(value, str): + value=int(binascii.hexlify(bytes(value,'utf-8')), 16) + return value%self.key #we will get the MSB from this value -class Bucket: - def __init__(self, r): - self.r = r # number of records held in block (blocking factor) - self.values = [] if values is None else values # Values (the data from the pk column) - self.ptrs = [] if ptrs is None else ptrs # ptrs (the indexes of each datapoint or the index of another bucket) \ No newline at end of file + +class Bucket: # kouvadaki + def __init__(self, b): + self.b = b # number of records held in block (blocking factor) + self.data = {} \ No newline at end of file From abecf96a3be83393cf86448a8a47be5a0ddeeb09 Mon Sep 17 00:00:00 2001 From: Jimsidi Date: Fri, 17 Feb 2023 10:40:53 +0200 Subject: [PATCH 15/25] Basic implementation of hash index split function: -split: Implemented the algorithm from the book (Database System Concepts, Book by Avi Silberschatz, Henry F. Korth, and S. Sudarshan (pg. 1197-1203)) --- miniDB/hash.py | 87 +++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 76 insertions(+), 11 deletions(-) diff --git a/miniDB/hash.py b/miniDB/hash.py index 72ff5793..305d753d 100644 --- a/miniDB/hash.py +++ b/miniDB/hash.py @@ -12,46 +12,111 @@ def __init__(self, b, key=2047): # 1st mersenne prime self.b = b # blocking factor self.hash_prefix = {} # search dictionary self.key = key # hash key - self.MSB = 1 # number of bits used for hash prefix + self.i = 1 # number of bits used for hash prefix # create as many buckets as hash prefix keys - self.buckets = [Bucket(self.b) for i in range(2**self.MSB)] # list of buckets. Every value is placed by hashing function in one of these buckets - for i in range(2**self.MSB): + self.buckets = [Bucket(self.b,self.i) for i in range(2**self.i)] # list of buckets. Every value is placed by hashing function in one of these buckets + for j in range(2**self.i): # dictionary keys are binary representations of MSB number range (0 to 2^msb-1) - self.hash_prefix[format(i,'0'+str(self.MSB)+'b')]=self.buckets[i] # map hash to equivalent bucket + self.hash_prefix[format(j,'0'+str(self.i)+'b')]=self.buckets[j] # map hash to equivalent bucket def insert(self, value, ptr): # key to be hashed h_key=self.calc_hash(value) + # formatted as binary value bin_val=format(h_key,'011b') - bits=bin_val[0:self.MSB] - selected=self.hash_prefix[bits] # selected bucket + bits=self.i_MSB(bin_val) + selected=self.hash_prefix[bits] # selected bucket (pointer) # Check if record fits in the bucket if (len(selected.data)<=selected.b): selected.data[value]=ptr else: - pass # split() + self.split(selected,bits) + self.insert(value,ptr) # recursive call after splitting + def _search(self, value, return_ops=False): pass - def split(self, node_id): - # split buckets - pass # :( + def split(self, bucket_j,bits): + ''' + https://www.db-book.com/ (pg. 1197-1203) + ''' + print("hi") + if self.i==bucket_j.i_bucket: # no pointer available for the new bucket + + self.i+=1 #increase hash prefix size (size is doubled) + #replace each element of hash_prefix with 2 elements containing the same index as the original element + new_prefix={} + for key, value in self.hash_prefix.items(): + new_prefix[key+'0']=value + new_prefix[key+'1']=value + self.hash_prefix=new_prefix + + bucket_z=Bucket(self.b,self.i) #new bucket's prefix length is equal to i + self.buckets.append(bucket_z) #add new bucket to the hash index + self.hash_prefix[bits+'1']=len(self.buckets)-1 #the 2nd element that points to the bucket_j now points to the bucket_z + + bucket_j.i_bucket=self.i #bucket_j's prefix length is now equal to i + + elif self.i>bucket_j.i_bucket: # there is a pointer available for the new bucket + bucket_j.i_bucket+=1 # update how many bits we are using + bucket_z=Bucket(self.b,bucket_j.i_bucket) + list_prefixes=[] + for prefix, ptr in self.hash_prefix.items(): + if ptr == bucket_j: + list_prefixes.append(prefix) + print(ptr) + for i in range(len(list_prefixes)//2,len(list_prefixes)): # half the pointers point to the new bucket + + self.hash_prefix[list_prefixes[i]]=len(self.buckets)-1 + + # apply the hash function to each record of bucket j and according to the first i bits,the record stays in bucket_j or moves to bucket_z + j_new_data={} + print(type(bucket_j)) + print(type(bucket_j.data)) + + for r_key,r_value in bucket_j.data.items(): + + # key to be hashed + h_key=self.calc_hash(r_key) + # formatted as binary value + bin_val=format(h_key,'011b') + bits=self.i_MSB(bin_val) + + bucket_insert=self.hash_prefix[bits] + if bucket_insert==bucket_j: + j_new_data[r_key]=r_value + else: + bucket_z.data[r_key]=r_value + + bucket_j.data=j_new_data + def show(self): # optionally, implement hash printing pass def calc_hash(self,value): + ''' + + ''' if isinstance(value, str): value=int(binascii.hexlify(bytes(value,'utf-8')), 16) return value%self.key #we will get the MSB from this value + def i_MSB(self,bin_value): + ''' + returns the i MSB of a binary value + ''' + return bin_value[0:self.i] + + class Bucket: # kouvadaki - def __init__(self, b): + def __init__(self, b,i_bucket): self.b = b # number of records held in block (blocking factor) + self.i_bucket=i_bucket self.data = {} \ No newline at end of file From 18e9489e7ac0b5064636d8d1dacf96128537bf21 Mon Sep 17 00:00:00 2001 From: Vassiliki Karagianniki <78561945+vassilikikrg@users.noreply.github.com> Date: Sat, 18 Feb 2023 12:25:48 +0200 Subject: [PATCH 16/25] additions to hash index+changes to support select with hash index -database.py: --added index_type column to meta_indexes table(init function) --fixed bug in create_index function + altered the insert statement to meta_indexes table when creating a new index(btree or hash) in order to keep the index type --altered _has_index function in order to return the index name and its type -in hash.py: --fixed some bugs,implemented show and search function of Hash Class and added find function to Bucket class also added comments --- miniDB/database.py | 35 ++++++++-------- miniDB/hash.py | 100 ++++++++++++++++++++++++++++++++------------- 2 files changed, 91 insertions(+), 44 deletions(-) diff --git a/miniDB/database.py b/miniDB/database.py index ca177922..4c455131 100644 --- a/miniDB/database.py +++ b/miniDB/database.py @@ -55,7 +55,8 @@ def __init__(self, name, load=True, verbose = True): self.create_table('meta_length', 'table_name,no_of_rows', 'str,int') self.create_table('meta_locks', 'table_name,pid,mode', 'str,int,str') self.create_table('meta_insert_stack', 'table_name,indexes', 'str,list') - self.create_table('meta_indexes', 'table_name,index_name,index_column', 'str,str,str')#added index_colum column in order to keep the column on which index is created + #added index_colum column and index_type in order to keep the column on which index is created and whether it is hash or btree + self.create_table('meta_indexes', 'table_name,index_name,index_column,index_type', 'str,str,str,str') self.save_database() def save_database(self): @@ -685,26 +686,24 @@ def create_index(self, index_name, table_name, index_type='btree',index_column=N else: raise Exception('Cannot create index. Table has no primary key.')# if no primary key, no index else: - try: - col_names=self.tables[table_name].column_names #we find the column names of the table - index_column_idx=col_names.index(index_column) #the index of the given column(on which we create the index) in the list of column names - if index_column_idx not in self.tables[table_name].unique_idx: - raise Exception('Cannot create index on a column that is not unique.') - except: - raise ValueError('The requested column to index does not exist in the table.') + if index_column not in self.tables[table_name].column_names: + raise ValueError('The requested column to index does not exist in the table.') + elif (index_column!=self.tables[table_name].pk) or (self.tables[table_name].unique is not None and index_column not in self.tables[table_name].unique): + raise Exception('Cannot create index on a column that is not unique.') + if index_name not in self.tables['meta_indexes'].column_by_name('index_name'):#if there isn't already an index with the same name if index_type=='btree': logging.info('Creating Btree index.') # insert a record with the name of the index,the table and the column on which it's created to the meta_indexes table - self.tables['meta_indexes']._insert([table_name, index_name,index_column]) + self.tables['meta_indexes']._insert([table_name, index_name,index_column,index_type]) # create the actual index self._construct_index(table_name, index_name,index_column) self.save_database() if index_type=='hash': logging.info('Creating Hash index.') - # insert a record with the name of the index,the table and the column on which it's created to the meta_indexes table - self.tables['meta_indexes']._insert([table_name, index_name,index_column]) + # insert a record with the name of the index,the table,the column on which it's created and the index type to the meta_indexes table + self.tables['meta_indexes']._insert([table_name, index_name,index_column,index_type]) # create the actual index self._construct_hash_index(table_name, index_name,index_column) self.save_database() @@ -726,6 +725,7 @@ def _construct_index(self, table_name, index_name,index_column): if key is None: continue bt.insert(key, idx) + bt.show() # save the btree self._save_index(index_name, bt) @@ -738,16 +738,17 @@ def _construct_hash_index(self, table_name, index_name,index_column): index_name: string. Name of the created index. index_column: string. Name of the column on which we create the index ''' - hash_table=Hash(15) # argument is blocking factor + hash_table=Hash(3) # argument is blocking factor # for each record of the table, insert its column value and index to the btree for idx, key in enumerate(self.tables[table_name].column_by_name(index_column)): if key is None: continue hash_table.insert(key, idx) + hash_table.show() # save the hashtable self._save_index(index_name, hash_table) - def _has_index(self, table_name,index_column): + def _has_index(self, table_name, index_column, index_type): ''' Check whether the specified table's column is indexed. @@ -756,15 +757,17 @@ def _has_index(self, table_name,index_column): index_column: string. Column name to be checked for index. Returns: - The index column if it is indexed, None otherwise. + True if the index column if it is indexed, false otherwise. ''' # Get all the data from meta_indexes - this is most likely a small table so this is not a problem data=self.tables['meta_indexes'].data for row in data: # Check rows that have a corresponding index to the selected table and column combination if row[0]==table_name and row[2]==index_column: - print(table_name,"has an index on",index_column) - return True + index_type=row[3] + print(table_name,"has a",index_type,"index on",index_column) + return row[1],row[3] # return index name and type combo + return False def _save_index(self, index_name, index): diff --git a/miniDB/hash.py b/miniDB/hash.py index 305d753d..32801a0c 100644 --- a/miniDB/hash.py +++ b/miniDB/hash.py @@ -7,7 +7,7 @@ class Hash: def __init__(self, b, key=2047): # 1st mersenne prime ''' - The tree abstraction. + The hash index abstraction. ''' self.b = b # blocking factor self.hash_prefix = {} # search dictionary @@ -29,23 +29,39 @@ def insert(self, value, ptr): bits=self.i_MSB(bin_val) selected=self.hash_prefix[bits] # selected bucket (pointer) # Check if record fits in the bucket - if (len(selected.data)<=selected.b): + if (len(selected.data)bucket_j.i_bucket: # there is a pointer available for the new bucket - bucket_j.i_bucket+=1 # update how many bits we are using - bucket_z=Bucket(self.b,bucket_j.i_bucket) - list_prefixes=[] + elif self.i>bucket_j.i: # there is a pointer available for the new bucket + bucket_j.i+=1 # update how many bits we are using + bucket_z=Bucket(self.b,bucket_j.i) + self.buckets.append(bucket_z) + + #find the lower half of hash prefixes/pointers that point to bucket_j and change them to point to bucket_z for prefix, ptr in self.hash_prefix.items(): - if ptr == bucket_j: - list_prefixes.append(prefix) - print(ptr) - for i in range(len(list_prefixes)//2,len(list_prefixes)): # half the pointers point to the new bucket + if prefix.startswith(bits[0:bucket_j.i]) and ptr==bucket_j: + self.hash_prefix[prefix]=bucket_z - self.hash_prefix[list_prefixes[i]]=len(self.buckets)-1 - # apply the hash function to each record of bucket j and according to the first i bits,the record stays in bucket_j or moves to bucket_z j_new_data={} - print(type(bucket_j)) - print(type(bucket_j.data)) for r_key,r_value in bucket_j.data.items(): @@ -96,12 +108,23 @@ def split(self, bucket_j,bits): def show(self): - # optionally, implement hash printing - pass + ''' + Prints the hash index. + + Args: + None + ''' + print("i="+str(self.i)) + for prefix, bucket in self.hash_prefix.items(): + print("Prefix="+prefix+", i_bucket="+str(bucket.i)+" => "+str(bucket.data)) + def calc_hash(self,value): ''' - + The hash function used to calculate the hash value of a given value. + + Args: + value: float or string. The value to be hashed. ''' if isinstance(value, str): value=int(binascii.hexlify(bytes(value,'utf-8')), 16) @@ -110,13 +133,34 @@ def calc_hash(self,value): def i_MSB(self,bin_value): ''' returns the i MSB of a binary value + + Args: + bin_value: string. The binary value to be truncated. ''' return bin_value[0:self.i] -class Bucket: # kouvadaki - def __init__(self, b,i_bucket): +class Bucket: + ''' + Bucket abstraction. + + Explanation of the attribute i: + Although i bits are required to find the correct bucket using the hash prefix dictionary, + several contiguous dictionary keys can point to the same bucket. + All these keys will have a common prefix, but the length of this prefix can be less than i. + So,we associate an integer with each bucket that gives the length of that common hash prefix. + ''' + def __init__(self, b,i): self.b = b # number of records held in block (blocking factor) - self.i_bucket=i_bucket - self.data = {} \ No newline at end of file + self.i=i # number of bits used for hash prefix + self.data = {} # dictionary of records and their pointers + + def find(self, value): + ''' + Returns the pointer of the record with the given value. + + Args: + value: float or string. The value of the record to be found. + ''' + return self.data[value] \ No newline at end of file From dcfead6432c4dc71ec3d808d5d24d646e0fa0bf6 Mon Sep 17 00:00:00 2001 From: Vassiliki Karagianniki <78561945+vassilikikrg@users.noreply.github.com> Date: Sun, 19 Feb 2023 20:46:07 +0200 Subject: [PATCH 17/25] Implementation of select where with hash index -database.py: ---modifications in select function in order to support the select using either btree or hash index based on the condition ---returned has_index function to its original state -hash.py: added find function that returns the pointer of the given value -table.py: implemented select_where_with_hash function that can only be used in case of equality condition --- miniDB/database.py | 76 ++++++++++++++++++++++++++-------------------- miniDB/hash.py | 9 ++++++ miniDB/table.py | 53 ++++++++++++++++++++++++++------ 3 files changed, 95 insertions(+), 43 deletions(-) diff --git a/miniDB/database.py b/miniDB/database.py index 4c455131..fb3990f3 100644 --- a/miniDB/database.py +++ b/miniDB/database.py @@ -367,27 +367,50 @@ def select(self, columns, table_name, condition, distinct=None, order_by=None, \ condition_column=condition.split(" ")[0] elif "and" in condition: condition_AND=True + condition_column = '' elif "or" in condition: condition_OR=True + condition_column = '' + elif "not " in condition: + cond=condition[4:] + condition_column = split_condition(cond)[0] else: condition_column = split_condition(condition)[0] else: condition_column = '' - # self.lock_table(table_name, mode='x') if self.is_locked(table_name): return + + # contains all indexes related to the condition column + if condition_column != '': + meta_data = self.select('*', 'meta_indexes', f'table_name={table_name} and index_column={condition_column}', return_object=True).data + else: + meta_data = [[]] if condition_AND is True or condition_OR is True: - table = self.tables[table_name]._select_where(columns, condition, distinct, order_by, desc, limit) - #comment!!!!! - - #elif self._has_index(table_name) and (condition_column==self.tables[table_name].column_names[self.tables[table_name].pk_idx] or condition_column in self.tables[table_name].column_names[self.tables[table_name].unique_idx]): - elif self._has_index(table_name,condition_column): - index_name = self.select('*', 'meta_indexes', f'table_name={table_name}', return_object=True).column_by_name('index_name')[0] - bt = self._load_idx(index_name) - table = self.tables[table_name]._select_where_with_btree(columns, bt, condition, distinct, order_by, desc, limit) + table = self.tables[table_name]._select_where(columns, condition, distinct, order_by, desc, limit) + elif meta_data!=[[]]: # if there is an index for the condition column + index_name=None + # search for a hash index since we prefer it from a btree in identity queries + if "not " not in condition and ">" not in condition and "<" not in condition: + for row in meta_data: + if row[3]=="hash": + index_name=row[1] + # if no hash was found or condition is range query , select the most recently created btree + if index_name is None: + for row in meta_data: + if row[3]=="btree": + index_name=row[1] + if index_name is None: # if no index was found + table = self.tables[table_name]._select_where(columns, condition, distinct, order_by, desc, limit) + else: + bt = self._load_idx(index_name) + table = self.tables[table_name]._select_where_with_btree(columns, bt, condition, distinct, order_by, desc, limit) + else: + hsh = self._load_idx(index_name) + table = self.tables[table_name]._select_where_with_hash(columns, hsh, condition) else: table = self.tables[table_name]._select_where(columns, condition, distinct, order_by, desc, limit) # self.unlock_table(table_name) @@ -686,11 +709,13 @@ def create_index(self, index_name, table_name, index_type='btree',index_column=N else: raise Exception('Cannot create index. Table has no primary key.')# if no primary key, no index else: - if index_column not in self.tables[table_name].column_names: - raise ValueError('The requested column to index does not exist in the table.') - elif (index_column!=self.tables[table_name].pk) or (self.tables[table_name].unique is not None and index_column not in self.tables[table_name].unique): - raise Exception('Cannot create index on a column that is not unique.') - + try: + col_names=self.tables[table_name].column_names #we find the column names of the table + index_column_idx=col_names.index(index_column) #the index of the given column(on which we create the index) in the list of column names + if index_column_idx not in self.tables[table_name].unique_idx: + raise Exception('Cannot create index on a column that is not unique.') + except: + raise ValueError('The requested column to index does not exist in the table.') if index_name not in self.tables['meta_indexes'].column_by_name('index_name'):#if there isn't already an index with the same name if index_type=='btree': @@ -698,7 +723,7 @@ def create_index(self, index_name, table_name, index_type='btree',index_column=N # insert a record with the name of the index,the table and the column on which it's created to the meta_indexes table self.tables['meta_indexes']._insert([table_name, index_name,index_column,index_type]) # create the actual index - self._construct_index(table_name, index_name,index_column) + self._construct_index(table_name, index_name,index_column,index_type) self.save_database() if index_type=='hash': logging.info('Creating Hash index.') @@ -725,7 +750,6 @@ def _construct_index(self, table_name, index_name,index_column): if key is None: continue bt.insert(key, idx) - bt.show() # save the btree self._save_index(index_name, bt) @@ -744,31 +768,17 @@ def _construct_hash_index(self, table_name, index_name,index_column): if key is None: continue hash_table.insert(key, idx) - hash_table.show() + # save the hashtable self._save_index(index_name, hash_table) def _has_index(self, table_name, index_column, index_type): ''' - Check whether the specified table's column is indexed. - + Check whether the specified table's primary key column is indexed. Args: table_name: string. Table name (must be part of database). - index_column: string. Column name to be checked for index. - - Returns: - True if the index column if it is indexed, false otherwise. ''' - # Get all the data from meta_indexes - this is most likely a small table so this is not a problem - data=self.tables['meta_indexes'].data - for row in data: - # Check rows that have a corresponding index to the selected table and column combination - if row[0]==table_name and row[2]==index_column: - index_type=row[3] - print(table_name,"has a",index_type,"index on",index_column) - return row[1],row[3] # return index name and type combo - - return False + return table_name in self.tables['meta_indexes'].column_by_name('table_name') def _save_index(self, index_name, index): ''' diff --git a/miniDB/hash.py b/miniDB/hash.py index 32801a0c..a4e6ed92 100644 --- a/miniDB/hash.py +++ b/miniDB/hash.py @@ -50,7 +50,16 @@ def _search(self, value): bits=self.i_MSB(bin_val) return self.hash_prefix[bits] # selected bucket (pointer) + def find(self, value): + ''' + Returns the pointer of the given value. + Args: + value: float or string. The value being searched for. + ''' + bucket=self._search(value)#the bucket that the given value exists or should exist in + return bucket.find(value) + def split(self, bucket_j,bits): ''' Splits the bucket_j into two buckets and updates the hash_prefix dictionary diff --git a/miniDB/table.py b/miniDB/table.py index 1dcb2c85..7b3ffe98 100644 --- a/miniDB/table.py +++ b/miniDB/table.py @@ -7,6 +7,7 @@ sys.path.append(f'{os.path.dirname(os.path.dirname(os.path.abspath(__file__)))}/miniDB') from misc import get_op, reverse_op_not, split_condition +from hash import Hash class Table: @@ -264,7 +265,6 @@ def _select_where(self, return_columns, condition=None, distinct=False, order_by rows = [ind for ind, x in enumerate(column) if value1 <= x <= value2]#append index of row if x between value1 and value2 except: raise TypeError('You need to provide numeric values inside a between statement') - #print("You need to provide only arithmetical values in order to evaluate between clause") # AND case elif "and" in condition: all_rows=[] @@ -284,13 +284,6 @@ def _select_where(self, return_columns, condition=None, distinct=False, order_by column = self.column_by_name(column_name) all_rows+=[ind for ind, x in enumerate(column) if get_op(operator, x, value)]#add indexes to all_rows rows=[*set(all_rows)] #keep all rows that match either of the conditions but first remove the double values - elif "not" in condition: - # NOT case - condition=condition.replace('not ','') # remove not keyword - column_name, operator, value = self._parse_condition(condition) - operator=reverse_op_not(operator) # reverse operator - column = self.column_by_name(column_name) - rows = [ind for ind, x in enumerate(column) if get_op(operator, x, value)] else: column_name, operator, value = self._parse_condition(condition) column = self.column_by_name(column_name) @@ -356,7 +349,6 @@ def _select_where_with_btree(self, return_columns, bt, condition, distinct=False rows1.append(ind) # btree find - bt.show() rows = bt.find(operator, value) try: @@ -386,6 +378,41 @@ def _select_where_with_btree(self, return_columns, bt, condition, distinct=False print('Column is neither a PK nor unique. Aborting')# if the column in condition is not a primary key or a unique column, abort the select return + def _select_where_with_hash(self, return_columns, hsh, condition): + + # if * return all columns, else find the column indexes for the columns specified + if return_columns == '*': + return_cols = [i for i in range(len(self.column_names))] + else: + return_cols = [self.column_names.index(colname) for colname in return_columns] + + + column_name, operator, value = self._parse_condition(condition) + + if operator != '=': + print('Hash index only supports equality. Aborting') + return + + #if the column in condition is a primary key or a unique column,continue the select + if column_name in self.unique or (self.pk is not None and column_name==self.pk) : + column = self.column_by_name(column_name) + #hash index find + ptr = hsh.find(value) + dict = {(key):([[self.data[ptr][j] for j in return_cols]] if key=="data" else value) for key,value in self.__dict__.items()} + + dict['column_names'] = [self.column_names[i] for i in return_cols] + dict['column_types'] = [self.column_types[i] for i in return_cols] + + s_table = Table(load=dict) + return s_table + + else: + print('Column is neither a PK nor unique. Aborting')# if the column in condition is not a primary key or a unique column, abort the select + return + + + + def order_by(self, column_name, desc=True): ''' Order table based on column. @@ -618,6 +645,11 @@ def _parse_condition(self, condition, join=False): Operatores supported: (<,<=,==,>=,>) join: boolean. Whether to join or not (False by default). ''' + not_condition = False + if "not " in condition: #not condition + condition=condition[4:] # remove not keyword + not_condition = True + # if both_columns (used by the join function) return the names of the names of the columns (left first) if join: return split_condition(condition) @@ -627,7 +659,8 @@ def _parse_condition(self, condition, join=False): if left not in self.column_names: raise ValueError(f'Condition is not valid (cant find column name)') coltype = self.column_types[self.column_names.index(left)] - + if not_condition: + return left,reverse_op_not(op), coltype(right) return left, op, coltype(right) From ebf13bca58da7448a0cfadb61bce83d2d59157ae Mon Sep 17 00:00:00 2001 From: Antonyfrtz Date: Sun, 19 Feb 2023 22:34:56 +0200 Subject: [PATCH 18/25] Btree select can handle all cases -- Between case is built in to select_where_with_btree -- AND/OR operators are forced to use linear search as checking the column for each condition and doing selection would require a massive overhaul --- miniDB/table.py | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/miniDB/table.py b/miniDB/table.py index 7b3ffe98..769984d4 100644 --- a/miniDB/table.py +++ b/miniDB/table.py @@ -262,7 +262,7 @@ def _select_where(self, return_columns, condition=None, distinct=False, order_by column=self.column_by_name(condition_list[0]) value1=int(condition_list[2]) value2=int(condition_list[4]) - rows = [ind for ind, x in enumerate(column) if value1 <= x <= value2]#append index of row if x between value1 and value2 + rows = [ind for ind, x in enumerate(column) if value1 <= x <= value2] #append index of row if x between value1 and value2 except: raise TypeError('You need to provide numeric values inside a between statement') # AND case @@ -331,8 +331,23 @@ def _select_where_with_btree(self, return_columns, bt, condition, distinct=False else: return_cols = [self.column_names.index(colname) for colname in return_columns] - - column_name, operator, value = self._parse_condition(condition) + rows=[] + if "between" in condition: # Supporting between search on btree + condition_list=condition.split(" ") + if len(condition_list)!=5 or condition_list[1]!="between" or condition_list[3]!="and": + raise Exception("Condition containing between keyword must have the following format: column between value1 and value2") + else: + try: + column_name=self.column_by_name(condition_list[0]) + value1=int(condition_list[2]) + value2=int(condition_list[4]) + rows1 = bt.find(">=",value1) # lower bound rows + rows2 = bt.find("<=",value2) # upper bound rows + rows = [row for row in rows1 if row in rows2] # intersection + except: + raise TypeError('You need to provide numeric values inside a between statement') + else: # simple where statement + column_name, operator, value = self._parse_condition(condition) #if the column in condition is a primary key or a unique column,continue the select if column_name in self.unique or (self.pk is not None and column_name==self.pk) : @@ -349,7 +364,8 @@ def _select_where_with_btree(self, return_columns, bt, condition, distinct=False rows1.append(ind) # btree find - rows = bt.find(operator, value) + if rows==[]: + rows = bt.find(operator, value) try: k = int(limit) From b6bf303efed38e887c0a8cc4c2d5db74b263be67 Mon Sep 17 00:00:00 2001 From: Antonyfrtz Date: Sun, 19 Feb 2023 23:57:30 +0200 Subject: [PATCH 19/25] fixed small bug on btree index creation --- miniDB/database.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/miniDB/database.py b/miniDB/database.py index fb3990f3..e3453c6a 100644 --- a/miniDB/database.py +++ b/miniDB/database.py @@ -723,7 +723,7 @@ def create_index(self, index_name, table_name, index_type='btree',index_column=N # insert a record with the name of the index,the table and the column on which it's created to the meta_indexes table self.tables['meta_indexes']._insert([table_name, index_name,index_column,index_type]) # create the actual index - self._construct_index(table_name, index_name,index_column,index_type) + self._construct_index(table_name, index_name,index_column) self.save_database() if index_type=='hash': logging.info('Creating Hash index.') @@ -763,7 +763,7 @@ def _construct_hash_index(self, table_name, index_name,index_column): index_column: string. Name of the column on which we create the index ''' hash_table=Hash(3) # argument is blocking factor - # for each record of the table, insert its column value and index to the btree + # for each record of the table, insert its column value and index to the hash index for idx, key in enumerate(self.tables[table_name].column_by_name(index_column)): if key is None: continue From 05a3eb44c94d2615146b939dcdc877c931316961 Mon Sep 17 00:00:00 2001 From: Vassiliki Karagianniki <78561945+vassilikikrg@users.noreply.github.com> Date: Mon, 20 Feb 2023 00:49:39 +0200 Subject: [PATCH 20/25] fixed bug in _select_where_with_btree function when between keyword is in condition --- miniDB/database.py | 4 ++-- miniDB/table.py | 39 +++++++++++++++++++++------------------ 2 files changed, 23 insertions(+), 20 deletions(-) diff --git a/miniDB/database.py b/miniDB/database.py index e3453c6a..d55abc47 100644 --- a/miniDB/database.py +++ b/miniDB/database.py @@ -394,7 +394,7 @@ def select(self, columns, table_name, condition, distinct=None, order_by=None, \ elif meta_data!=[[]]: # if there is an index for the condition column index_name=None # search for a hash index since we prefer it from a btree in identity queries - if "not " not in condition and ">" not in condition and "<" not in condition: + if "not " not in condition and ">" not in condition and "<" not in condition and "between" not in condition and "like" not in condition and "in" not in condition and "is" not in condition: for row in meta_data: if row[3]=="hash": index_name=row[1] @@ -772,7 +772,7 @@ def _construct_hash_index(self, table_name, index_name,index_column): # save the hashtable self._save_index(index_name, hash_table) - def _has_index(self, table_name, index_column, index_type): + def _has_index(self, table_name): ''' Check whether the specified table's primary key column is indexed. Args: diff --git a/miniDB/table.py b/miniDB/table.py index 769984d4..926aa6a7 100644 --- a/miniDB/table.py +++ b/miniDB/table.py @@ -331,19 +331,19 @@ def _select_where_with_btree(self, return_columns, bt, condition, distinct=False else: return_cols = [self.column_names.index(colname) for colname in return_columns] - rows=[] + value1=None + value2=None + conditionBetween=False if "between" in condition: # Supporting between search on btree condition_list=condition.split(" ") if len(condition_list)!=5 or condition_list[1]!="between" or condition_list[3]!="and": raise Exception("Condition containing between keyword must have the following format: column between value1 and value2") else: try: - column_name=self.column_by_name(condition_list[0]) + column_name=condition_list[0] value1=int(condition_list[2]) value2=int(condition_list[4]) - rows1 = bt.find(">=",value1) # lower bound rows - rows2 = bt.find("<=",value2) # upper bound rows - rows = [row for row in rows1 if row in rows2] # intersection + conditionBetween=True except: raise TypeError('You need to provide numeric values inside a between statement') else: # simple where statement @@ -352,19 +352,22 @@ def _select_where_with_btree(self, return_columns, bt, condition, distinct=False #if the column in condition is a primary key or a unique column,continue the select if column_name in self.unique or (self.pk is not None and column_name==self.pk) : # here we run the same select twice, sequentially and using the btree. - # we then check the results match and compare performance (number of operation) - column = self.column_by_name(column_name) - - # sequential - rows1 = [] - opsseq = 0 - for ind, x in enumerate(column): - opsseq+=1 - if get_op(operator, x, value): - rows1.append(ind) - - # btree find - if rows==[]: + # we then check the results match and compare performance (number of operations) + if conditionBetween: + rows1 = bt.find(">=",value1) # lower bound rows + rows2 = bt.find("<=",value2) # upper bound rows + rows = [row for row in rows1 if row in rows2] # intersection + else: + column = self.column_by_name(column_name) + # sequential + rows1 = [] + opsseq = 0 + for ind, x in enumerate(column): + opsseq+=1 + if get_op(operator, x, value): + rows1.append(ind) + + # btree find rows = bt.find(operator, value) try: From 1485de2970f657bed9fe97036e5447231a00a9d0 Mon Sep 17 00:00:00 2001 From: Vassiliki Karagianniki <78561945+vassilikikrg@users.noreply.github.com> Date: Mon, 20 Feb 2023 16:28:36 +0200 Subject: [PATCH 21/25] Created basic rules for building equivalent query plans --- miniDB/equivalentQueries.py | 87 +++++++++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 miniDB/equivalentQueries.py diff --git a/miniDB/equivalentQueries.py b/miniDB/equivalentQueries.py new file mode 100644 index 00000000..2b302ea7 --- /dev/null +++ b/miniDB/equivalentQueries.py @@ -0,0 +1,87 @@ +def rule1(query): + ''' + Given a query, apply the transformation rule + σ q1∧q2 (R) = σ q1(σ q2 (R)) + query: the query to transform + ''' + + if query['where'] is not None and query['where'].find('and') != -1: + # Split the query into two parts + condition1, condition2 = query['where'].split('and') + # Apply the rule + equiv_query = query.copy() + equiv_query['where'] = condition1 + equiv_query['from'] = {'select': '*', 'from': equiv_query['from'], 'where': condition2,'distinct': None, 'orderby': None,'limit': None,'desc': None} + return equiv_query + return None +def rule2(query): + +def rule4(query): + ''' + Given a query, apply the transformation rule + R ⋈q S = S ⋈q R + ''' + + if 'select' in query.keys() and query['from'].find('join') != -1: + left=query['from']['left'] + right=query['from']['right'] + equiv_query = query.copy() + equiv_query['from']['left'] = right + equiv_query['from']['right'] = left + return equiv_query + +def rule5(query): + ''' + Given a query, apply the transformation rules: + -(only for natural join) + (R ⋈ S) ⋈ T = R ⋈ (S ⋈ T) + + -( θ-join ) + (R ⋈q1 S) ⋈ q2^q3 T = R ⋈ q1^q3 (S ⋈ q2 T) + ''' + # multiple joins are not supported by minidb for now + if 'select' in query.keys() and query['from'].find('join') != -1 and query['where'] is not None: + condition=query['where'] + if condition.find('and') != -1: + condition1=query['join']['on'] + condition2, condition3 = condition.split('and') + # Apply the rule + equiv_query = query.copy() + return query + + + +def rule6(query): + ''' + Given a query, apply the transformation rules + case 1: σ q1(R ⋈q S) = σ q1(R) ⋈q S + case 2: σ q1(R ⋈q S) = R ⋈q σ q1(S) + case 3: σ q1^q2 (R ⋈q S) = σ q1(R) ⋈q σ q2 (S) + ''' + equivalent_queries = [] + if 'select' in query.keys() and query['from'].find('join') != -1: + if query['where'].find('and') != -1: + # Split the condition into two parts + condition1, condition2 = query['where'].split('and') + # Apply the rule(case of double select condition: condition1 corresponds to left table and condition2 corresponds to right table) + subquery1 = {'select': '*', 'from': query['from']['left'], 'where': condition1, 'distinct': None, 'orderby': None,'limit': None,'desc': None} + subquery2 = {'select': '*', 'from': query['from']['right'], 'where': condition2, 'distinct': None, 'orderby': None,'limit': None,'desc': None} + equiv_query= query.copy() + equiv_query['from']['left']=subquery1 + equivalent_queries['from']['right']=subquery2 + equivalent_queries.append(equiv_query) + + # Apply the rule(case: condition corresponds only to left table) + subquery1 = {'select': '*', 'from': query['from']['left'], 'where': query['where'], 'distinct': None, 'orderby': None,'limit': None,'desc': None} + equiv_query= query.copy() + equiv_query['from']['left']=subquery1 + equivalent_queries.append(equiv_query) + # Apply the rule(case: condition corresponds only to right table) + subquery2 = {'select': '*', 'from': query['from']['right'], 'where': query['where'], 'distinct': None, 'orderby': None,'limit': None,'desc': None} + equiv_query= query.copy() + equiv_query['from']['right']=subquery2 + equivalent_queries.append(equiv_query) + return equivalent_queries + + + \ No newline at end of file From cffa6ac2c7d88d769a8a66e96be9ce2890d40e1a Mon Sep 17 00:00:00 2001 From: Antonyfrtz Date: Mon, 20 Feb 2023 19:21:15 +0200 Subject: [PATCH 22/25] Finished RA equivalence transformation rules - Rules tested and operational - Started work on recursive creation of equivalentQueries list --- bash.exe.stackdump | 16 +++++++++ miniDB/equivalentQueries.py | 70 ++++++++++++++++++++++++++++++------- 2 files changed, 73 insertions(+), 13 deletions(-) create mode 100644 bash.exe.stackdump diff --git a/bash.exe.stackdump b/bash.exe.stackdump new file mode 100644 index 00000000..ffc5b866 --- /dev/null +++ b/bash.exe.stackdump @@ -0,0 +1,16 @@ +Stack trace: +Frame Function Args +000FFFFCD30 0018006286E (001802825B0, 0018026DE51, 00000000059, 000FFFFB710) +000FFFFCD30 0018004846A (000FFFFCD30, 00100000000, 00000000000, 00000000001) +000FFFFCD30 001800484A2 (00000000000, 00000000000, 00000000059, 16B0ABC5BB60) +000FFFFCD30 0018006E166 (00180045323, 00180350B78, 00000000000, 0000000000D) +000FFFFCD30 0018006E179 (00180045170, 001802357E0, 001800448F2, 000FFFFC910) +000FFFFCD30 001800707F4 (00000000013, 00000000001, 000FFFFC910, 00180270615) +000FFFFCD30 0018005AAFF (000FFFFCA60, 00000000000, 00000000000, 008FFFFFFFF) +000FFFFCD30 0018005B245 (00800000010, 00000000000, 000FFFFCD30, 000000303E9) +000FFFFCD30 0018005B757 (001800D8DFE, 00000000000, 00000000000, 00000000000) +000FFFFCD30 0018005BA66 (00000000000, 000FFFFCD30, FFFFFFFFFFFFFFC6, 00000000000) +000FFFFCD30 00180048C0C (00000000000, 00000000000, 00000000000, 00000000000) +000FFFFFFF0 00180047716 (00000000000, 00000000000, 00000000000, 00000000000) +000FFFFFFF0 001800477C4 (00000000000, 00000000000, 00000000000, 00000000000) +End of stack trace diff --git a/miniDB/equivalentQueries.py b/miniDB/equivalentQueries.py index 2b302ea7..b00e7e1f 100644 --- a/miniDB/equivalentQueries.py +++ b/miniDB/equivalentQueries.py @@ -1,3 +1,7 @@ + +import copy + + def rule1(query): ''' Given a query, apply the transformation rule @@ -9,12 +13,37 @@ def rule1(query): # Split the query into two parts condition1, condition2 = query['where'].split('and') # Apply the rule - equiv_query = query.copy() + equiv_query = copy.deepcopy(query) equiv_query['where'] = condition1 equiv_query['from'] = {'select': '*', 'from': equiv_query['from'], 'where': condition2,'distinct': None, 'orderby': None,'limit': None,'desc': None} return equiv_query return None -def rule2(query): + +def rule2(query): + ''' + Rule 2: σθ1(σθ2(R)) = σθ2(σθ1(R)) + ''' + if 'select' in query.keys() and 'select' in query['from'].keys(): #and isinstance(query['from'],dict) + condition1=query['where'] # θ1 + condition2=query['from']['where'] # θ2 + equiv_query = copy.deepcopy(query) + equiv_query['where'] = condition2 + equiv_query['from']['where'] = condition1 + return equiv_query + return None + +def rule3(query): + ''' + Rule 3b: σθ1(R ⋈θ2 S) = R ⋈(θ1^θ2) S + ''' + if 'select' in query.keys() and isinstance(query['from'], dict) and query['from'].get('join') is not None: + condition1=query['where'] # θ1 + condition2=query['from']['on'] # θ2 + equiv_query = copy.deepcopy(query) + equiv_query['where'] = None + equiv_query['from']['on'] = condition1 + ' and ' + condition2 # θ1^θ2 as the join condition + return equiv_query + return None def rule4(query): ''' @@ -22,32 +51,34 @@ def rule4(query): R ⋈q S = S ⋈q R ''' - if 'select' in query.keys() and query['from'].find('join') != -1: + if 'select' in query.keys() and isinstance(query['from'], dict) and query['from'].get('join') is not None: left=query['from']['left'] right=query['from']['right'] - equiv_query = query.copy() + equiv_query = copy.deepcopy(query) equiv_query['from']['left'] = right equiv_query['from']['right'] = left return equiv_query + return None def rule5(query): ''' Given a query, apply the transformation rules: - -(only for natural join) - (R ⋈ S) ⋈ T = R ⋈ (S ⋈ T) - -( θ-join ) (R ⋈q1 S) ⋈ q2^q3 T = R ⋈ q1^q3 (S ⋈ q2 T) ''' # multiple joins are not supported by minidb for now + ''' if 'select' in query.keys() and query['from'].find('join') != -1 and query['where'] is not None: condition=query['where'] if condition.find('and') != -1: condition1=query['join']['on'] condition2, condition3 = condition.split('and') # Apply the rule - equiv_query = query.copy() + equiv_query = copy.deepcopy(query) return query + return None + ''' + pass @@ -59,29 +90,42 @@ def rule6(query): case 3: σ q1^q2 (R ⋈q S) = σ q1(R) ⋈q σ q2 (S) ''' equivalent_queries = [] - if 'select' in query.keys() and query['from'].find('join') != -1: + if 'select' in query.keys() and isinstance(query['from'], dict) and query['from'].get('join') is not None: if query['where'].find('and') != -1: # Split the condition into two parts condition1, condition2 = query['where'].split('and') # Apply the rule(case of double select condition: condition1 corresponds to left table and condition2 corresponds to right table) subquery1 = {'select': '*', 'from': query['from']['left'], 'where': condition1, 'distinct': None, 'orderby': None,'limit': None,'desc': None} subquery2 = {'select': '*', 'from': query['from']['right'], 'where': condition2, 'distinct': None, 'orderby': None,'limit': None,'desc': None} - equiv_query= query.copy() + equiv_query= copy.deepcopy(query) equiv_query['from']['left']=subquery1 equivalent_queries['from']['right']=subquery2 equivalent_queries.append(equiv_query) # Apply the rule(case: condition corresponds only to left table) subquery1 = {'select': '*', 'from': query['from']['left'], 'where': query['where'], 'distinct': None, 'orderby': None,'limit': None,'desc': None} - equiv_query= query.copy() + equiv_query= copy.deepcopy(query) equiv_query['from']['left']=subquery1 equivalent_queries.append(equiv_query) # Apply the rule(case: condition corresponds only to right table) subquery2 = {'select': '*', 'from': query['from']['right'], 'where': query['where'], 'distinct': None, 'orderby': None,'limit': None,'desc': None} - equiv_query= query.copy() + equiv_query= copy.deepcopy(query) equiv_query['from']['right']=subquery2 equivalent_queries.append(equiv_query) return equivalent_queries + +equiv_queries=[] +lastrule = None +def equiv_recursive(query): + global lastrule # the last rule applied + for rule in [rule1, rule2, rule3, rule4, rule5, rule6]: + equiv_query = rule(query) # apply the rule + if equiv_query is not None: + lastrule = rule + equiv_queries.append(equiv_query) # add the equivalent query to the list + - \ No newline at end of file +# Example dictionary +exampleQuery = {'select': '*', 'from': {'select': '*', 'from': 'R', 'where': 'a > 5', 'distinct': None, 'orderby': None,'limit': None,'desc': None}, 'where': 'b < 10', 'distinct': None, 'orderby': None,'limit': None,'desc': None} +equiv_recursive(exampleQuery) \ No newline at end of file From 043e7003ff96543b1f24057a99ca3ebafb60275c Mon Sep 17 00:00:00 2001 From: Antonyfrtz Date: Mon, 20 Feb 2023 22:30:12 +0200 Subject: [PATCH 23/25] added equivalentQueries to mdb.py and fixed bugs - Equivalent queries can now be found using the equivalentQueries function in mdb.py - Call this by typing "equivalent of " in the miniDB mdb.py shell - Fixed bugs in the equivalentQueries function, avoiding cycles in the recursive calls - Fixed bugs in the equivalentQueries function, avoiding infinite loops in the recursive calls - Fixed bugs in the equivalentQueries function, relating to the rules for equivalent queries --- bash.exe.stackdump | 2 +- mdb.py | 4 ++ miniDB/equivalentQueries.py | 96 ++++++++++++++++++++----------------- 3 files changed, 57 insertions(+), 45 deletions(-) diff --git a/bash.exe.stackdump b/bash.exe.stackdump index ffc5b866..9bc552b0 100644 --- a/bash.exe.stackdump +++ b/bash.exe.stackdump @@ -2,7 +2,7 @@ Stack trace: Frame Function Args 000FFFFCD30 0018006286E (001802825B0, 0018026DE51, 00000000059, 000FFFFB710) 000FFFFCD30 0018004846A (000FFFFCD30, 00100000000, 00000000000, 00000000001) -000FFFFCD30 001800484A2 (00000000000, 00000000000, 00000000059, 16B0ABC5BB60) +000FFFFCD30 001800484A2 (00000000000, 00000000000, 00000000059, 432508F9B254) 000FFFFCD30 0018006E166 (00180045323, 00180350B78, 00000000000, 0000000000D) 000FFFFCD30 0018006E179 (00180045170, 001802357E0, 001800448F2, 000FFFFC910) 000FFFFCD30 001800707F4 (00000000013, 00000000001, 000FFFFC910, 00180270615) diff --git a/mdb.py b/mdb.py index 755a5fb2..34ec7d3e 100644 --- a/mdb.py +++ b/mdb.py @@ -5,6 +5,7 @@ import readline import traceback import shutil +from miniDB.equivalentQueries import equiv_print sys.path.append('miniDB') from miniDB.database import Database @@ -303,6 +304,9 @@ def remove_db(db_name): elif line.startswith('explain'): dic = interpret(line.replace('explain ','')) pprint(dic, sort_dicts=False) + elif line.startswith('equivalent of'): + dic = interpret(line.replace('equivalent of ','')) + equiv_print(dic) else: dic = interpret(line) result = execute_dic(dic) diff --git a/miniDB/equivalentQueries.py b/miniDB/equivalentQueries.py index b00e7e1f..9b03b281 100644 --- a/miniDB/equivalentQueries.py +++ b/miniDB/equivalentQueries.py @@ -8,12 +8,10 @@ def rule1(query): σ q1∧q2 (R) = σ q1(σ q2 (R)) query: the query to transform ''' - - if query['where'] is not None and query['where'].find('and') != -1: - # Split the query into two parts - condition1, condition2 = query['where'].split('and') - # Apply the rule - equiv_query = copy.deepcopy(query) + if 'select' in query.keys() and isinstance(query['from'],dict) and 'select ' in query['from'].keys(): + equiv_query = copy.deepcopy(query) + # Split the condition into two parts + condition1, condition2 = equiv_query['where'].split('and') equiv_query['where'] = condition1 equiv_query['from'] = {'select': '*', 'from': equiv_query['from'], 'where': condition2,'distinct': None, 'orderby': None,'limit': None,'desc': None} return equiv_query @@ -23,10 +21,10 @@ def rule2(query): ''' Rule 2: σθ1(σθ2(R)) = σθ2(σθ1(R)) ''' - if 'select' in query.keys() and 'select' in query['from'].keys(): #and isinstance(query['from'],dict) - condition1=query['where'] # θ1 - condition2=query['from']['where'] # θ2 + if 'select' in query.keys() and 'select' in query['from'].keys(): equiv_query = copy.deepcopy(query) + condition1=equiv_query['where'] # θ1 + condition2=equiv_query['from']['where'] # θ2 equiv_query['where'] = condition2 equiv_query['from']['where'] = condition1 return equiv_query @@ -36,10 +34,10 @@ def rule3(query): ''' Rule 3b: σθ1(R ⋈θ2 S) = R ⋈(θ1^θ2) S ''' - if 'select' in query.keys() and isinstance(query['from'], dict) and query['from'].get('join') is not None: - condition1=query['where'] # θ1 - condition2=query['from']['on'] # θ2 + if 'select' in query.keys() and (isinstance(query['from'], dict) and 'join' in query['from'].keys()) and query['where'] is not None and query['from']['on'] is not None: equiv_query = copy.deepcopy(query) + condition1=equiv_query['where'] # θ1 + condition2=equiv_query['from']['on'] # θ2 equiv_query['where'] = None equiv_query['from']['on'] = condition1 + ' and ' + condition2 # θ1^θ2 as the join condition return equiv_query @@ -51,10 +49,10 @@ def rule4(query): R ⋈q S = S ⋈q R ''' - if 'select' in query.keys() and isinstance(query['from'], dict) and query['from'].get('join') is not None: - left=query['from']['left'] - right=query['from']['right'] + if 'join' in query.keys(): equiv_query = copy.deepcopy(query) + left=equiv_query['from']['left'] + right=equiv_query['from']['right'] equiv_query['from']['left'] = right equiv_query['from']['right'] = left return equiv_query @@ -67,19 +65,7 @@ def rule5(query): (R ⋈q1 S) ⋈ q2^q3 T = R ⋈ q1^q3 (S ⋈ q2 T) ''' # multiple joins are not supported by minidb for now - ''' - if 'select' in query.keys() and query['from'].find('join') != -1 and query['where'] is not None: - condition=query['where'] - if condition.find('and') != -1: - condition1=query['join']['on'] - condition2, condition3 = condition.split('and') - # Apply the rule - equiv_query = copy.deepcopy(query) - return query - return None - ''' pass - def rule6(query): @@ -90,42 +76,64 @@ def rule6(query): case 3: σ q1^q2 (R ⋈q S) = σ q1(R) ⋈q σ q2 (S) ''' equivalent_queries = [] - if 'select' in query.keys() and isinstance(query['from'], dict) and query['from'].get('join') is not None: - if query['where'].find('and') != -1: + if 'select' in query.keys() and ((isinstance(query['from'], dict) and 'join' in query['from'].keys())): + if query['where'] is not None and 'and' in query['where']: + equiv_query= copy.deepcopy(query) # Split the condition into two parts - condition1, condition2 = query['where'].split('and') + condition1, condition2 = equiv_query['where'].split('and') # Apply the rule(case of double select condition: condition1 corresponds to left table and condition2 corresponds to right table) subquery1 = {'select': '*', 'from': query['from']['left'], 'where': condition1, 'distinct': None, 'orderby': None,'limit': None,'desc': None} subquery2 = {'select': '*', 'from': query['from']['right'], 'where': condition2, 'distinct': None, 'orderby': None,'limit': None,'desc': None} - equiv_query= copy.deepcopy(query) equiv_query['from']['left']=subquery1 - equivalent_queries['from']['right']=subquery2 + equiv_query['from']['right']=subquery2 + equiv_query['where']=None equivalent_queries.append(equiv_query) # Apply the rule(case: condition corresponds only to left table) - subquery1 = {'select': '*', 'from': query['from']['left'], 'where': query['where'], 'distinct': None, 'orderby': None,'limit': None,'desc': None} equiv_query= copy.deepcopy(query) + subquery1 = {'select': '*', 'from': query['from']['left'], 'where': query['where'], 'distinct': None, 'orderby': None,'limit': None,'desc': None} equiv_query['from']['left']=subquery1 + equiv_query['where']=None equivalent_queries.append(equiv_query) # Apply the rule(case: condition corresponds only to right table) - subquery2 = {'select': '*', 'from': query['from']['right'], 'where': query['where'], 'distinct': None, 'orderby': None,'limit': None,'desc': None} equiv_query= copy.deepcopy(query) + subquery2 = {'select': '*', 'from': query['from']['right'], 'where': query['where'], 'distinct': None, 'orderby': None,'limit': None,'desc': None} equiv_query['from']['right']=subquery2 + equiv_query['where']=None equivalent_queries.append(equiv_query) - return equivalent_queries - + return equivalent_queries + return None -equiv_queries=[] -lastrule = None -def equiv_recursive(query): - global lastrule # the last rule applied +def equiv_recursive(query, equiv_queries=None, lastrule=None, visited=None): + # keep track of the visited queries as a frozen set + if equiv_queries is None: + equiv_queries = [] + if query in equiv_queries: + return equiv_queries + equiv_queries.append(query) for rule in [rule1, rule2, rule3, rule4, rule5, rule6]: - equiv_query = rule(query) # apply the rule - if equiv_query is not None: + equiv_query = rule(query) + if equiv_query is not None and lastrule != rule: lastrule = rule - equiv_queries.append(equiv_query) # add the equivalent query to the list + if rule == rule6: + for q in equiv_query: + equiv_queries.append(q) + equiv_recursive(q, equiv_queries, lastrule, visited) + else: + equiv_queries.append(equiv_query) + equiv_recursive(equiv_query, equiv_queries, lastrule, visited) + return equiv_queries +def equiv_print(query): + equiv_queries=equiv_recursive(query) + for count,q in enumerate(equiv_queries): + print ("\n"+"Query "+str(count+1)+": \n"+str(q)+"\n") # Example dictionary exampleQuery = {'select': '*', 'from': {'select': '*', 'from': 'R', 'where': 'a > 5', 'distinct': None, 'orderby': None,'limit': None,'desc': None}, 'where': 'b < 10', 'distinct': None, 'orderby': None,'limit': None,'desc': None} -equiv_recursive(exampleQuery) \ No newline at end of file +# Second example dictionary with join +exampleQuery2 = {'select': '*', 'from': {'join': 'inner', 'left': 'instructor', 'right': 'department', 'on': 'instructor.dept_name=department.dept_name'}, 'where': 'not instructor.dept_name=biology ', 'distinct': None, 'order by': None, 'limit': None, 'desc': None} +# Third example dictionary with join and 'and' condition +exampleQuery3 = {'select': '*', 'from': {'join': 'inner', 'left': 'instructor', 'right': 'department', 'on': 'instructor.dept_name=department.dept_name'}, 'where': 'not instructor.dept_name=biology and instructor.salary>60000', 'distinct': None, 'order by': None, 'limit': None, 'desc': None} +# Print query +#equiv_print(exampleQuery3) \ No newline at end of file From 32cdb0cf40d2217573c760b5f5c4b8e7d75375a3 Mon Sep 17 00:00:00 2001 From: Antonyfrtz Date: Mon, 20 Feb 2023 23:02:26 +0200 Subject: [PATCH 24/25] AND, OR, and NOT operators supported in delete - delete_where() now supports AND, OR, and NOT operators Note: this commit has been tested but not thoroughly. Please report any bugs. In case this messes up other things, please revert to the previous commit. --- miniDB/database.py | 2 +- miniDB/table.py | 50 +++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 46 insertions(+), 6 deletions(-) diff --git a/miniDB/database.py b/miniDB/database.py index d55abc47..46851497 100644 --- a/miniDB/database.py +++ b/miniDB/database.py @@ -768,7 +768,7 @@ def _construct_hash_index(self, table_name, index_name,index_column): if key is None: continue hash_table.insert(key, idx) - + hash_table.show() # save the hashtable self._save_index(index_name, hash_table) diff --git a/miniDB/table.py b/miniDB/table.py index 926aa6a7..0f93d2da 100644 --- a/miniDB/table.py +++ b/miniDB/table.py @@ -200,20 +200,60 @@ def _delete_where(self, condition): Operatores supported: (<,<=,==,>=,>) ''' - column_name, operator, value = self._parse_condition(condition) - indexes_to_del = [] + rows = [] + if condition is not None: + + # BETWEEN case + if "between" in condition: + condition_list=condition.split(" ") + if len(condition_list)!=5 or condition_list[1]!="between" or condition_list[3]!="and": + raise Exception("Condition containing between keyword must have the following format: column between value1 and value2") + else: + try: + column=self.column_by_name(condition_list[0]) + value1=int(condition_list[2]) + value2=int(condition_list[4]) + rows = [ind for ind, x in enumerate(column) if value1 <= x <= value2] #append index of row if x between value1 and value2 + except: + raise TypeError('You need to provide numeric values inside a between statement') + # AND case + elif "and" in condition: + all_rows=[] + seperated_conditions=condition.split(" and ") + for cond in seperated_conditions: #select all the rows that match each condition + column_name, operator, value = self._parse_condition(cond) + column = self.column_by_name(column_name) + all_rows+=[ind for ind, x in enumerate(column) if get_op(operator, x, value)]#add indexes to all_rows + # Keep the rows from the all_rows list that appear as many times as the number of conditions in the AND clauses + rows=[*set(i for i in all_rows if all_rows.count(i) > len(seperated_conditions)-1)] + # OR case + elif "or" in condition: + all_rows=[] + seperated_conditions=condition.split(" or ") + for cond in seperated_conditions: #select all the rows that match each condition + column_name, operator, value = self._parse_condition(cond) + column = self.column_by_name(column_name) + all_rows+=[ind for ind, x in enumerate(column) if get_op(operator, x, value)]#add indexes to all_rows + rows=[*set(all_rows)] #keep all rows that match either of the conditions but first remove the double values + else: + column_name, operator, value = self._parse_condition(condition) + column = self.column_by_name(column_name) + rows = [ind for ind, x in enumerate(column) if get_op(operator, x, value)] + + else: + rows = [i for i in range(len(self.data))] column = self.column_by_name(column_name) for index, row_value in enumerate(column): if get_op(operator, row_value, value): - indexes_to_del.append(index) + rows.append(index) # we pop from highest to lowest index in order to avoid removing the wrong item # since we dont delete, we dont have to to pop in that order, but since delete is used # to delete from meta tables too, we still implement it. - for index in sorted(indexes_to_del, reverse=True): + for index in sorted(rows, reverse=True): if self._name[:4] != 'meta': # if the table is not a metatable, replace the row with a row of nones self.data[index] = [None for _ in range(len(self.column_names))] @@ -222,7 +262,7 @@ def _delete_where(self, condition): # self._update() # we have to return the deleted indexes, since they will be appended to the insert_stack - return indexes_to_del + return rows def _select_where(self, return_columns, condition=None, distinct=False, order_by=None, desc=True, limit=None): From 54f21bbc0d77b1aad7c7fae7965e8905b268f450 Mon Sep 17 00:00:00 2001 From: Antonis Fritzelas <77737062+Antonyfrtz@users.noreply.github.com> Date: Wed, 15 Mar 2023 02:08:08 +0200 Subject: [PATCH 25/25] Update README.md --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index 7e276754..5fe7d17e 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,13 @@ mdblogo

+# Fork by P20074,P20199,P20220 + +This fork provides implementations for key features of a modern RDBMS which include + - Enriching WHERE statement by supporting (a) NOT and BETWEEN operators and (b) AND and OR operators + - Enriching indexing functionality by supporting (a) BTree index over unique (non-PK) columns and (b) Hash index over PK or unique columns + - Implementing miniDB’s query optimiser by building equivalent query plans based on respective RA expressions + # miniDB The miniDB project is a minimal and easy to expand and develop for RMDBS tool, written exclusivelly in Python 3. MiniDB's main goal is to provide the user with as much functionality as possible while being easy to understand and even easier to expand. Thus, miniDB's primary market are students and researchers that want to work with a tool that they can understand through and through, while being able to implement additional features as quickly as possible.