diff --git a/.gitignore b/.gitignore
index f559dbea..55802537 100644
--- a/.gitignore
+++ b/.gitignore
@@ -4,3 +4,5 @@ dbdata/
__pycache__/
.idea/
.pytest_cache
+.vscode/launch.json
+.vscode/settings.json
diff --git a/README.md b/README.md
index 7e276754..5fe7d17e 100644
--- a/README.md
+++ b/README.md
@@ -2,6 +2,13 @@
+# Fork by P20074,P20199,P20220
+
+This fork provides implementations for key features of a modern RDBMS which include
+ - Enriching WHERE statement by supporting (a) NOT and BETWEEN operators and (b) AND and OR operators
+ - Enriching indexing functionality by supporting (a) BTree index over unique (non-PK) columns and (b) Hash index over PK or unique columns
+ - Implementing miniDB’s query optimiser by building equivalent query plans based on respective RA expressions
+
# miniDB
The miniDB project is a minimal and easy to expand and develop for RMDBS tool, written exclusivelly in Python 3. MiniDB's main goal is to provide the user with as much functionality as possible while being easy to understand and even easier to expand. Thus, miniDB's primary market are students and researchers that want to work with a tool that they can understand through and through, while being able to implement additional features as quickly as possible.
diff --git a/bash.exe.stackdump b/bash.exe.stackdump
new file mode 100644
index 00000000..9bc552b0
--- /dev/null
+++ b/bash.exe.stackdump
@@ -0,0 +1,16 @@
+Stack trace:
+Frame Function Args
+000FFFFCD30 0018006286E (001802825B0, 0018026DE51, 00000000059, 000FFFFB710)
+000FFFFCD30 0018004846A (000FFFFCD30, 00100000000, 00000000000, 00000000001)
+000FFFFCD30 001800484A2 (00000000000, 00000000000, 00000000059, 432508F9B254)
+000FFFFCD30 0018006E166 (00180045323, 00180350B78, 00000000000, 0000000000D)
+000FFFFCD30 0018006E179 (00180045170, 001802357E0, 001800448F2, 000FFFFC910)
+000FFFFCD30 001800707F4 (00000000013, 00000000001, 000FFFFC910, 00180270615)
+000FFFFCD30 0018005AAFF (000FFFFCA60, 00000000000, 00000000000, 008FFFFFFFF)
+000FFFFCD30 0018005B245 (00800000010, 00000000000, 000FFFFCD30, 000000303E9)
+000FFFFCD30 0018005B757 (001800D8DFE, 00000000000, 00000000000, 00000000000)
+000FFFFCD30 0018005BA66 (00000000000, 000FFFFCD30, FFFFFFFFFFFFFFC6, 00000000000)
+000FFFFCD30 00180048C0C (00000000000, 00000000000, 00000000000, 00000000000)
+000FFFFFFF0 00180047716 (00000000000, 00000000000, 00000000000, 00000000000)
+000FFFFFFF0 001800477C4 (00000000000, 00000000000, 00000000000, 00000000000)
+End of stack trace
diff --git a/mdb.py b/mdb.py
index a981e5be..34ec7d3e 100644
--- a/mdb.py
+++ b/mdb.py
@@ -5,10 +5,11 @@
import readline
import traceback
import shutil
+from miniDB.equivalentQueries import equiv_print
sys.path.append('miniDB')
-from database import Database
-from table import Table
+from miniDB.database import Database
+from miniDB.table import Table
# art font is "big"
art = '''
_ _ _____ ____
@@ -45,14 +46,14 @@ def create_query_plan(query, keywords, action):
This can and will be used recursively
'''
- dic = {val: None for val in keywords if val!=';'}
+ dic = {val: None for val in keywords if val!=';'} # dict of query words
- ql = [val for val in query.split(' ') if val !='']
+ ql = [val for val in query.split(' ') if val !=''] # list of query words
kw_in_query = []
kw_positions = []
i=0
- while iindex column is not specified
return dic
@@ -130,7 +143,7 @@ def evaluate_from_clause(dic):
Evaluate the part of the query (argument or subquery) that is supplied as the 'from' argument
'''
join_types = ['inner', 'left', 'right', 'full', 'sm', 'inl']
- from_split = dic['from'].split(' ')
+ from_split = dic['from'].split(' ') # if from key in () then we have an inner query from the join_types
if from_split[0] == '(' and from_split[-1] == ')':
subquery = ' '.join(from_split[1:-1])
dic['from'] = interpret(subquery)
@@ -162,7 +175,7 @@ def evaluate_from_clause(dic):
def interpret(query):
'''
- Interpret the query.
+ Interpret the query. (keywords per action)
'''
kw_per_action = {'create table': ['create table'],
'drop table': ['drop table'],
@@ -180,11 +193,13 @@ def interpret(query):
'create view' : ['create view', 'as']
}
- if query[-1]!=';':
+ if query[-1]!=';': # append ; to query if not there
query+=';'
+ # format () and ; with one whitespace before and after
query = query.replace("(", " ( ").replace(")", " ) ").replace(";", " ;").strip()
+ # find action from first word in query
for kw in kw_per_action.keys():
if query.startswith(kw):
action = kw
@@ -213,7 +228,7 @@ def interpret_meta(command):
cdb - change/create database
rmdb - delete database
"""
- action = command.split(' ')[0].removesuffix(';')
+ action = command.split(' ')[0].replace(';','')
db_name = db._name if search_between(command, action,';')=='' else search_between(command, action,';')
@@ -223,10 +238,10 @@ def interpret_meta(command):
verbose = False
def list_databases(db_name):
- [print(fold.removesuffix('_db')) for fold in os.listdir('dbdata')]
+ [print(fold.replace('_db','')) for fold in os.listdir('dbdata')]
def list_tables(db_name):
- [print(pklf.removesuffix('.pkl')) for pklf in os.listdir(f'dbdata/{db_name}_db') if pklf.endswith('.pkl')\
+ [print(pklf.replace('.pkl','')) for pklf in os.listdir(f'dbdata/{db_name}_db') if pklf.endswith('.pkl')\
and not pklf.startswith('meta')]
def change_db(db_name):
@@ -258,7 +273,7 @@ def remove_db(db_name):
for line in open(fname, 'r').read().splitlines():
if line.startswith('--'): continue
if line.startswith('explain'):
- dic = interpret(line.removeprefix('explain '))
+ dic = interpret(line.replace('explain ',''))
pprint(dic, sort_dicts=False)
else :
dic = interpret(line.lower())
@@ -284,11 +299,14 @@ def remove_db(db_name):
try:
if line=='exit':
break
- if line.split(' ')[0].removesuffix(';') in ['lsdb', 'lstb', 'cdb', 'rmdb']:
+ if line.split(' ')[0].replace(';','') in ['lsdb', 'lstb', 'cdb', 'rmdb']:
interpret_meta(line)
elif line.startswith('explain'):
- dic = interpret(line.removeprefix('explain '))
+ dic = interpret(line.replace('explain ',''))
pprint(dic, sort_dicts=False)
+ elif line.startswith('equivalent of'):
+ dic = interpret(line.replace('equivalent of ',''))
+ equiv_print(dic)
else:
dic = interpret(line)
result = execute_dic(dic)
diff --git a/miniDB/database.py b/miniDB/database.py
index a3ac6be7..46851497 100644
--- a/miniDB/database.py
+++ b/miniDB/database.py
@@ -13,6 +13,7 @@
from joins import Inlj, Smj
from btree import Btree
+from hash import Hash # file name might need change
from misc import split_condition
from table import Table
@@ -54,7 +55,8 @@ def __init__(self, name, load=True, verbose = True):
self.create_table('meta_length', 'table_name,no_of_rows', 'str,int')
self.create_table('meta_locks', 'table_name,pid,mode', 'str,int,str')
self.create_table('meta_insert_stack', 'table_name,indexes', 'str,list')
- self.create_table('meta_indexes', 'table_name,index_name', 'str,str')
+ #added index_colum column and index_type in order to keep the column on which index is created and whether it is hash or btree
+ self.create_table('meta_indexes', 'table_name,index_name,index_column,index_type', 'str,str,str,str')
self.save_database()
def save_database(self):
@@ -101,7 +103,7 @@ def _update(self):
self._update_meta_insert_stack()
- def create_table(self, name, column_names, column_types, primary_key=None, load=None):
+ def create_table(self, name, column_names, column_types, primary_key=None,unique=None, load=None):
'''
This method create a new table. This table is saved and can be accessed via db_object.tables['table_name'] or db_object.table_name
@@ -113,7 +115,7 @@ def create_table(self, name, column_names, column_types, primary_key=None, load=
load: boolean. Defines table object parameters as the name of the table and the column names.
'''
# print('here -> ', column_names.split(','))
- self.tables.update({name: Table(name=name, column_names=column_names.split(','), column_types=column_types.split(','), primary_key=primary_key, load=load)})
+ self.tables.update({name: Table(name=name, column_names=column_names.split(','), column_types=column_types.split(','), primary_key=primary_key,unique=unique, load=load)})
# self._name = Table(name=name, column_names=column_names, column_types=column_types, load=load)
# check that new dynamic var doesnt exist already
# self.no_of_tables += 1
@@ -354,22 +356,61 @@ def select(self, columns, table_name, condition, distinct=None, order_by=None, \
# print(table_name)
self.load_database()
- if isinstance(table_name,Table):
+ if isinstance(table_name,Table): # is table in database?
return table_name._select_where(columns, condition, distinct, order_by, desc, limit)
- if condition is not None:
- condition_column = split_condition(condition)[0]
+ condition_AND=False
+ condition_OR=False
+
+ if condition is not None: # get column of where clause
+ if "between" in condition: #if condition contains the keyword "between",then condition has the format of table.column between value1 and value2
+ condition_column=condition.split(" ")[0]
+ elif "and" in condition:
+ condition_AND=True
+ condition_column = ''
+ elif "or" in condition:
+ condition_OR=True
+ condition_column = ''
+ elif "not " in condition:
+ cond=condition[4:]
+ condition_column = split_condition(cond)[0]
+ else:
+ condition_column = split_condition(condition)[0]
else:
condition_column = ''
-
# self.lock_table(table_name, mode='x')
if self.is_locked(table_name):
return
- if self._has_index(table_name) and condition_column==self.tables[table_name].column_names[self.tables[table_name].pk_idx]:
- index_name = self.select('*', 'meta_indexes', f'table_name={table_name}', return_object=True).column_by_name('index_name')[0]
- bt = self._load_idx(index_name)
- table = self.tables[table_name]._select_where_with_btree(columns, bt, condition, distinct, order_by, desc, limit)
+
+ # contains all indexes related to the condition column
+ if condition_column != '':
+ meta_data = self.select('*', 'meta_indexes', f'table_name={table_name} and index_column={condition_column}', return_object=True).data
+ else:
+ meta_data = [[]]
+
+ if condition_AND is True or condition_OR is True:
+ table = self.tables[table_name]._select_where(columns, condition, distinct, order_by, desc, limit)
+ elif meta_data!=[[]]: # if there is an index for the condition column
+ index_name=None
+ # search for a hash index since we prefer it from a btree in identity queries
+ if "not " not in condition and ">" not in condition and "<" not in condition and "between" not in condition and "like" not in condition and "in" not in condition and "is" not in condition:
+ for row in meta_data:
+ if row[3]=="hash":
+ index_name=row[1]
+ # if no hash was found or condition is range query , select the most recently created btree
+ if index_name is None:
+ for row in meta_data:
+ if row[3]=="btree":
+ index_name=row[1]
+ if index_name is None: # if no index was found
+ table = self.tables[table_name]._select_where(columns, condition, distinct, order_by, desc, limit)
+ else:
+ bt = self._load_idx(index_name)
+ table = self.tables[table_name]._select_where_with_btree(columns, bt, condition, distinct, order_by, desc, limit)
+ else:
+ hsh = self._load_idx(index_name)
+ table = self.tables[table_name]._select_where_with_hash(columns, hsh, condition)
else:
table = self.tables[table_name]._select_where(columns, condition, distinct, order_by, desc, limit)
# self.unlock_table(table_name)
@@ -650,7 +691,7 @@ def _update_meta_insert_stack_for_tb(self, table_name, new_stack):
# indexes
- def create_index(self, index_name, table_name, index_type='btree'):
+ def create_index(self, index_name, table_name, index_type='btree',index_column=None):
'''
Creates an index on a specified table with a given name.
Important: An index can only be created on a primary key (the user does not specify the column).
@@ -658,44 +699,82 @@ def create_index(self, index_name, table_name, index_type='btree'):
Args:
table_name: string. Table name (must be part of database).
index_name: string. Name of the created index.
+ index_column: string. Name of the column on which we create the index
'''
- if self.tables[table_name].pk_idx is None: # if no primary key, no index
- raise Exception('Cannot create index. Table has no primary key.')
- if index_name not in self.tables['meta_indexes'].column_by_name('index_name'):
- # currently only btree is supported. This can be changed by adding another if.
+
+ #if index_column not specified by user,index creation on primary key
+ if index_column is None:
+ if self.tables[table_name].pk_idx is not None:
+ index_column=self.tables[table_name].pk
+ else:
+ raise Exception('Cannot create index. Table has no primary key.')# if no primary key, no index
+ else:
+ try:
+ col_names=self.tables[table_name].column_names #we find the column names of the table
+ index_column_idx=col_names.index(index_column) #the index of the given column(on which we create the index) in the list of column names
+ if index_column_idx not in self.tables[table_name].unique_idx:
+ raise Exception('Cannot create index on a column that is not unique.')
+ except:
+ raise ValueError('The requested column to index does not exist in the table.')
+
+ if index_name not in self.tables['meta_indexes'].column_by_name('index_name'):#if there isn't already an index with the same name
if index_type=='btree':
logging.info('Creating Btree index.')
- # insert a record with the name of the index and the table on which it's created to the meta_indexes table
- self.tables['meta_indexes']._insert([table_name, index_name])
- # crate the actual index
- self._construct_index(table_name, index_name)
+ # insert a record with the name of the index,the table and the column on which it's created to the meta_indexes table
+ self.tables['meta_indexes']._insert([table_name, index_name,index_column,index_type])
+ # create the actual index
+ self._construct_index(table_name, index_name,index_column)
+ self.save_database()
+ if index_type=='hash':
+ logging.info('Creating Hash index.')
+ # insert a record with the name of the index,the table,the column on which it's created and the index type to the meta_indexes table
+ self.tables['meta_indexes']._insert([table_name, index_name,index_column,index_type])
+ # create the actual index
+ self._construct_hash_index(table_name, index_name,index_column)
self.save_database()
else:
raise Exception('Cannot create index. Another index with the same name already exists.')
- def _construct_index(self, table_name, index_name):
+ def _construct_index(self, table_name, index_name,index_column):
'''
Construct a btree on a table and save.
Args:
table_name: string. Table name (must be part of database).
index_name: string. Name of the created index.
+ index_column: string. Name of the column on which we create the index
'''
bt = Btree(3) # 3 is arbitrary
-
- # for each record in the primary key of the table, insert its value and index to the btree
- for idx, key in enumerate(self.tables[table_name].column_by_name(self.tables[table_name].pk)):
+ # for each record of the table, insert its column value and index to the btree
+ for idx, key in enumerate(self.tables[table_name].column_by_name(index_column)):
if key is None:
continue
bt.insert(key, idx)
# save the btree
self._save_index(index_name, bt)
+ def _construct_hash_index(self, table_name, index_name,index_column):
+ '''
+ Construct a hash index on a table and save.
+
+ Args:
+ table_name: string. Table name (must be part of database).
+ index_name: string. Name of the created index.
+ index_column: string. Name of the column on which we create the index
+ '''
+ hash_table=Hash(3) # argument is blocking factor
+ # for each record of the table, insert its column value and index to the hash index
+ for idx, key in enumerate(self.tables[table_name].column_by_name(index_column)):
+ if key is None:
+ continue
+ hash_table.insert(key, idx)
+ hash_table.show()
+ # save the hashtable
+ self._save_index(index_name, hash_table)
def _has_index(self, table_name):
'''
Check whether the specified table's primary key column is indexed.
-
Args:
table_name: string. Table name (must be part of database).
'''
diff --git a/miniDB/equivalentQueries.py b/miniDB/equivalentQueries.py
new file mode 100644
index 00000000..9b03b281
--- /dev/null
+++ b/miniDB/equivalentQueries.py
@@ -0,0 +1,139 @@
+
+import copy
+
+
+def rule1(query):
+ '''
+ Given a query, apply the transformation rule
+ σ q1∧q2 (R) = σ q1(σ q2 (R))
+ query: the query to transform
+ '''
+ if 'select' in query.keys() and isinstance(query['from'],dict) and 'select ' in query['from'].keys():
+ equiv_query = copy.deepcopy(query)
+ # Split the condition into two parts
+ condition1, condition2 = equiv_query['where'].split('and')
+ equiv_query['where'] = condition1
+ equiv_query['from'] = {'select': '*', 'from': equiv_query['from'], 'where': condition2,'distinct': None, 'orderby': None,'limit': None,'desc': None}
+ return equiv_query
+ return None
+
+def rule2(query):
+ '''
+ Rule 2: σθ1(σθ2(R)) = σθ2(σθ1(R))
+ '''
+ if 'select' in query.keys() and 'select' in query['from'].keys():
+ equiv_query = copy.deepcopy(query)
+ condition1=equiv_query['where'] # θ1
+ condition2=equiv_query['from']['where'] # θ2
+ equiv_query['where'] = condition2
+ equiv_query['from']['where'] = condition1
+ return equiv_query
+ return None
+
+def rule3(query):
+ '''
+ Rule 3b: σθ1(R ⋈θ2 S) = R ⋈(θ1^θ2) S
+ '''
+ if 'select' in query.keys() and (isinstance(query['from'], dict) and 'join' in query['from'].keys()) and query['where'] is not None and query['from']['on'] is not None:
+ equiv_query = copy.deepcopy(query)
+ condition1=equiv_query['where'] # θ1
+ condition2=equiv_query['from']['on'] # θ2
+ equiv_query['where'] = None
+ equiv_query['from']['on'] = condition1 + ' and ' + condition2 # θ1^θ2 as the join condition
+ return equiv_query
+ return None
+
+def rule4(query):
+ '''
+ Given a query, apply the transformation rule
+ R ⋈q S = S ⋈q R
+ '''
+
+ if 'join' in query.keys():
+ equiv_query = copy.deepcopy(query)
+ left=equiv_query['from']['left']
+ right=equiv_query['from']['right']
+ equiv_query['from']['left'] = right
+ equiv_query['from']['right'] = left
+ return equiv_query
+ return None
+
+def rule5(query):
+ '''
+ Given a query, apply the transformation rules:
+ -( θ-join )
+ (R ⋈q1 S) ⋈ q2^q3 T = R ⋈ q1^q3 (S ⋈ q2 T)
+ '''
+ # multiple joins are not supported by minidb for now
+ pass
+
+
+def rule6(query):
+ '''
+ Given a query, apply the transformation rules
+ case 1: σ q1(R ⋈q S) = σ q1(R) ⋈q S
+ case 2: σ q1(R ⋈q S) = R ⋈q σ q1(S)
+ case 3: σ q1^q2 (R ⋈q S) = σ q1(R) ⋈q σ q2 (S)
+ '''
+ equivalent_queries = []
+ if 'select' in query.keys() and ((isinstance(query['from'], dict) and 'join' in query['from'].keys())):
+ if query['where'] is not None and 'and' in query['where']:
+ equiv_query= copy.deepcopy(query)
+ # Split the condition into two parts
+ condition1, condition2 = equiv_query['where'].split('and')
+ # Apply the rule(case of double select condition: condition1 corresponds to left table and condition2 corresponds to right table)
+ subquery1 = {'select': '*', 'from': query['from']['left'], 'where': condition1, 'distinct': None, 'orderby': None,'limit': None,'desc': None}
+ subquery2 = {'select': '*', 'from': query['from']['right'], 'where': condition2, 'distinct': None, 'orderby': None,'limit': None,'desc': None}
+ equiv_query['from']['left']=subquery1
+ equiv_query['from']['right']=subquery2
+ equiv_query['where']=None
+ equivalent_queries.append(equiv_query)
+
+ # Apply the rule(case: condition corresponds only to left table)
+ equiv_query= copy.deepcopy(query)
+ subquery1 = {'select': '*', 'from': query['from']['left'], 'where': query['where'], 'distinct': None, 'orderby': None,'limit': None,'desc': None}
+ equiv_query['from']['left']=subquery1
+ equiv_query['where']=None
+ equivalent_queries.append(equiv_query)
+ # Apply the rule(case: condition corresponds only to right table)
+ equiv_query= copy.deepcopy(query)
+ subquery2 = {'select': '*', 'from': query['from']['right'], 'where': query['where'], 'distinct': None, 'orderby': None,'limit': None,'desc': None}
+ equiv_query['from']['right']=subquery2
+ equiv_query['where']=None
+ equivalent_queries.append(equiv_query)
+ return equivalent_queries
+ return None
+
+def equiv_recursive(query, equiv_queries=None, lastrule=None, visited=None):
+ # keep track of the visited queries as a frozen set
+ if equiv_queries is None:
+ equiv_queries = []
+ if query in equiv_queries:
+ return equiv_queries
+ equiv_queries.append(query)
+ for rule in [rule1, rule2, rule3, rule4, rule5, rule6]:
+ equiv_query = rule(query)
+ if equiv_query is not None and lastrule != rule:
+ lastrule = rule
+ if rule == rule6:
+ for q in equiv_query:
+ equiv_queries.append(q)
+ equiv_recursive(q, equiv_queries, lastrule, visited)
+ else:
+ equiv_queries.append(equiv_query)
+ equiv_recursive(equiv_query, equiv_queries, lastrule, visited)
+ return equiv_queries
+
+def equiv_print(query):
+ equiv_queries=equiv_recursive(query)
+ for count,q in enumerate(equiv_queries):
+ print ("\n"+"Query "+str(count+1)+": \n"+str(q)+"\n")
+
+# Example dictionary
+exampleQuery = {'select': '*', 'from': {'select': '*', 'from': 'R', 'where': 'a > 5', 'distinct': None, 'orderby': None,'limit': None,'desc': None}, 'where': 'b < 10', 'distinct': None, 'orderby': None,'limit': None,'desc': None}
+# Second example dictionary with join
+exampleQuery2 = {'select': '*', 'from': {'join': 'inner', 'left': 'instructor', 'right': 'department', 'on': 'instructor.dept_name=department.dept_name'}, 'where': 'not instructor.dept_name=biology ', 'distinct': None, 'order by': None, 'limit': None, 'desc': None}
+# Third example dictionary with join and 'and' condition
+exampleQuery3 = {'select': '*', 'from': {'join': 'inner', 'left': 'instructor', 'right': 'department', 'on': 'instructor.dept_name=department.dept_name'}, 'where': 'not instructor.dept_name=biology and instructor.salary>60000', 'distinct': None, 'order by': None, 'limit': None, 'desc': None}
+# Print query
+#equiv_print(exampleQuery3)
\ No newline at end of file
diff --git a/miniDB/hash.py b/miniDB/hash.py
new file mode 100644
index 00000000..a4e6ed92
--- /dev/null
+++ b/miniDB/hash.py
@@ -0,0 +1,175 @@
+'''
+https://gunet2.cs.unipi.gr/modules/document/file.php/TMC110/Lectures/05-FilesIndexing.pdf Slide 20 (Extendible Hashing)
+'''
+import binascii # we will convert values to hex bytes and then take the modulo of the corresponding int with the key
+
+class Hash:
+
+ def __init__(self, b, key=2047): # 1st mersenne prime
+ '''
+ The hash index abstraction.
+ '''
+ self.b = b # blocking factor
+ self.hash_prefix = {} # search dictionary
+ self.key = key # hash key
+ self.i = 1 # number of bits used for hash prefix
+ # create as many buckets as hash prefix keys
+ self.buckets = [Bucket(self.b,self.i) for i in range(2**self.i)] # list of buckets. Every value is placed by hashing function in one of these buckets
+ for j in range(2**self.i):
+ # dictionary keys are binary representations of MSB number range (0 to 2^msb-1)
+ self.hash_prefix[format(j,'0'+str(self.i)+'b')]=self.buckets[j] # map hash to equivalent bucket
+
+
+ def insert(self, value, ptr):
+ # key to be hashed
+ h_key=self.calc_hash(value)
+
+ # formatted as binary value
+ bin_val=format(h_key,'011b')
+ bits=self.i_MSB(bin_val)
+ selected=self.hash_prefix[bits] # selected bucket (pointer)
+ # Check if record fits in the bucket
+ if (len(selected.data)bucket_j.i: # there is a pointer available for the new bucket
+ bucket_j.i+=1 # update how many bits we are using
+ bucket_z=Bucket(self.b,bucket_j.i)
+ self.buckets.append(bucket_z)
+
+ #find the lower half of hash prefixes/pointers that point to bucket_j and change them to point to bucket_z
+ for prefix, ptr in self.hash_prefix.items():
+ if prefix.startswith(bits[0:bucket_j.i]) and ptr==bucket_j:
+ self.hash_prefix[prefix]=bucket_z
+
+ # apply the hash function to each record of bucket j and according to the first i bits,the record stays in bucket_j or moves to bucket_z
+ j_new_data={}
+
+ for r_key,r_value in bucket_j.data.items():
+
+ # key to be hashed
+ h_key=self.calc_hash(r_key)
+ # formatted as binary value
+ bin_val=format(h_key,'011b')
+ bits=self.i_MSB(bin_val)
+
+ bucket_insert=self.hash_prefix[bits]
+ if bucket_insert==bucket_j:
+ j_new_data[r_key]=r_value
+ else:
+ bucket_z.data[r_key]=r_value
+
+ bucket_j.data=j_new_data
+
+
+ def show(self):
+ '''
+ Prints the hash index.
+
+ Args:
+ None
+ '''
+ print("i="+str(self.i))
+ for prefix, bucket in self.hash_prefix.items():
+ print("Prefix="+prefix+", i_bucket="+str(bucket.i)+" => "+str(bucket.data))
+
+
+ def calc_hash(self,value):
+ '''
+ The hash function used to calculate the hash value of a given value.
+
+ Args:
+ value: float or string. The value to be hashed.
+ '''
+ if isinstance(value, str):
+ value=int(binascii.hexlify(bytes(value,'utf-8')), 16)
+ return value%self.key #we will get the MSB from this value
+
+ def i_MSB(self,bin_value):
+ '''
+ returns the i MSB of a binary value
+
+ Args:
+ bin_value: string. The binary value to be truncated.
+ '''
+ return bin_value[0:self.i]
+
+
+
+class Bucket:
+ '''
+ Bucket abstraction.
+
+ Explanation of the attribute i:
+ Although i bits are required to find the correct bucket using the hash prefix dictionary,
+ several contiguous dictionary keys can point to the same bucket.
+ All these keys will have a common prefix, but the length of this prefix can be less than i.
+ So,we associate an integer with each bucket that gives the length of that common hash prefix.
+ '''
+ def __init__(self, b,i):
+ self.b = b # number of records held in block (blocking factor)
+ self.i=i # number of bits used for hash prefix
+ self.data = {} # dictionary of records and their pointers
+
+ def find(self, value):
+ '''
+ Returns the pointer of the record with the given value.
+
+ Args:
+ value: float or string. The value of the record to be found.
+ '''
+ return self.data[value]
\ No newline at end of file
diff --git a/miniDB/misc.py b/miniDB/misc.py
index aefada74..550969bc 100644
--- a/miniDB/misc.py
+++ b/miniDB/misc.py
@@ -8,7 +8,9 @@ def get_op(op, a, b):
'<': operator.lt,
'>=': operator.ge,
'<=': operator.le,
- '=': operator.eq}
+ '=': operator.eq,
+ '<>': operator.ne
+ }
try:
return ops[op](a,b)
@@ -20,12 +22,14 @@ def split_condition(condition):
'<=': operator.le,
'=': operator.eq,
'>': operator.gt,
- '<': operator.lt}
+ '<': operator.lt,
+ '<>': operator.ne
+ }
for op_key in ops.keys():
- splt=condition.split(op_key)
- if len(splt)>1:
- left, right = splt[0].strip(), splt[1].strip()
+ splt=condition.split(op_key) # split string on operator
+ if len(splt)>1: # if split successful
+ left, right = splt[0].strip(), splt[1].strip() # values for condition
if right[0] == '"' == right[-1]: # If the value has leading and trailing quotes, remove them.
right = right.strip('"')
@@ -39,7 +43,7 @@ def split_condition(condition):
def reverse_op(op):
'''
- Reverse the operator given
+ Reverse the operator given
'''
return {
'>' : '<',
@@ -48,3 +52,15 @@ def reverse_op(op):
'<=' : '>=',
'=' : '='
}.get(op)
+
+def reverse_op_not(op):
+ '''
+ Reverse the operator given for NOT keyword
+ '''
+ return {
+ '>' : '<=',
+ '>=' : '<',
+ '<' : '>=',
+ '<=' : '>',
+ '=' : '<>'
+ }.get(op)
diff --git a/miniDB/table.py b/miniDB/table.py
index f5c7d937..0f93d2da 100644
--- a/miniDB/table.py
+++ b/miniDB/table.py
@@ -6,7 +6,8 @@
sys.path.append(f'{os.path.dirname(os.path.dirname(os.path.abspath(__file__)))}/miniDB')
-from misc import get_op, split_condition
+from misc import get_op, reverse_op_not, split_condition
+from hash import Hash
class Table:
@@ -26,7 +27,7 @@ class Table:
- a dictionary that includes the appropriate info (all the attributes in __init__)
'''
- def __init__(self, name=None, column_names=None, column_types=None, primary_key=None, load=None):
+ def __init__(self, name=None, column_names=None, column_types=None, primary_key=None,unique=None, load=None):
if load is not None:
# if load is a dict, replace the object dict with it (replaces the object with the specified one)
@@ -70,6 +71,15 @@ def __init__(self, name=None, column_names=None, column_types=None, primary_key=
self.pk = primary_key
# self._update()
+ # if unique is set, keep its index as an attribute
+ if unique is not None:
+ self.unique_idx=[]
+ for u in unique:
+ self.unique_idx.append(self.column_names.index(u))
+ else:
+ self.unique_idx = None
+
+ self.unique = unique
# if any of the name, columns_names and column types are none. return an empty table object
def column_by_name(self, column_name):
@@ -125,11 +135,19 @@ def _insert(self, row, insert_stack=[]):
print(exc)
# if value is to be appended to the primary_key column, check that it doesnt alrady exist (no duplicate primary keys)
+
if i==self.pk_idx and row[i] in self.column_by_name(self.pk):
- raise ValueError(f'## ERROR -> Value {row[i]} already exists in primary key column.')
+ raise ValueError(f'## ERROR -> Value {row[i]} already exists in primary key column "{self.pk}".')
elif i==self.pk_idx and row[i] is None:
raise ValueError(f'ERROR -> The value of the primary key cannot be None.')
-
+
+ # if value is to be appended to a unique column, check that it doesnt alrady exist
+ if self.unique_idx is not None:
+ if i in self.unique_idx and row[i] in self.column_by_name(self.unique[self.unique_idx.index(i)]):
+ raise ValueError(f'## ERROR -> Value {row[i]} already exists in unique column.')
+ elif i in self.unique_idx and row[i] is None:
+ raise ValueError(f'ERROR -> The value of a unique field cannot be None.')
+
# if insert_stack is not empty, append to its last index
if insert_stack != []:
self.data[insert_stack[-1]] = row
@@ -182,20 +200,60 @@ def _delete_where(self, condition):
Operatores supported: (<,<=,==,>=,>)
'''
- column_name, operator, value = self._parse_condition(condition)
- indexes_to_del = []
+ rows = []
+ if condition is not None:
+
+ # BETWEEN case
+ if "between" in condition:
+ condition_list=condition.split(" ")
+ if len(condition_list)!=5 or condition_list[1]!="between" or condition_list[3]!="and":
+ raise Exception("Condition containing between keyword must have the following format: column between value1 and value2")
+ else:
+ try:
+ column=self.column_by_name(condition_list[0])
+ value1=int(condition_list[2])
+ value2=int(condition_list[4])
+ rows = [ind for ind, x in enumerate(column) if value1 <= x <= value2] #append index of row if x between value1 and value2
+ except:
+ raise TypeError('You need to provide numeric values inside a between statement')
+ # AND case
+ elif "and" in condition:
+ all_rows=[]
+ seperated_conditions=condition.split(" and ")
+ for cond in seperated_conditions: #select all the rows that match each condition
+ column_name, operator, value = self._parse_condition(cond)
+ column = self.column_by_name(column_name)
+ all_rows+=[ind for ind, x in enumerate(column) if get_op(operator, x, value)]#add indexes to all_rows
+ # Keep the rows from the all_rows list that appear as many times as the number of conditions in the AND clauses
+ rows=[*set(i for i in all_rows if all_rows.count(i) > len(seperated_conditions)-1)]
+ # OR case
+ elif "or" in condition:
+ all_rows=[]
+ seperated_conditions=condition.split(" or ")
+ for cond in seperated_conditions: #select all the rows that match each condition
+ column_name, operator, value = self._parse_condition(cond)
+ column = self.column_by_name(column_name)
+ all_rows+=[ind for ind, x in enumerate(column) if get_op(operator, x, value)]#add indexes to all_rows
+ rows=[*set(all_rows)] #keep all rows that match either of the conditions but first remove the double values
+ else:
+ column_name, operator, value = self._parse_condition(condition)
+ column = self.column_by_name(column_name)
+ rows = [ind for ind, x in enumerate(column) if get_op(operator, x, value)]
+
+ else:
+ rows = [i for i in range(len(self.data))]
column = self.column_by_name(column_name)
for index, row_value in enumerate(column):
if get_op(operator, row_value, value):
- indexes_to_del.append(index)
+ rows.append(index)
# we pop from highest to lowest index in order to avoid removing the wrong item
# since we dont delete, we dont have to to pop in that order, but since delete is used
# to delete from meta tables too, we still implement it.
- for index in sorted(indexes_to_del, reverse=True):
+ for index in sorted(rows, reverse=True):
if self._name[:4] != 'meta':
# if the table is not a metatable, replace the row with a row of nones
self.data[index] = [None for _ in range(len(self.column_names))]
@@ -204,7 +262,7 @@ def _delete_where(self, condition):
# self._update()
# we have to return the deleted indexes, since they will be appended to the insert_stack
- return indexes_to_del
+ return rows
def _select_where(self, return_columns, condition=None, distinct=False, order_by=None, desc=True, limit=None):
@@ -233,9 +291,44 @@ def _select_where(self, return_columns, condition=None, distinct=False, order_by
# if condition is None, return all rows
# if not, return the rows with values where condition is met for value
if condition is not None:
- column_name, operator, value = self._parse_condition(condition)
- column = self.column_by_name(column_name)
- rows = [ind for ind, x in enumerate(column) if get_op(operator, x, value)]
+
+ # BETWEEN case
+ if "between" in condition:
+ condition_list=condition.split(" ")
+ if len(condition_list)!=5 or condition_list[1]!="between" or condition_list[3]!="and":
+ raise Exception("Condition containing between keyword must have the following format: column between value1 and value2")
+ else:
+ try:
+ column=self.column_by_name(condition_list[0])
+ value1=int(condition_list[2])
+ value2=int(condition_list[4])
+ rows = [ind for ind, x in enumerate(column) if value1 <= x <= value2] #append index of row if x between value1 and value2
+ except:
+ raise TypeError('You need to provide numeric values inside a between statement')
+ # AND case
+ elif "and" in condition:
+ all_rows=[]
+ seperated_conditions=condition.split(" and ")
+ for cond in seperated_conditions: #select all the rows that match each condition
+ column_name, operator, value = self._parse_condition(cond)
+ column = self.column_by_name(column_name)
+ all_rows+=[ind for ind, x in enumerate(column) if get_op(operator, x, value)]#add indexes to all_rows
+ # Keep the rows from the all_rows list that appear as many times as the number of conditions in the AND clauses
+ rows=[*set(i for i in all_rows if all_rows.count(i) > len(seperated_conditions)-1)]
+ # OR case
+ elif "or" in condition:
+ all_rows=[]
+ seperated_conditions=condition.split(" or ")
+ for cond in seperated_conditions: #select all the rows that match each condition
+ column_name, operator, value = self._parse_condition(cond)
+ column = self.column_by_name(column_name)
+ all_rows+=[ind for ind, x in enumerate(column) if get_op(operator, x, value)]#add indexes to all_rows
+ rows=[*set(all_rows)] #keep all rows that match either of the conditions but first remove the double values
+ else:
+ column_name, operator, value = self._parse_condition(condition)
+ column = self.column_by_name(column_name)
+ rows = [ind for ind, x in enumerate(column) if get_op(operator, x, value)]
+
else:
rows = [i for i in range(len(self.data))]
@@ -268,7 +361,7 @@ def _select_where(self, return_columns, condition=None, distinct=False, order_by
s_table.data = [row for row in s_table.data if any(row)][:int(limit)]
return s_table
-
+
def _select_where_with_btree(self, return_columns, bt, condition, distinct=False, order_by=None, desc=True, limit=None):
@@ -278,51 +371,106 @@ def _select_where_with_btree(self, return_columns, bt, condition, distinct=False
else:
return_cols = [self.column_names.index(colname) for colname in return_columns]
+ value1=None
+ value2=None
+ conditionBetween=False
+ if "between" in condition: # Supporting between search on btree
+ condition_list=condition.split(" ")
+ if len(condition_list)!=5 or condition_list[1]!="between" or condition_list[3]!="and":
+ raise Exception("Condition containing between keyword must have the following format: column between value1 and value2")
+ else:
+ try:
+ column_name=condition_list[0]
+ value1=int(condition_list[2])
+ value2=int(condition_list[4])
+ conditionBetween=True
+ except:
+ raise TypeError('You need to provide numeric values inside a between statement')
+ else: # simple where statement
+ column_name, operator, value = self._parse_condition(condition)
- column_name, operator, value = self._parse_condition(condition)
+ #if the column in condition is a primary key or a unique column,continue the select
+ if column_name in self.unique or (self.pk is not None and column_name==self.pk) :
+ # here we run the same select twice, sequentially and using the btree.
+ # we then check the results match and compare performance (number of operations)
+ if conditionBetween:
+ rows1 = bt.find(">=",value1) # lower bound rows
+ rows2 = bt.find("<=",value2) # upper bound rows
+ rows = [row for row in rows1 if row in rows2] # intersection
+ else:
+ column = self.column_by_name(column_name)
+ # sequential
+ rows1 = []
+ opsseq = 0
+ for ind, x in enumerate(column):
+ opsseq+=1
+ if get_op(operator, x, value):
+ rows1.append(ind)
+
+ # btree find
+ rows = bt.find(operator, value)
- # if the column in condition is not a primary key, abort the select
- if column_name != self.column_names[self.pk_idx]:
- print('Column is not PK. Aborting')
+ try:
+ k = int(limit)
+ except TypeError:
+ k = None
+ # same as simple select from now on
+ rows = rows[:k]
+ # TODO: this needs to be dumbed down
+ dict = {(key):([[self.data[i][j] for j in return_cols] for i in rows] if key=="data" else value) for key,value in self.__dict__.items()}
- # here we run the same select twice, sequentially and using the btree.
- # we then check the results match and compare performance (number of operation)
- column = self.column_by_name(column_name)
+ dict['column_names'] = [self.column_names[i] for i in return_cols]
+ dict['column_types'] = [self.column_types[i] for i in return_cols]
- # sequential
- rows1 = []
- opsseq = 0
- for ind, x in enumerate(column):
- opsseq+=1
- if get_op(operator, x, value):
- rows1.append(ind)
+ s_table = Table(load=dict)
- # btree find
- rows = bt.find(operator, value)
+ s_table.data = list(set(map(lambda x: tuple(x), s_table.data))) if distinct else s_table.data
- try:
- k = int(limit)
- except TypeError:
- k = None
- # same as simple select from now on
- rows = rows[:k]
- # TODO: this needs to be dumbed down
- dict = {(key):([[self.data[i][j] for j in return_cols] for i in rows] if key=="data" else value) for key,value in self.__dict__.items()}
+ if order_by:
+ s_table.order_by(order_by, desc)
- dict['column_names'] = [self.column_names[i] for i in return_cols]
- dict['column_types'] = [self.column_types[i] for i in return_cols]
+ if isinstance(limit,str):
+ s_table.data = [row for row in s_table.data if row is not None][:int(limit)]
- s_table = Table(load=dict)
+ return s_table
+ else:
+ print('Column is neither a PK nor unique. Aborting')# if the column in condition is not a primary key or a unique column, abort the select
+ return
- s_table.data = list(set(map(lambda x: tuple(x), s_table.data))) if distinct else s_table.data
+ def _select_where_with_hash(self, return_columns, hsh, condition):
+
+ # if * return all columns, else find the column indexes for the columns specified
+ if return_columns == '*':
+ return_cols = [i for i in range(len(self.column_names))]
+ else:
+ return_cols = [self.column_names.index(colname) for colname in return_columns]
- if order_by:
- s_table.order_by(order_by, desc)
- if isinstance(limit,str):
- s_table.data = [row for row in s_table.data if row is not None][:int(limit)]
+ column_name, operator, value = self._parse_condition(condition)
+
+ if operator != '=':
+ print('Hash index only supports equality. Aborting')
+ return
+
+ #if the column in condition is a primary key or a unique column,continue the select
+ if column_name in self.unique or (self.pk is not None and column_name==self.pk) :
+ column = self.column_by_name(column_name)
+ #hash index find
+ ptr = hsh.find(value)
+ dict = {(key):([[self.data[ptr][j] for j in return_cols]] if key=="data" else value) for key,value in self.__dict__.items()}
+
+ dict['column_names'] = [self.column_names[i] for i in return_cols]
+ dict['column_types'] = [self.column_types[i] for i in return_cols]
+
+ s_table = Table(load=dict)
+ return s_table
+
+ else:
+ print('Column is neither a PK nor unique. Aborting')# if the column in condition is not a primary key or a unique column, abort the select
+ return
+
+
- return s_table
def order_by(self, column_name, desc=True):
'''
@@ -533,6 +681,10 @@ def show(self, no_of_rows=None, is_locked=False):
if self.pk_idx is not None:
# table has a primary key, add PK next to the appropriate column
headers[self.pk_idx] = headers[self.pk_idx]+' #PK#'
+ if self.unique_idx is not None:
+ # table has unique columns, add UQ next to the appropriate column
+ for i in self.unique_idx:
+ headers[i] = headers[i]+' #UQ#'
# detect the rows that are no tfull of nones (these rows have been deleted)
# if we dont skip these rows, the returning table has empty rows at the deleted positions
non_none_rows = [row for row in self.data if any(row)]
@@ -552,6 +704,11 @@ def _parse_condition(self, condition, join=False):
Operatores supported: (<,<=,==,>=,>)
join: boolean. Whether to join or not (False by default).
'''
+ not_condition = False
+ if "not " in condition: #not condition
+ condition=condition[4:] # remove not keyword
+ not_condition = True
+
# if both_columns (used by the join function) return the names of the names of the columns (left first)
if join:
return split_condition(condition)
@@ -561,7 +718,8 @@ def _parse_condition(self, condition, join=False):
if left not in self.column_names:
raise ValueError(f'Condition is not valid (cant find column name)')
coltype = self.column_types[self.column_names.index(left)]
-
+ if not_condition:
+ return left,reverse_op_not(op), coltype(right)
return left, op, coltype(right)
diff --git a/sql_files/smallRelationsInsertFile.sql b/sql_files/smallRelationsInsertFile.sql
index d05d81b9..117cd8e2 100644
--- a/sql_files/smallRelationsInsertFile.sql
+++ b/sql_files/smallRelationsInsertFile.sql
@@ -1,4 +1,4 @@
-create table classroom (building str, room_number str, capacity int);
+create table classroom (building str, room_number str, capacity int unique);
create table department (dept_name str primary key, building str, budget int);
create table course (course_id str primary key, title str, dept_name str, credits int);
create table instructor (ID str primary key, name str, dept_name str, salary int);