From 4609a349ed1193d799542d32176270bbc3e90851 Mon Sep 17 00:00:00 2001 From: apostolossvls Date: Sun, 19 Feb 2023 11:41:37 +0200 Subject: [PATCH 1/2] Added Not, Between. Needs Tweaking --- miniDB/dashboard.py | 1 + miniDB/database.py | 5 +- miniDB/misc.py | 1 + miniDB/table.py | 18 +- miniDBold/__init__.py | 0 miniDBold/btree.py | 348 +++++++++++++++++++ miniDBold/dashboard.py | 15 + miniDBold/database.py | 748 +++++++++++++++++++++++++++++++++++++++++ miniDBold/joins.py | 309 +++++++++++++++++ miniDBold/misc.py | 65 ++++ miniDBold/table.py | 579 +++++++++++++++++++++++++++++++ 11 files changed, 2084 insertions(+), 5 deletions(-) create mode 100644 miniDBold/__init__.py create mode 100644 miniDBold/btree.py create mode 100644 miniDBold/dashboard.py create mode 100644 miniDBold/database.py create mode 100644 miniDBold/joins.py create mode 100644 miniDBold/misc.py create mode 100644 miniDBold/table.py diff --git a/miniDB/dashboard.py b/miniDB/dashboard.py index ee97b0a8..5a2c6f8b 100644 --- a/miniDB/dashboard.py +++ b/miniDB/dashboard.py @@ -13,3 +13,4 @@ if sys.argv[2]=='meta' and name[:4]!='meta': continue db.show_table(name) + diff --git a/miniDB/database.py b/miniDB/database.py index a3ac6be7..9809f1fb 100644 --- a/miniDB/database.py +++ b/miniDB/database.py @@ -358,7 +358,8 @@ def select(self, columns, table_name, condition, distinct=None, order_by=None, \ return table_name._select_where(columns, condition, distinct, order_by, desc, limit) if condition is not None: - condition_column = split_condition(condition)[0] + if((" between " not in condition) and ("not " not in condition)): + condition_column = split_condition(condition)[0] else: condition_column = '' @@ -745,4 +746,4 @@ def drop_index(self, index_name): warnings.warn(f'"{self.savedir}/indexes/meta_{index_name}_index.pkl" not found.') self.save_database() - \ No newline at end of file + \ No newline at end of file diff --git a/miniDB/misc.py b/miniDB/misc.py index aefada74..2975b265 100644 --- a/miniDB/misc.py +++ b/miniDB/misc.py @@ -48,3 +48,4 @@ def reverse_op(op): '<=' : '>=', '=' : '=' }.get(op) + diff --git a/miniDB/table.py b/miniDB/table.py index f5c7d937..fb8ec374 100644 --- a/miniDB/table.py +++ b/miniDB/table.py @@ -233,9 +233,20 @@ def _select_where(self, return_columns, condition=None, distinct=False, order_by # if condition is None, return all rows # if not, return the rows with values where condition is met for value if condition is not None: - column_name, operator, value = self._parse_condition(condition) - column = self.column_by_name(column_name) - rows = [ind for ind, x in enumerate(column) if get_op(operator, x, value)] + if((" between " not in condition) and ("not " not in condition)): + column_name, operator, value = self._parse_condition(condition) + column = self.column_by_name(column_name) + rows = [ind for ind, x in enumerate(column) if get_op(operator, x, value)] + elif(" between " in str(condition)): + + # filter + rows = [i for i in range(len(self.data))] + + elif("not " in str(condition)): + + # filter + rows = [i for i in range(len(self.data))] + else: rows = [i for i in range(len(self.data))] @@ -577,3 +588,4 @@ def _load_from_file(self, filename): f.close() self.__dict__.update(tmp_dict.__dict__) + \ No newline at end of file diff --git a/miniDBold/__init__.py b/miniDBold/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/miniDBold/btree.py b/miniDBold/btree.py new file mode 100644 index 00000000..37e8223b --- /dev/null +++ b/miniDBold/btree.py @@ -0,0 +1,348 @@ +''' +https://en.wikipedia.org/wiki/B%2B_tree +''' + +class Node: + ''' + Node abstraction. Represents a single bucket + ''' + def __init__(self, b, values=None, ptrs=None,left_sibling=None, right_sibling=None, parent=None, is_leaf=False): + self.b = b # branching factor + self.values = [] if values is None else values # Values (the data from the pk column) + self.ptrs = [] if ptrs is None else ptrs # ptrs (the indexes of each datapoint or the index of another bucket) + self.left_sibling = left_sibling # the index of a buckets left sibling + self.right_sibling = right_sibling # the index of a buckets right sibling + self.parent = parent # the index of a buckets parent + self.is_leaf = is_leaf # a boolean value signaling whether the node is a leaf or not + + + def find(self, value, return_ops=False): + ''' + Returns the index of the next node to search for a value if the node is not a leaf (a ptrs of the available ones). + If it is a leaf (we have found the appropriate node), nothing is returned. + + Args: + value: float. The value being searched for. + return_ops: boolean. Set to True if you want to use the number of operations (for benchmarking). + ''' + ops = 0 # number of operations (<>= etc). Used for benchmarking + if self.is_leaf: # + return + + # for each value in the node, if the user supplied value is smaller, return the btrees value index + # else (no value in the node is larger) return the last ptr + for index, existing_val in enumerate(self.values): + ops+=1 + if value is None or existing_val is None: + continue + if value= etc). Used for benchmarking + + #start with the root node + node = self.nodes[self.root] + # while the node that we are searching in is not a leaf + # keep searching + while not node.is_leaf: + idx, ops1 = node.find(value, return_ops=True) + node = self.nodes[idx] + ops += ops1 + + # finally return the index of the appropriate node (and the ops if you want to) + if return_ops: + return self.nodes.index(node), ops + else: + return self.nodes.index(node) + + + def split(self, node_id): + ''' + Split the node with index=node_id. + + Args: + node_id: float. The corresponding ID of the node. + ''' + # fetch the node to be split + node = self.nodes[node_id] + # the value that will be propagated to the parent is the middle one. + new_parent_value = node.values[len(node.values)//2] + if node.is_leaf: + # if the node is a leaf, the parent value should be a part of the new node (right) + # Important: in a b+tree, every value should appear in a leaf + right_values = node.values[len(node.values)//2:] + right_ptrs = node.ptrs[len(node.ptrs)//2:] + + # create the new node with the right half of the old nodes values and ptrs (including the middle ones) + right = Node(self.b, right_values, right_ptrs,\ + left_sibling=node_id, right_sibling=node.right_sibling, parent=node.parent, is_leaf=node.is_leaf) + # since the new node (right) will be the next one to be appended to the nodes list + # its index will be equal to the length of the nodes list. + # Thus we set the old nodes (now left) right sibling to the right nodes future index (len of nodes) + if node.right_sibling is not None: + self.nodes[node.right_sibling].left_sibling = len(self.nodes) + node.right_sibling = len(self.nodes) + + + else: + # if the node is not a leaf, the parent value shoudl NOT be part of the new node + right_values = node.values[len(node.values)//2+1:] + if self.b%2==1: + right_ptrs = node.ptrs[len(node.ptrs)//2:] + else: + right_ptrs = node.ptrs[len(node.ptrs)//2+1:] + + # if nonleafs should be connected change the following two lines and add siblings + right = Node(self.b, right_values, right_ptrs,\ + parent=node.parent, is_leaf=node.is_leaf) + # make sure that a non leaf node doesnt have a parent + node.right_sibling = None + # the right node's kids should have him as a parent (if not all nodes will have left as parent) + for ptr in right_ptrs: + self.nodes[ptr].parent = len(self.nodes) + + # old node (left) keeps only the first half of the values/ptrs + node.values = node.values[:len(node.values)//2] + if self.b%2==1: + node.ptrs = node.ptrs[:len(node.ptrs)//2] + else: + node.ptrs = node.ptrs[:len(node.ptrs)//2+1] + + # append the new node (right) to the nodes list + self.nodes.append(right) + + # If the new nodes have no parents (a new level needs to be added + if node.parent is None: + # its the root that is split + # new root contains the parent value and ptrs to the two recently split nodes + parent = Node(self.b, [new_parent_value], [node_id, len(self.nodes)-1]\ + ,parent=node.parent, is_leaf=False) + + # set root, and parent of split celss to the index of the new root node (len of nodes-1) + self.nodes.append(parent) + self.root = len(self.nodes)-1 + node.parent = len(self.nodes)-1 + right.parent = len(self.nodes)-1 + else: + # insert the parent value to the parent + + self.nodes[node.parent].insert(new_parent_value, len(self.nodes)-1) + # check whether the parent needs to be split + if len(self.nodes[node.parent].values)==self.b: + self.split(node.parent) + + + + + def show(self): + ''' + Show important info for each node (sort by level - root first, then left to right). + ''' + nds = [] + nds.append(self.root) + for ptr in nds: + if self.nodes[ptr].is_leaf: + continue + nds.extend(self.nodes[ptr].ptrs) + + for ptr in nds: + print(f'## {ptr} ##') + self.nodes[ptr].show() + print('----') + + + def plot(self): + ## arrange the nodes top to bottom left to right + nds = [] + nds.append(self.root) + for ptr in nds: + if self.nodes[ptr].is_leaf: + continue + nds.extend(self.nodes[ptr].ptrs) + + # add each node and each link + g = 'digraph G{\nforcelabels=true;\n' + + for i in nds: + node = self.nodes[i] + g+=f'{i} [label="{node.values}"]\n' + if node.is_leaf: + continue + # if node.left_sibling is not None: + # g+=f'"{node.values}"->"{self.nodes[node.left_sibling].values}" [color="blue" constraint=false];\n' + # if node.right_sibling is not None: + # g+=f'"{node.values}"->"{self.nodes[node.right_sibling].values}" [color="green" constraint=false];\n' + # + # g+=f'"{node.values}"->"{self.nodes[node.parent].values}" [color="red" constraint=false];\n' + else: + for child in node.ptrs: + g+=f'{child} [label="{self.nodes[child].values}"]\n' + g+=f'{i}->{child};\n' + g +="}" + + try: + from graphviz import Source + src = Source(g) + src.render('bplustree', view=True) + except ImportError: + print('"graphviz" package not found. Writing to graph.gv.') + with open('graph.gv','w') as f: + f.write(g) + + def find(self, operator, value): + ''' + Return ptrs of elements where btree_value"operator"value. + Important, the user supplied "value" is the right value of the operation. That is why the operation are reversed below. + The left value of the op is the btree value. + + Args: + operator: string. The provided evaluation operator. + value: float. The value being searched for. + ''' + results = [] + # find the index of the node that the element should exist in + leaf_idx, ops = self._search(value, True) + target_node = self.nodes[leaf_idx] + + if operator == '=': + # if the element exist, append to list, else pass and return + try: + results.append(target_node.ptrs[target_node.values.index(value)]) + # print('Found') + except: + # print('Not found') + pass + + # for all other ops, the code is the same, only the operations themselves and the sibling indexes change + # for > and >= (btree value is >/>= of user supplied value), we return all the right siblings (all values are larger than current cell) + # for < and <= (btree value is ': + for idx, node_value in enumerate(target_node.values): + ops+=1 + if node_value > value: + results.append(target_node.ptrs[idx]) + while target_node.right_sibling is not None: + target_node = self.nodes[target_node.right_sibling] + results.extend(target_node.ptrs) + + + if operator == '>=': + for idx, node_value in enumerate(target_node.values): + ops+=1 + if node_value >= value: + results.append(target_node.ptrs[idx]) + while target_node.right_sibling is not None: + target_node = self.nodes[target_node.right_sibling] + results.extend(target_node.ptrs) + + if operator == '<': + for idx, node_value in enumerate(target_node.values): + ops+=1 + if node_value < value: + results.append(target_node.ptrs[idx]) + while target_node.left_sibling is not None: + target_node = self.nodes[target_node.left_sibling] + results.extend(target_node.ptrs) + + if operator == '<=': + for idx, node_value in enumerate(target_node.values): + ops+=1 + if node_value <= value: + results.append(target_node.ptrs[idx]) + while target_node.left_sibling is not None: + target_node = self.nodes[target_node.left_sibling] + results.extend(target_node.ptrs) + + # print the number of operations (usefull for benchamrking) + # print(f'With BTree -> {ops} comparison operations') + return results diff --git a/miniDBold/dashboard.py b/miniDBold/dashboard.py new file mode 100644 index 00000000..ee97b0a8 --- /dev/null +++ b/miniDBold/dashboard.py @@ -0,0 +1,15 @@ +import sys +import os + +sys.path.append(f'{os.path.dirname(os.path.dirname(os.path.abspath(__file__)))}/miniDB') + +from database import Database + + +db = Database(sys.argv[1]) + + +for name in list(db.tables): + if sys.argv[2]=='meta' and name[:4]!='meta': + continue + db.show_table(name) diff --git a/miniDBold/database.py b/miniDBold/database.py new file mode 100644 index 00000000..a3ac6be7 --- /dev/null +++ b/miniDBold/database.py @@ -0,0 +1,748 @@ +from __future__ import annotations +import pickle +from time import sleep, localtime, strftime +import os,sys +import logging +import warnings +import readline +from tabulate import tabulate + +sys.path.append(f'{os.path.dirname(os.path.dirname(os.path.abspath(__file__)))}/miniDB') +from miniDB import table +sys.modules['table'] = table + +from joins import Inlj, Smj +from btree import Btree +from misc import split_condition +from table import Table + + +# readline.clear_history() + +class Database: + ''' + Main Database class, containing tables. + ''' + + def __init__(self, name, load=True, verbose = True): + self.tables = {} + self._name = name + self.verbose = verbose + + self.savedir = f'dbdata/{name}_db' + + if load: + try: + self.load_database() + logging.info(f'Loaded "{name}".') + return + except: + if verbose: + warnings.warn(f'Database "{name}" does not exist. Creating new.') + + # create dbdata directory if it doesnt exist + if not os.path.exists('dbdata'): + os.mkdir('dbdata') + + # create new dbs save directory + try: + os.mkdir(self.savedir) + except: + pass + + # create all the meta tables + self.create_table('meta_length', 'table_name,no_of_rows', 'str,int') + self.create_table('meta_locks', 'table_name,pid,mode', 'str,int,str') + self.create_table('meta_insert_stack', 'table_name,indexes', 'str,list') + self.create_table('meta_indexes', 'table_name,index_name', 'str,str') + self.save_database() + + def save_database(self): + ''' + Save database as a pkl file. This method saves the database object, including all tables and attributes. + ''' + for name, table in self.tables.items(): + with open(f'{self.savedir}/{name}.pkl', 'wb') as f: + pickle.dump(table, f) + + def _save_locks(self): + ''' + Stores the meta_locks table to file as meta_locks.pkl. + ''' + with open(f'{self.savedir}/meta_locks.pkl', 'wb') as f: + pickle.dump(self.tables['meta_locks'], f) + + def load_database(self): + ''' + Load all tables that are part of the database (indices noted here are loaded). + + Args: + path: string. Directory (path) of the database on the system. + ''' + path = f'dbdata/{self._name}_db' + for file in os.listdir(path): + + if file[-3:]!='pkl': # if used to load only pkl files + continue + f = open(path+'/'+file, 'rb') + tmp_dict = pickle.load(f) + f.close() + name = f'{file.split(".")[0]}' + self.tables.update({name: tmp_dict}) + # setattr(self, name, self.tables[name]) + + #### IO #### + + def _update(self): + ''' + Update all meta tables. + ''' + self._update_meta_length() + self._update_meta_insert_stack() + + + def create_table(self, name, column_names, column_types, primary_key=None, load=None): + ''' + This method create a new table. This table is saved and can be accessed via db_object.tables['table_name'] or db_object.table_name + + Args: + name: string. Name of table. + column_names: list. Names of columns. + column_types: list. Types of columns. + primary_key: string. The primary key (if it exists). + load: boolean. Defines table object parameters as the name of the table and the column names. + ''' + # print('here -> ', column_names.split(',')) + self.tables.update({name: Table(name=name, column_names=column_names.split(','), column_types=column_types.split(','), primary_key=primary_key, load=load)}) + # self._name = Table(name=name, column_names=column_names, column_types=column_types, load=load) + # check that new dynamic var doesnt exist already + # self.no_of_tables += 1 + self._update() + self.save_database() + # (self.tables[name]) + if self.verbose: + print(f'Created table "{name}".') + + + def drop_table(self, table_name): + ''' + Drop table from current database. + + Args: + table_name: string. Name of table. + ''' + self.load_database() + self.lock_table(table_name) + + self.tables.pop(table_name) + if os.path.isfile(f'{self.savedir}/{table_name}.pkl'): + os.remove(f'{self.savedir}/{table_name}.pkl') + else: + warnings.warn(f'"{self.savedir}/{table_name}.pkl" not found.') + self.delete_from('meta_locks', f'table_name={table_name}') + self.delete_from('meta_length', f'table_name={table_name}') + self.delete_from('meta_insert_stack', f'table_name={table_name}') + + if self._has_index(table_name): + to_be_deleted = [] + for key, table in enumerate(self.tables['meta_indexes'].column_by_name('table_name')): + if table == table_name: + to_be_deleted.append(key) + + for i in reversed(to_be_deleted): + self.drop_index(self.tables['meta_indexes'].data[i][1]) + + try: + delattr(self, table_name) + except AttributeError: + pass + # self._update() + self.save_database() + + + def import_table(self, table_name, filename, column_types=None, primary_key=None): + ''' + Creates table from CSV file. + + Args: + filename: string. CSV filename. If not specified, filename's name will be used. + column_types: list. Types of columns. If not specified, all will be set to type str. + primary_key: string. The primary key (if it exists). + ''' + file = open(filename, 'r') + + first_line=True + for line in file.readlines(): + if first_line: + colnames = line.strip('\n') + if column_types is None: + column_types = ",".join(['str' for _ in colnames.split(',')]) + self.create_table(name=table_name, column_names=colnames, column_types=column_types, primary_key=primary_key) + lock_ownership = self.lock_table(table_name, mode='x') + first_line = False + continue + self.tables[table_name]._insert(line.strip('\n').split(',')) + + if lock_ownership: + self.unlock_table(table_name) + self._update() + self.save_database() + + + def export(self, table_name, filename=None): + ''' + Transform table to CSV. + + Args: + table_name: string. Name of table. + filename: string. Output CSV filename. + ''' + res = '' + for row in [self.tables[table_name].column_names]+self.tables[table_name].data: + res+=str(row)[1:-1].replace('\'', '').replace('"','').replace(' ','')+'\n' + + if filename is None: + filename = f'{table_name}.csv' + + with open(filename, 'w') as file: + file.write(res) + + def table_from_object(self, new_table): + ''' + Add table object to database. + + Args: + new_table: string. Name of new table. + ''' + + self.tables.update({new_table._name: new_table}) + if new_table._name not in self.__dir__(): + setattr(self, new_table._name, new_table) + else: + raise Exception(f'"{new_table._name}" attribute already exists in class "{self.__class__.__name__}".') + self._update() + self.save_database() + + + + ##### table functions ##### + + # In every table function a load command is executed to fetch the most recent table. + # In every table function, we first check whether the table is locked. Since we have implemented + # only the X lock, if the tables is locked we always abort. + # After every table function, we update and save. Update updates all the meta tables and save saves all + # tables. + + # these function calls are named close to the ones in postgres + + def cast(self, column_name, table_name, cast_type): + ''' + Modify the type of the specified column and cast all prexisting values. + (Executes type() for every value in column and saves) + + Args: + table_name: string. Name of table (must be part of database). + column_name: string. The column that will be casted (must be part of database). + cast_type: type. Cast type (do not encapsulate in quotes). + ''' + self.load_database() + + lock_ownership = self.lock_table(table_name, mode='x') + self.tables[table_name]._cast_column(column_name, eval(cast_type)) + if lock_ownership: + self.unlock_table(table_name) + self._update() + self.save_database() + + def insert_into(self, table_name, row_str): + ''' + Inserts data to given table. + + Args: + table_name: string. Name of table (must be part of database). + row: list. A list of values to be inserted (will be casted to a predifined type automatically). + lock_load_save: boolean. If False, user needs to load, lock and save the states of the database (CAUTION). Useful for bulk-loading. + ''' + row = row_str.strip().split(',') + self.load_database() + # fetch the insert_stack. For more info on the insert_stack + # check the insert_stack meta table + lock_ownership = self.lock_table(table_name, mode='x') + insert_stack = self._get_insert_stack_for_table(table_name) + try: + self.tables[table_name]._insert(row, insert_stack) + except Exception as e: + logging.info(e) + logging.info('ABORTED') + self._update_meta_insert_stack_for_tb(table_name, insert_stack[:-1]) + + if lock_ownership: + self.unlock_table(table_name) + self._update() + self.save_database() + + + def update_table(self, table_name, set_args, condition): + ''' + Update the value of a column where a condition is met. + + Args: + table_name: string. Name of table (must be part of database). + set_value: string. New value of the predifined column name. + set_column: string. The column to be altered. + condition: string. A condition using the following format: + 'column[<,<=,==,>=,>]value' or + 'value[<,<=,==,>=,>]column'. + + Operatores supported: (<,<=,==,>=,>) + ''' + set_column, set_value = set_args.replace(' ','').split('=') + self.load_database() + + lock_ownership = self.lock_table(table_name, mode='x') + self.tables[table_name]._update_rows(set_value, set_column, condition) + if lock_ownership: + self.unlock_table(table_name) + self._update() + self.save_database() + + def delete_from(self, table_name, condition): + ''' + Delete rows of table where condition is met. + + Args: + table_name: string. Name of table (must be part of database). + condition: string. A condition using the following format: + 'column[<,<=,==,>=,>]value' or + 'value[<,<=,==,>=,>]column'. + + Operatores supported: (<,<=,==,>=,>) + ''' + self.load_database() + + lock_ownership = self.lock_table(table_name, mode='x') + deleted = self.tables[table_name]._delete_where(condition) + if lock_ownership: + self.unlock_table(table_name) + self._update() + self.save_database() + # we need the save above to avoid loading the old database that still contains the deleted elements + if table_name[:4]!='meta': + self._add_to_insert_stack(table_name, deleted) + self.save_database() + + def select(self, columns, table_name, condition, distinct=None, order_by=None, \ + limit=True, desc=None, save_as=None, return_object=True): + ''' + Selects and outputs a table's data where condtion is met. + + Args: + table_name: string. Name of table (must be part of database). + columns: list. The columns that will be part of the output table (use '*' to select all available columns) + condition: string. A condition using the following format: + 'column[<,<=,==,>=,>]value' or + 'value[<,<=,==,>=,>]column'. + + Operatores supported: (<,<=,==,>=,>) + order_by: string. A column name that signals that the resulting table should be ordered based on it (no order if None). + desc: boolean. If True, order_by will return results in descending order (True by default). + limit: int. An integer that defines the number of rows that will be returned (all rows if None). + save_as: string. The name that will be used to save the resulting table into the database (no save if None). + return_object: boolean. If True, the result will be a table object (useful for internal use - the result will be printed by default). + distinct: boolean. If True, the resulting table will contain only unique rows. + ''' + + # print(table_name) + self.load_database() + if isinstance(table_name,Table): + return table_name._select_where(columns, condition, distinct, order_by, desc, limit) + + if condition is not None: + condition_column = split_condition(condition)[0] + else: + condition_column = '' + + + # self.lock_table(table_name, mode='x') + if self.is_locked(table_name): + return + if self._has_index(table_name) and condition_column==self.tables[table_name].column_names[self.tables[table_name].pk_idx]: + index_name = self.select('*', 'meta_indexes', f'table_name={table_name}', return_object=True).column_by_name('index_name')[0] + bt = self._load_idx(index_name) + table = self.tables[table_name]._select_where_with_btree(columns, bt, condition, distinct, order_by, desc, limit) + else: + table = self.tables[table_name]._select_where(columns, condition, distinct, order_by, desc, limit) + # self.unlock_table(table_name) + if save_as is not None: + table._name = save_as + self.table_from_object(table) + else: + if return_object: + return table + else: + return table.show() + + + def show_table(self, table_name, no_of_rows=None): + ''' + Print table in a readable tabular design (using tabulate). + + Args: + table_name: string. Name of table (must be part of database). + ''' + self.load_database() + + self.tables[table_name].show(no_of_rows, self.is_locked(table_name)) + + + def sort(self, table_name, column_name, asc=False): + ''' + Sorts a table based on a column. + + Args: + table_name: string. Name of table (must be part of database). + column_name: string. the column name that will be used to sort. + asc: If True sort will return results in ascending order (False by default). + ''' + + self.load_database() + + lock_ownership = self.lock_table(table_name, mode='x') + self.tables[table_name]._sort(column_name, asc=asc) + if lock_ownership: + self.unlock_table(table_name) + self._update() + self.save_database() + + def create_view(self, table_name, table): + ''' + Create a virtual table based on the result-set of the SQL statement provided. + + Args: + table_name: string. Name of the table that will be saved. + table: table. The table that will be saved. + ''' + table._name = table_name + self.table_from_object(table) + + def join(self, mode, left_table, right_table, condition, save_as=None, return_object=True): + ''' + Join two tables that are part of the database where condition is met. + + Args: + left_table: string. Name of the left table (must be in DB) or Table obj. + right_table: string. Name of the right table (must be in DB) or Table obj. + condition: string. A condition using the following format: + 'column[<,<=,==,>=,>]value' or + 'value[<,<=,==,>=,>]column'. + + Operators supported: (<,<=,==,>=,>) + save_as: string. The output filename that will be used to save the resulting table in the database (won't save if None). + return_object: boolean. If True, the result will be a table object (useful for internal usage - the result will be printed by default). + ''' + self.load_database() + if self.is_locked(left_table) or self.is_locked(right_table): + return + + left_table = left_table if isinstance(left_table, Table) else self.tables[left_table] + right_table = right_table if isinstance(right_table, Table) else self.tables[right_table] + + + if mode=='inner': + res = left_table._inner_join(right_table, condition) + + elif mode=='left': + res = left_table._left_join(right_table, condition) + + elif mode=='right': + res = left_table._right_join(right_table, condition) + + elif mode=='full': + res = left_table._full_join(right_table, condition) + + elif mode=='inl': + # Check if there is an index of either of the two tables available, as if there isn't we can't use inlj + leftIndexExists = self._has_index(left_table._name) + rightIndexExists = self._has_index(right_table._name) + + if not leftIndexExists and not rightIndexExists: + res = None + raise Exception('Index-nested-loop join cannot be executed. Use inner join instead.\n') + elif rightIndexExists: + index_name = self.select('*', 'meta_indexes', f'table_name={right_table._name}', return_object=True).column_by_name('index_name')[0] + res = Inlj(condition, left_table, right_table, self._load_idx(index_name), 'right').join() + elif leftIndexExists: + index_name = self.select('*', 'meta_indexes', f'table_name={left_table._name}', return_object=True).column_by_name('index_name')[0] + res = Inlj(condition, left_table, right_table, self._load_idx(index_name), 'left').join() + + elif mode=='sm': + res = Smj(condition, left_table, right_table).join() + + else: + raise NotImplementedError + + if save_as is not None: + res._name = save_as + self.table_from_object(res) + else: + if return_object: + return res + else: + res.show() + + if return_object: + return res + else: + res.show() + + def lock_table(self, table_name, mode='x'): + ''' + Locks the specified table using the exclusive lock (X). + + Args: + table_name: string. Table name (must be part of database). + ''' + if table_name[:4]=='meta' or table_name not in self.tables.keys() or isinstance(table_name,Table): + return + + with open(f'{self.savedir}/meta_locks.pkl', 'rb') as f: + self.tables.update({'meta_locks': pickle.load(f)}) + + try: + pid = self.tables['meta_locks']._select_where('pid',f'table_name={table_name}').data[0][0] + if pid!=os.getpid(): + raise Exception(f'Table "{table_name}" is locked by process with pid={pid}') + else: + return False + + except IndexError: + pass + + if mode=='x': + self.tables['meta_locks']._insert([table_name, os.getpid(), mode]) + else: + raise NotImplementedError + self._save_locks() + return True + # print(f'Locking table "{table_name}"') + + def unlock_table(self, table_name, force=False): + ''' + Unlocks the specified table that is exclusively locked (X). + + Args: + table_name: string. Table name (must be part of database). + ''' + if table_name not in self.tables.keys(): + raise Exception(f'Table "{table_name}" is not in database') + + if not force: + try: + # pid = self.select('*','meta_locks', f'table_name={table_name}', return_object=True).data[0][1] + pid = self.tables['meta_locks']._select_where('pid',f'table_name={table_name}').data[0][0] + if pid!=os.getpid(): + raise Exception(f'Table "{table_name}" is locked by the process with pid={pid}') + except IndexError: + pass + self.tables['meta_locks']._delete_where(f'table_name={table_name}') + self._save_locks() + # print(f'Unlocking table "{table_name}"') + + def is_locked(self, table_name): + ''' + Check whether the specified table is exclusively locked (X). + + Args: + table_name: string. Table name (must be part of database). + ''' + if isinstance(table_name,Table) or table_name[:4]=='meta': # meta tables will never be locked (they are internal) + return False + + with open(f'{self.savedir}/meta_locks.pkl', 'rb') as f: + self.tables.update({'meta_locks': pickle.load(f)}) + + try: + pid = self.tables['meta_locks']._select_where('pid',f'table_name={table_name}').data[0][0] + if pid!=os.getpid(): + raise Exception(f'Table "{table_name}" is locked by the process with pid={pid}') + + except IndexError: + pass + return False + + + #### META #### + + # The following functions are used to update, alter, load and save the meta tables. + # Important: Meta tables contain info regarding the NON meta tables ONLY. + # i.e. meta_length will not show the number of rows in meta_locks etc. + + def _update_meta_length(self): + ''' + Updates the meta_length table. + ''' + for table in self.tables.values(): + if table._name[:4]=='meta': #skip meta tables + continue + if table._name not in self.tables['meta_length'].column_by_name('table_name'): # if new table, add record with 0 no. of rows + self.tables['meta_length']._insert([table._name, 0]) + + # the result needs to represent the rows that contain data. Since we use an insert_stack + # some rows are filled with Nones. We skip these rows. + non_none_rows = len([row for row in table.data if any(row)]) + self.tables['meta_length']._update_rows(non_none_rows, 'no_of_rows', f'table_name={table._name}') + # self.update_row('meta_length', len(table.data), 'no_of_rows', 'table_name', '==', table._name) + + def _update_meta_locks(self): + ''' + Updates the meta_locks table. + ''' + for table in self.tables.values(): + if table._name[:4]=='meta': #skip meta tables + continue + if table._name not in self.tables['meta_locks'].column_by_name('table_name'): + + self.tables['meta_locks']._insert([table._name, False]) + # self.insert('meta_locks', [table._name, False]) + + def _update_meta_insert_stack(self): + ''' + Updates the meta_insert_stack table. + ''' + for table in self.tables.values(): + if table._name[:4]=='meta': #skip meta tables + continue + if table._name not in self.tables['meta_insert_stack'].column_by_name('table_name'): + self.tables['meta_insert_stack']._insert([table._name, []]) + + + def _add_to_insert_stack(self, table_name, indexes): + ''' + Adds provided indices to the insert stack of the specified table. + + Args: + table_name: string. Table name (must be part of database). + indexes: list. The list of indices that will be added to the insert stack (the indices of the newly deleted elements). + ''' + old_lst = self._get_insert_stack_for_table(table_name) + self._update_meta_insert_stack_for_tb(table_name, old_lst+indexes) + + def _get_insert_stack_for_table(self, table_name): + ''' + Returns the insert stack of the specified table. + + Args: + table_name: string. Table name (must be part of database). + ''' + return self.tables['meta_insert_stack']._select_where('*', f'table_name={table_name}').column_by_name('indexes')[0] + # res = self.select('meta_insert_stack', '*', f'table_name={table_name}', return_object=True).indexes[0] + # return res + + def _update_meta_insert_stack_for_tb(self, table_name, new_stack): + ''' + Replaces the insert stack of a table with the one supplied by the user. + + Args: + table_name: string. Table name (must be part of database). + new_stack: string. The stack that will be used to replace the existing one. + ''' + self.tables['meta_insert_stack']._update_rows(new_stack, 'indexes', f'table_name={table_name}') + + + # indexes + def create_index(self, index_name, table_name, index_type='btree'): + ''' + Creates an index on a specified table with a given name. + Important: An index can only be created on a primary key (the user does not specify the column). + + Args: + table_name: string. Table name (must be part of database). + index_name: string. Name of the created index. + ''' + if self.tables[table_name].pk_idx is None: # if no primary key, no index + raise Exception('Cannot create index. Table has no primary key.') + if index_name not in self.tables['meta_indexes'].column_by_name('index_name'): + # currently only btree is supported. This can be changed by adding another if. + if index_type=='btree': + logging.info('Creating Btree index.') + # insert a record with the name of the index and the table on which it's created to the meta_indexes table + self.tables['meta_indexes']._insert([table_name, index_name]) + # crate the actual index + self._construct_index(table_name, index_name) + self.save_database() + else: + raise Exception('Cannot create index. Another index with the same name already exists.') + + def _construct_index(self, table_name, index_name): + ''' + Construct a btree on a table and save. + + Args: + table_name: string. Table name (must be part of database). + index_name: string. Name of the created index. + ''' + bt = Btree(3) # 3 is arbitrary + + # for each record in the primary key of the table, insert its value and index to the btree + for idx, key in enumerate(self.tables[table_name].column_by_name(self.tables[table_name].pk)): + if key is None: + continue + bt.insert(key, idx) + # save the btree + self._save_index(index_name, bt) + + + def _has_index(self, table_name): + ''' + Check whether the specified table's primary key column is indexed. + + Args: + table_name: string. Table name (must be part of database). + ''' + return table_name in self.tables['meta_indexes'].column_by_name('table_name') + + def _save_index(self, index_name, index): + ''' + Save the index object. + + Args: + index_name: string. Name of the created index. + index: obj. The actual index object (btree object). + ''' + try: + os.mkdir(f'{self.savedir}/indexes') + except: + pass + + with open(f'{self.savedir}/indexes/meta_{index_name}_index.pkl', 'wb') as f: + pickle.dump(index, f) + + def _load_idx(self, index_name): + ''' + Load and return the specified index. + + Args: + index_name: string. Name of created index. + ''' + f = open(f'{self.savedir}/indexes/meta_{index_name}_index.pkl', 'rb') + index = pickle.load(f) + f.close() + return index + + def drop_index(self, index_name): + ''' + Drop index from current database. + + Args: + index_name: string. Name of index. + ''' + if index_name in self.tables['meta_indexes'].column_by_name('index_name'): + self.delete_from('meta_indexes', f'index_name = {index_name}') + + if os.path.isfile(f'{self.savedir}/indexes/meta_{index_name}_index.pkl'): + os.remove(f'{self.savedir}/indexes/meta_{index_name}_index.pkl') + else: + warnings.warn(f'"{self.savedir}/indexes/meta_{index_name}_index.pkl" not found.') + + self.save_database() + \ No newline at end of file diff --git a/miniDBold/joins.py b/miniDBold/joins.py new file mode 100644 index 00000000..81fd0915 --- /dev/null +++ b/miniDBold/joins.py @@ -0,0 +1,309 @@ +import heapq +import shutil +import os +import ast +import sys + +sys.path.append(f'{os.path.dirname(os.path.dirname(os.path.abspath(__file__)))}/miniDB') + +from misc import reverse_op +from table import Table + + + +class Inlj: + def __init__(self, condition, left_table, right_table, index, index_saved): + self.left_table = left_table + self.right_table = right_table + self.condition = condition + self.join_table = None + self.results = None + self.index = index + self.index_saved = index_saved + + def join(self): + # Get the column of the left and right tables and the operator, from the condition of the join + column_name_left, operator, column_name_right = Table()._parse_condition(self.condition, join=True) + + reversed = False + # If we have the index of the left table, reverse the order of the tables + if(self.index_saved=='left'): + self.right_table, self.left_table = self.left_table, self.right_table + column_name_left, column_name_right = column_name_right, column_name_left + reversed = True + + # Try to find the left column, as even if a reverse took place, it is the only one needed + # If it fails, raise exception + try: + self.column_index_left = self.left_table.column_names.index(column_name_left) + except: + raise Exception(f'Column "{column_name_left}" doesn\'t exist in the left table. Valid columns: {self.left_table.column_names}.') + + # Create the names that appear over the tables when the final joined table is presented to the user + left_names = [f'{self.left_table._name}.{colname}' if self.left_table._name!='' else colname for colname in self.left_table.column_names] + right_names = [f'{self.right_table._name}.{colname}' if self.right_table._name!='' else colname for colname in self.right_table.column_names] + + join_table_colnames = left_names + right_names if not reversed else right_names + left_names + join_table_coltypes = self.left_table.column_types + self.right_table.column_types if not reversed else self.right_table.column_types + self.left_table.column_types + self.join_table = Table(name='', column_names=join_table_colnames, column_types=join_table_coltypes) + + # The operator needs to be reversed as we search based on the elements of the index. + # For example, if A > B and we search based on B, we need to search for B < A + operator = reverse_op(operator) if self.index_saved == 'right' else operator + + # Implementation of the index-nested-loop join + # If the tables had been reversed in the beginning, then the joined table appears + # with the tables shown in the order they appeared in the query + for row_left in self.left_table.data: + # The value that will be searched for in the index + left_value = row_left[self.column_index_left] + self.results = self.index.find(operator, left_value) + if len(self.results) > 0: + for element in self.results: + self.join_table._insert(row_left + self.right_table.data[element] if not reversed else self.right_table.data[element] + row_left) + + return self.join_table + +class Smj: + + def __init__(self, condition, left_table, right_table): + self.left_table = left_table + self.right_table = right_table + self.condition = condition + + def join(self): + # Get the column of the left and right tables and the operator, from the condition of the join + column_name_left, operator, column_name_right = Table()._parse_condition(self.condition, join=True) + column_index_left = self.left_table.column_names.index(column_name_left) + column_index_right = self.right_table.column_names.index(column_name_right) + + if(operator != "="): + raise Exception('Sort-Merge Join is used when the condition operator is "=".\n') + + # Create a temporary folder for the external sort to happen. The folder will be deleted in the end + os.makedirs('tempSMJfolder/externalSortFolder', exist_ok=True) + + # Create the names that appear over the tables when the final joined table is presented to the user + left_names = [f'{self.left_table._name}.{colname}' if self.left_table._name!='' else colname for colname in self.left_table.column_names] + right_names = [f'{self.right_table._name}.{colname}' if self.right_table._name!='' else colname for colname in self.right_table.column_names] + + # Write all the records of the right table to a local file in the following format: + # 'column_name_value [whole record]' + # Use special character '@@@' to represent spaces as spaces break the code. + with open('tempSMJfolder/externalSortFolder/rightTableFile', 'w+') as rt: + for row in self.right_table.data: + if row[column_index_right] is not None: + rt.write(f'{row[self.right_table.column_names.index(column_name_right)]} {str(row).replace(" ","@@@")}\n') + + # Same for the left lable + with open('tempSMJfolder/externalSortFolder/leftTableFile', 'w+') as lt: + for row in self.left_table.data: + if row[column_index_left] is not None: + lt.write(f'{row[self.left_table.column_names.index(column_name_left)]} {str(row).replace(" ","@@@")}\n') + + # Create an ExternalMergeSort object and sort both right table and left table local files + ems = self.ExternalMergeSort() + ems.runExternalSort('rightTableFile') + # Re-initialization of all values + ems = self.ExternalMergeSort() + ems.runExternalSort('leftTableFile') + + # Now there are sorted versions of the local files, so the initial ones can be removed + os.remove('tempSMJfolder/externalSortFolder/rightTableFile') + os.remove('tempSMJfolder/externalSortFolder/leftTableFile') + + # This does the final merge on sort-merge join + with open('tempSMJfolder/externalSortFolder/sorting of rightTableFile', 'r') as right, open('tempSMJfolder/externalSortFolder/sorting of leftTableFile', 'r') as left, open('tempSMJfolder/externalSortFolder/final', 'w+') as final: + mark = None #Used to return to previous values of the file + l = None + r = None + + # The algorithm runs until EOF of the left table file + while l != '': + try: + # If the mark is non-existent, set it equal to the current line of the right table sorted file + # While both files' column_values aren't equal, progress the current lines + if mark is None: + mark = right.tell() + l = left.readline() + r = right.readline() + while l.split()[0] < r.split()[0]: + l = left.readline() + while l.split()[0] > r.split()[0]: + mark = right.tell() + r = right.readline() + + # Now that the column_values are equal save both records to the final, joined_tables local file + # Then progress the right table's current line and continue with the procedure + if l.split()[0] == r.split()[0]: + final.write(l.replace("\n","")[l.index("["):] + " " + r.replace("\n","")[r.index("["):] + '\n') + r = right.readline() + + # Else, if left_value isn't equal to right_value after having found at least one equality of column_values + # return right table's current line to the mark, as the algorithm dictates + else: + right.seek(mark) + mark = None + # Finally, if the right file reaches EOF and IndexError happens, return right table's current line to the mark + except IndexError: + right.seek(mark) + mark = None + + # Now that the final joined file exists, the sorted files are not needed and are thus deleted + os.remove('tempSMJfolder/externalSortFolder/sorting of rightTableFile') + os.remove('tempSMJfolder/externalSortFolder/sorting of leftTableFile') + + join_table_name = '' + join_table_colnames = left_names + right_names + join_table_coltypes = self.left_table.column_types + self.right_table.column_types + join_table = Table(name=join_table_name, column_names=join_table_colnames, column_types= join_table_coltypes) + + # Save merged file first. The hypothesis is that the RAM cannot fit the file, thus we have it saved + # However we load the file to display it like this, might need to be changed in the future + with open('tempSMJfolder/externalSortFolder/final', 'r') as f: + for line in f: + records = line.split() + # ast.literal_eval creates the list [a,b,c] from the string '[a,b,c]' + join_table._insert(ast.literal_eval(records[0].replace('@@@', ' ')) + ast.literal_eval(records[1].replace('@@@', ' '))) + + # Finally, the final file and the externalSortFolder are not needed, as the joined table + # exists in a variable and can be presented to the user + os.remove(f'{os.getcwd()}/tempSMJfolder/externalSortFolder/final') + os.rmdir(f'{os.getcwd()}/tempSMJfolder/externalSortFolder') + os.rmdir(f'{os.getcwd()}/tempSMJfolder') + + return join_table + + class ExternalMergeSort: + # Total number of split files + numFiles = 0 + # Total number of numbers in the first file + sumFiles = 0 + # File name, so as to recognize the sorted file + startingFileName = '' + + # Sort the given array with the merge sort algorithm + def mergeSort(self, arr): + if len(arr) > 1: + mid = len(arr)//2 + L, R = arr[:mid], arr[mid:] + + # Recursive call of merge sort + self.mergeSort(L) + self.mergeSort(R) + + i = j = k = 0 + + while i < len(L) and j < len(R): + if L[i] < R[j]: + arr[k] = L[i] + i += 1 + else: + arr[k] = R[j] + j += 1 + k += 1 + + while i < len(L): + arr[k] = L[i] + i += 1 + k += 1 + + while j < len(R): + arr[k] = R[j] + j += 1 + k += 1 + + # Function to split the big file into smaller chunks of size specified by the user + def splitFile(self, largeFile, chunkSize:int): + # Variables used throughout the class + self.startingFileName = largeFile + self.numFiles = 1 + + # Split the file in chunks of chunkSize bytes + with open(f'tempSMJfolder/externalSortFolder/{largeFile}') as f: + chunk = f.readlines(chunkSize) + while chunk: + os.makedirs(os.path.dirname(f'tempSMJfolder/externalSortFolder/tempSplitFiles {self.startingFileName}/{self.numFiles}'), exist_ok=True) + with open(f'tempSMJfolder/externalSortFolder/tempSplitFiles {self.startingFileName}/{self.numFiles}', 'w+') as chunk_file: + for el in chunk: + chunk_file.write(el) + + chunk = f.readlines(chunkSize) + self.numFiles += 1 + + return self.numFiles + + # Function to sort a chunk of the starting file using merge sort + def sortSmallFile(self, fileToBeSorted): + arr = [] + + with open(f'tempSMJfolder/externalSortFolder/tempSplitFiles {self.startingFileName}/{fileToBeSorted}', 'r') as fts: + # If the contents of the file are integers + try: + with open(f'tempSMJfolder/externalSortFolder/tempSplitFiles {self.startingFileName}/{fileToBeSorted}', 'r') as fts: + arr = list(map(int, fts.read().splitlines())) + # If the contents are alphanumeric values + except: + with open(f'tempSMJfolder/externalSortFolder/tempSplitFiles {self.startingFileName}/{fileToBeSorted}', 'r') as fts: + arr = list(map(str, fts.read().splitlines())) + + self.sumFiles += len(arr) + self.mergeSort(arr) + + with open(f'tempSMJfolder/externalSortFolder/tempSplitFiles {self.startingFileName}/{fileToBeSorted}', 'w') as fts: + for el in arr: + fts.write(f'{el}\n') + + # K-Way Merge with priority queue implementation + def k_wayMerge(self, number): + # Create dictionary of files opened. Open all the files + # That will be merged + fileNames = {} + for i in range(1, number): + fileNames[i] = open(f'tempSMJfolder/externalSortFolder/tempSplitFiles {self.startingFileName}/{i}', 'r') + + output = [] + + # (X,Y) where + # X is the value of the element and + # Y is the key of the file in fileName + try: + pq = [(int(fileNames[i].readline().replace('\n', '')), i) for i in range(1, len(fileNames) + 1)] + except: + for i in range(1, len(fileNames) + 1): + fileNames[i].seek(0) + pq = [(fileNames[i].readline().replace('\n', ''), i) for i in range(1, len(fileNames) + 1)] + + # Create heap for the external merge sort + heapq.heapify(pq) + + while len(output) < self.sumFiles: + elem, file_key = heapq.heappop(pq) + output.append(elem) + next = fileNames[file_key].readline().replace('\n', '') + + # When on EOF, an empty string will be returned + # So if the value is not an empty string, add the integer to the heap + if next != '': + try: + heapq.heappush(pq, (int(next), file_key)) + except: + heapq.heappush(pq, (next, file_key)) + + with open(f'tempSMJfolder/externalSortFolder/sorting of {self.startingFileName}', 'w+') as sf: + for el in output: + sf.write(f'{el}\n') + + return output + + def runExternalSort(self, filename): + # 30 is just an example + self.splitFile(filename, 30) + + for i in range(1, self.numFiles): + self.sortSmallFile(i) + + self.k_wayMerge(self.numFiles) + + # After the k-way Merge is completed, remove the folder containing the temporary split files + shutil.rmtree(f'tempSMJfolder/externalSortFolder/tempSplitFiles {self.startingFileName}/') diff --git a/miniDBold/misc.py b/miniDBold/misc.py new file mode 100644 index 00000000..2a2ae985 --- /dev/null +++ b/miniDBold/misc.py @@ -0,0 +1,65 @@ +import operator + +def get_op(op, a, b): + ''' + Get op as a function of a and b by using a symbol + ''' + ops = {'>': operator.gt, + '<': operator.lt, + '>=': operator.ge, + '<=': operator.le, + '=': operator.eq, + 'not': operator.ne} + + try: + return ops[op](a,b) + except TypeError: # if a or b is None (deleted record), python3 raises typerror + return False + +def split_condition(condition): + ops = {'>=': operator.ge, + '<=': operator.le, + '=': operator.eq, + '>': operator.gt, + '<': operator.lt, + 'not': not_op} + + for op_key in ops.keys(): + splt=condition.split(op_key) + if len(splt)>1: + left, right = splt[0].strip(), splt[1].strip() + + if right[0] == '"' == right[-1]: # If the value has leading and trailing quotes, remove them. + right = right.strip('"') + elif ' ' in right: # If it has whitespaces but no leading and trailing double quotes, throw. + raise ValueError(f'Invalid condition: {condition}\nValue must be enclosed in double quotation marks to include whitespaces.') + + if right.find('"') != -1: # If there are any double quotes in the value, throw. (Notice we've already removed the leading and trailing ones) + raise ValueError(f'Invalid condition: {condition}\nDouble quotation marks are not allowed inside values.') + + return left, op_key, right + +def reverse_op(op): + ''' + Reverse the operator given + ''' + return { + '>' : '<', + '>=' : '<=', + '<' : '>', + '<=' : '>=', + '=' : '=' + }.get(op) + +def not_op(op): + ''' + Return opposite of the operator given + ''' + return { + '>' : '<=', + '>=' : '<', + '<' : '>=', + '<=' : '>', + '=' : '!=', + '!=' : '=' + }.get(op) diff --git a/miniDBold/table.py b/miniDBold/table.py new file mode 100644 index 00000000..f5c7d937 --- /dev/null +++ b/miniDBold/table.py @@ -0,0 +1,579 @@ +from __future__ import annotations +from tabulate import tabulate +import pickle +import os +import sys + +sys.path.append(f'{os.path.dirname(os.path.dirname(os.path.abspath(__file__)))}/miniDB') + +from misc import get_op, split_condition + + +class Table: + ''' + Table object represents a table inside a database + + A Table object can be created either by assigning: + - a table name (string) + - column names (list of strings) + - column types (list of functions like str/int etc) + - primary (name of the primary key column) + + OR + + - by assigning a value to the variable called load. This value can be: + - a path to a Table file saved using the save function + - a dictionary that includes the appropriate info (all the attributes in __init__) + + ''' + def __init__(self, name=None, column_names=None, column_types=None, primary_key=None, load=None): + + if load is not None: + # if load is a dict, replace the object dict with it (replaces the object with the specified one) + if isinstance(load, dict): + self.__dict__.update(load) + # self._update() + # if load is str, load from a file + elif isinstance(load, str): + self._load_from_file(load) + + # if name, columns_names and column types are not none + elif (name is not None) and (column_names is not None) and (column_types is not None): + + self._name = name + + if len(column_names)!=len(column_types): + raise ValueError('Need same number of column names and types.') + + self.column_names = column_names + + self.columns = [] + + for col in self.column_names: + if col not in self.__dir__(): + # this is used in order to be able to call a column using its name as an attribute. + # example: instead of table.columns['column_name'], we do table.column_name + setattr(self, col, []) + self.columns.append([]) + else: + raise Exception(f'"{col}" attribute already exists in "{self.__class__.__name__} "class.') + + self.column_types = [eval(ct) if not isinstance(ct, type) else ct for ct in column_types] + self.data = [] # data is a list of lists, a list of rows that is. + + # if primary key is set, keep its index as an attribute + if primary_key is not None: + self.pk_idx = self.column_names.index(primary_key) + else: + self.pk_idx = None + + self.pk = primary_key + # self._update() + + # if any of the name, columns_names and column types are none. return an empty table object + + def column_by_name(self, column_name): + return [row[self.column_names.index(column_name)] for row in self.data] + + + def _update(self): + ''' + Update all the available columns with the appended rows. + ''' + self.columns = [[row[i] for row in self.data] for i in range(len(self.column_names))] + for ind, col in enumerate(self.column_names): + setattr(self, col, self.columns[ind]) + + def _cast_column(self, column_name, cast_type): + ''' + Cast all values of a column using a specified type. + + Args: + column_name: string. The column that will be casted. + cast_type: type. Cast type (do not encapsulate in quotes). + ''' + # get the column from its name + column_idx = self.column_names.index(column_name) + # for every column's value in each row, replace it with itself but casted as the specified type + for i in range(len(self.data)): + self.data[i][column_idx] = cast_type(self.data[i][column_idx]) + # change the type of the column + self.column_types[column_idx] = cast_type + # self._update() + + + def _insert(self, row, insert_stack=[]): + ''' + Insert row to table. + + Args: + row: list. A list of values to be inserted (will be casted to a predifined type automatically). + insert_stack: list. The insert stack (empty by default). + ''' + if len(row)!=len(self.column_names): + raise ValueError(f'ERROR -> Cannot insert {len(row)} values. Only {len(self.column_names)} columns exist') + + for i in range(len(row)): + # for each value, cast and replace it in row. + try: + row[i] = self.column_types[i](row[i]) + except ValueError: + if row[i] != 'NULL': + raise ValueError(f'ERROR -> Value {row[i]} of type {type(row[i])} is not of type {self.column_types[i]}.') + except TypeError as exc: + if row[i] != None: + print(exc) + + # if value is to be appended to the primary_key column, check that it doesnt alrady exist (no duplicate primary keys) + if i==self.pk_idx and row[i] in self.column_by_name(self.pk): + raise ValueError(f'## ERROR -> Value {row[i]} already exists in primary key column.') + elif i==self.pk_idx and row[i] is None: + raise ValueError(f'ERROR -> The value of the primary key cannot be None.') + + # if insert_stack is not empty, append to its last index + if insert_stack != []: + self.data[insert_stack[-1]] = row + else: # else append to the end + self.data.append(row) + # self._update() + + def _update_rows(self, set_value, set_column, condition): + ''' + Update where Condition is met. + + Args: + set_value: string. The provided set value. + set_column: string. The column to be altered. + condition: string. A condition using the following format: + 'column[<,<=,=,>=,>]value' or + 'value[<,<=,=,>=,>]column'. + + Operatores supported: (<,<=,=,>=,>) + ''' + # parse the condition + column_name, operator, value = self._parse_condition(condition) + + # get the condition and the set column + column = self.column_by_name(column_name) + set_column_idx = self.column_names.index(set_column) + + # set_columns_indx = [self.column_names.index(set_column_name) for set_column_name in set_column_names] + + # for each value in column, if condition, replace it with set_value + for row_ind, column_value in enumerate(column): + if get_op(operator, column_value, value): + self.data[row_ind][set_column_idx] = set_value + + # self._update() + # print(f"Updated {len(indexes_to_del)} rows") + + + def _delete_where(self, condition): + ''' + Deletes rows where condition is met. + + Important: delete replaces the rows to be deleted with rows filled with Nones. + These rows are then appended to the insert_stack. + + Args: + condition: string. A condition using the following format: + 'column[<,<=,==,>=,>]value' or + 'value[<,<=,==,>=,>]column'. + + Operatores supported: (<,<=,==,>=,>) + ''' + column_name, operator, value = self._parse_condition(condition) + + indexes_to_del = [] + + column = self.column_by_name(column_name) + for index, row_value in enumerate(column): + if get_op(operator, row_value, value): + indexes_to_del.append(index) + + # we pop from highest to lowest index in order to avoid removing the wrong item + # since we dont delete, we dont have to to pop in that order, but since delete is used + # to delete from meta tables too, we still implement it. + + for index in sorted(indexes_to_del, reverse=True): + if self._name[:4] != 'meta': + # if the table is not a metatable, replace the row with a row of nones + self.data[index] = [None for _ in range(len(self.column_names))] + else: + self.data.pop(index) + + # self._update() + # we have to return the deleted indexes, since they will be appended to the insert_stack + return indexes_to_del + + + def _select_where(self, return_columns, condition=None, distinct=False, order_by=None, desc=True, limit=None): + ''' + Select and return a table containing specified columns and rows where condition is met. + + Args: + return_columns: list. The columns to be returned. + condition: string. A condition using the following format: + 'column[<,<=,==,>=,>]value' or + 'value[<,<=,==,>=,>]column'. + + Operatores supported: (<,<=,==,>=,>) + distinct: boolean. If True, the resulting table will contain only unique rows (False by default). + order_by: string. A column name that signals that the resulting table should be ordered based on it (no order if None). + desc: boolean. If True, order_by will return results in descending order (False by default). + limit: int. An integer that defines the number of rows that will be returned (all rows if None). + ''' + + # if * return all columns, else find the column indexes for the columns specified + if return_columns == '*': + return_cols = [i for i in range(len(self.column_names))] + else: + return_cols = [self.column_names.index(col.strip()) for col in return_columns.split(',')] + + # if condition is None, return all rows + # if not, return the rows with values where condition is met for value + if condition is not None: + column_name, operator, value = self._parse_condition(condition) + column = self.column_by_name(column_name) + rows = [ind for ind, x in enumerate(column) if get_op(operator, x, value)] + else: + rows = [i for i in range(len(self.data))] + + # copy the old dict, but only the rows and columns of data with index in rows/columns (the indexes that we want returned) + dict = {(key):([[self.data[i][j] for j in return_cols] for i in rows] if key=="data" else value) for key,value in self.__dict__.items()} + + # we need to set the new column names/types and no of columns, since we might + # only return some columns + dict['column_names'] = [self.column_names[i] for i in return_cols] + dict['column_types'] = [self.column_types[i] for i in return_cols] + + s_table = Table(load=dict) + + s_table.data = list(set(map(lambda x: tuple(x), s_table.data))) if distinct else s_table.data + + if order_by: + s_table.order_by(order_by, desc) + + # if isinstance(limit, str): + # try: + # k = int(limit) + # except ValueError: + # raise Exception("The value following 'top' in the query should be a number.") + + # # Remove from the table's data all the None-filled rows, as they are not shown by default + # # Then, show the first k rows + # s_table.data.remove(len(s_table.column_names) * [None]) + # s_table.data = s_table.data[:k] + if isinstance(limit,str): + s_table.data = [row for row in s_table.data if any(row)][:int(limit)] + + return s_table + + + def _select_where_with_btree(self, return_columns, bt, condition, distinct=False, order_by=None, desc=True, limit=None): + + # if * return all columns, else find the column indexes for the columns specified + if return_columns == '*': + return_cols = [i for i in range(len(self.column_names))] + else: + return_cols = [self.column_names.index(colname) for colname in return_columns] + + + column_name, operator, value = self._parse_condition(condition) + + # if the column in condition is not a primary key, abort the select + if column_name != self.column_names[self.pk_idx]: + print('Column is not PK. Aborting') + + # here we run the same select twice, sequentially and using the btree. + # we then check the results match and compare performance (number of operation) + column = self.column_by_name(column_name) + + # sequential + rows1 = [] + opsseq = 0 + for ind, x in enumerate(column): + opsseq+=1 + if get_op(operator, x, value): + rows1.append(ind) + + # btree find + rows = bt.find(operator, value) + + try: + k = int(limit) + except TypeError: + k = None + # same as simple select from now on + rows = rows[:k] + # TODO: this needs to be dumbed down + dict = {(key):([[self.data[i][j] for j in return_cols] for i in rows] if key=="data" else value) for key,value in self.__dict__.items()} + + dict['column_names'] = [self.column_names[i] for i in return_cols] + dict['column_types'] = [self.column_types[i] for i in return_cols] + + s_table = Table(load=dict) + + s_table.data = list(set(map(lambda x: tuple(x), s_table.data))) if distinct else s_table.data + + if order_by: + s_table.order_by(order_by, desc) + + if isinstance(limit,str): + s_table.data = [row for row in s_table.data if row is not None][:int(limit)] + + return s_table + + def order_by(self, column_name, desc=True): + ''' + Order table based on column. + + Args: + column_name: string. Name of column. + desc: boolean. If True, order_by will return results in descending order (False by default). + ''' + column = [val if val is not None else 0 for val in self.column_by_name(column_name)] + idx = sorted(range(len(column)), key=lambda k: column[k], reverse=desc) + # print(idx) + self.data = [self.data[i] for i in idx] + # self._update() + + + def _general_join_processing(self, table_right:Table, condition, join_type): + ''' + Performs the processes all the join operations need (regardless of type) so that there is no code repetition. + + Args: + condition: string. A condition using the following format: + 'column[<,<=,==,>=,>]value' or + 'value[<,<=,==,>=,>]column'. + + Operators supported: (<,<=,==,>=,>) + ''' + # get columns and operator + column_name_left, operator, column_name_right = self._parse_condition(condition, join=True) + # try to find both columns, if you fail raise error + + if(operator != '=' and join_type in ['left','right','full']): + class CustomFailException(Exception): + pass + raise CustomFailException('Outer Joins can only be used if the condition operator is "=".\n') + + try: + column_index_left = self.column_names.index(column_name_left) + except: + raise Exception(f'Column "{column_name_left}" dont exist in left table. Valid columns: {self.column_names}.') + + try: + column_index_right = table_right.column_names.index(column_name_right) + except: + raise Exception(f'Column "{column_name_right}" dont exist in right table. Valid columns: {table_right.column_names}.') + + # get the column names of both tables with the table name in front + # ex. for left -> name becomes left_table_name_name etc + left_names = [f'{self._name}.{colname}' if self._name!='' else colname for colname in self.column_names] + right_names = [f'{table_right._name}.{colname}' if table_right._name!='' else colname for colname in table_right.column_names] + + # define the new tables name, its column names and types + join_table_name = '' + join_table_colnames = left_names+right_names + join_table_coltypes = self.column_types+table_right.column_types + join_table = Table(name=join_table_name, column_names=join_table_colnames, column_types= join_table_coltypes) + + return join_table, column_index_left, column_index_right, operator + + + def _inner_join(self, table_right: Table, condition): + ''' + Join table (left) with a supplied table (right) where condition is met. + + Args: + condition: string. A condition using the following format: + 'column[<,<=,==,>=,>]value' or + 'value[<,<=,==,>=,>]column'. + + Operators supported: (<,<=,==,>=,>) + ''' + join_table, column_index_left, column_index_right, operator = self._general_join_processing(table_right, condition, 'inner') + + # count the number of operations (<,> etc) + no_of_ops = 0 + # this code is dumb on purpose... it needs to illustrate the underline technique + # for each value in left column and right column, if condition, append the corresponding row to the new table + for row_left in self.data: + left_value = row_left[column_index_left] + for row_right in table_right.data: + right_value = row_right[column_index_right] + if(left_value is None and right_value is None): + continue + no_of_ops+=1 + if get_op(operator, left_value, right_value): #EQ_OP + join_table._insert(row_left+row_right) + + return join_table + + def _left_join(self, table_right: Table, condition): + ''' + Perform a left join on the table with the supplied table (right). + + Args: + condition: string. A condition using the following format: + 'column[<,<=,==,>=,>]value' or + 'value[<,<=,==,>=,>]column'. + + Operators supported: (<,<=,==,>=,>) + ''' + join_table, column_index_left, column_index_right, operator = self._general_join_processing(table_right, condition, 'left') + + right_column = table_right.column_by_name(table_right.column_names[column_index_right]) + right_table_row_length = len(table_right.column_names) + + for row_left in self.data: + left_value = row_left[column_index_left] + if left_value is None: + continue + elif left_value not in right_column: + join_table._insert(row_left + right_table_row_length*["NULL"]) + else: + for row_right in table_right.data: + right_value = row_right[column_index_right] + if left_value == right_value: + join_table._insert(row_left + row_right) + + return join_table + + def _right_join(self, table_right: Table, condition): + ''' + Perform a right join on the table with the supplied table (right). + + Args: + condition: string. A condition using the following format: + 'column[<,<=,==,>=,>]value' or + 'value[<,<=,==,>=,>]column'. + + Operators supported: (<,<=,==,>=,>) + ''' + join_table, column_index_left, column_index_right, operator = self._general_join_processing(table_right, condition, 'right') + + left_column = self.column_by_name(self.column_names[column_index_left]) + left_table_row_length = len(self.column_names) + + for row_right in table_right.data: + right_value = row_right[column_index_right] + if right_value is None: + continue + elif right_value not in left_column: + join_table._insert(left_table_row_length*["NULL"] + row_right) + else: + for row_left in self.data: + left_value = row_left[column_index_left] + if left_value == right_value: + join_table._insert(row_left + row_right) + + return join_table + + def _full_join(self, table_right: Table, condition): + ''' + Perform a full join on the table with the supplied table (right). + + Args: + condition: string. A condition using the following format: + 'column[<,<=,==,>=,>]value' or + 'value[<,<=,==,>=,>]column'. + + Operators supported: (<,<=,==,>=,>) + ''' + join_table, column_index_left, column_index_right, operator = self._general_join_processing(table_right, condition, 'full') + + right_column = table_right.column_by_name(table_right.column_names[column_index_right]) + left_column = self.column_by_name(self.column_names[column_index_left]) + + right_table_row_length = len(table_right.column_names) + left_table_row_length = len(self.column_names) + + for row_left in self.data: + left_value = row_left[column_index_left] + if left_value is None: + continue + if left_value not in right_column: + join_table._insert(row_left + right_table_row_length*["NULL"]) + else: + for row_right in table_right.data: + right_value = row_right[column_index_right] + if left_value == right_value: + join_table._insert(row_left + row_right) + + for row_right in table_right.data: + right_value = row_right[column_index_right] + + if right_value is None: + continue + elif right_value not in left_column: + join_table._insert(left_table_row_length*["NULL"] + row_right) + + return join_table + + def show(self, no_of_rows=None, is_locked=False): + ''' + Print the table in a nice readable format. + + Args: + no_of_rows: int. Number of rows. + is_locked: boolean. Whether it is locked (False by default). + ''' + output = "" + # if the table is locked, add locked keyword to title + if is_locked: + output += f"\n## {self._name} (locked) ##\n" + else: + output += f"\n## {self._name} ##\n" + + # headers -> "column name (column type)" + headers = [f'{col} ({tp.__name__})' for col, tp in zip(self.column_names, self.column_types)] + if self.pk_idx is not None: + # table has a primary key, add PK next to the appropriate column + headers[self.pk_idx] = headers[self.pk_idx]+' #PK#' + # detect the rows that are no tfull of nones (these rows have been deleted) + # if we dont skip these rows, the returning table has empty rows at the deleted positions + non_none_rows = [row for row in self.data if any(row)] + # print using tabulate + print(tabulate(non_none_rows[:no_of_rows], headers=headers)+'\n') + + + def _parse_condition(self, condition, join=False): + ''' + Parse the single string condition and return the value of the column and the operator. + + Args: + condition: string. A condition using the following format: + 'column[<,<=,==,>=,>]value' or + 'value[<,<=,==,>=,>]column'. + + Operatores supported: (<,<=,==,>=,>) + join: boolean. Whether to join or not (False by default). + ''' + # if both_columns (used by the join function) return the names of the names of the columns (left first) + if join: + return split_condition(condition) + + # cast the value with the specified column's type and return the column name, the operator and the casted value + left, op, right = split_condition(condition) + if left not in self.column_names: + raise ValueError(f'Condition is not valid (cant find column name)') + coltype = self.column_types[self.column_names.index(left)] + + return left, op, coltype(right) + + + def _load_from_file(self, filename): + ''' + Load table from a pkl file (not used currently). + + Args: + filename: string. Name of pkl file. + ''' + f = open(filename, 'rb') + tmp_dict = pickle.load(f) + f.close() + + self.__dict__.update(tmp_dict.__dict__) From ad8a85a8660417b6fe85cac32d73d7926ce7a657 Mon Sep 17 00:00:00 2001 From: apostolossvls Date: Mon, 20 Feb 2023 18:08:40 +0200 Subject: [PATCH 2/2] "Not" function & Cleanup Added "not" function Removed old backup files. --- miniDB/misc.py | 14 + miniDBold/__init__.py | 0 miniDBold/btree.py | 348 ------------------- miniDBold/dashboard.py | 15 - miniDBold/database.py | 748 ----------------------------------------- miniDBold/joins.py | 309 ----------------- miniDBold/misc.py | 65 ---- miniDBold/table.py | 579 ------------------------------- 8 files changed, 14 insertions(+), 2064 deletions(-) delete mode 100644 miniDBold/__init__.py delete mode 100644 miniDBold/btree.py delete mode 100644 miniDBold/dashboard.py delete mode 100644 miniDBold/database.py delete mode 100644 miniDBold/joins.py delete mode 100644 miniDBold/misc.py delete mode 100644 miniDBold/table.py diff --git a/miniDB/misc.py b/miniDB/misc.py index 2975b265..7bddc8b8 100644 --- a/miniDB/misc.py +++ b/miniDB/misc.py @@ -49,3 +49,17 @@ def reverse_op(op): '=' : '=' }.get(op) +def not_op(op): + ''' + Return opposite of the operator given + ''' + return { + '>' : '<=', + '>=' : '<', + '<' : '>=', + '<=' : '>', + '=' : '!=', + '!=' : '=' + }.get(op) + + diff --git a/miniDBold/__init__.py b/miniDBold/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/miniDBold/btree.py b/miniDBold/btree.py deleted file mode 100644 index 37e8223b..00000000 --- a/miniDBold/btree.py +++ /dev/null @@ -1,348 +0,0 @@ -''' -https://en.wikipedia.org/wiki/B%2B_tree -''' - -class Node: - ''' - Node abstraction. Represents a single bucket - ''' - def __init__(self, b, values=None, ptrs=None,left_sibling=None, right_sibling=None, parent=None, is_leaf=False): - self.b = b # branching factor - self.values = [] if values is None else values # Values (the data from the pk column) - self.ptrs = [] if ptrs is None else ptrs # ptrs (the indexes of each datapoint or the index of another bucket) - self.left_sibling = left_sibling # the index of a buckets left sibling - self.right_sibling = right_sibling # the index of a buckets right sibling - self.parent = parent # the index of a buckets parent - self.is_leaf = is_leaf # a boolean value signaling whether the node is a leaf or not - - - def find(self, value, return_ops=False): - ''' - Returns the index of the next node to search for a value if the node is not a leaf (a ptrs of the available ones). - If it is a leaf (we have found the appropriate node), nothing is returned. - - Args: - value: float. The value being searched for. - return_ops: boolean. Set to True if you want to use the number of operations (for benchmarking). - ''' - ops = 0 # number of operations (<>= etc). Used for benchmarking - if self.is_leaf: # - return - - # for each value in the node, if the user supplied value is smaller, return the btrees value index - # else (no value in the node is larger) return the last ptr - for index, existing_val in enumerate(self.values): - ops+=1 - if value is None or existing_val is None: - continue - if value= etc). Used for benchmarking - - #start with the root node - node = self.nodes[self.root] - # while the node that we are searching in is not a leaf - # keep searching - while not node.is_leaf: - idx, ops1 = node.find(value, return_ops=True) - node = self.nodes[idx] - ops += ops1 - - # finally return the index of the appropriate node (and the ops if you want to) - if return_ops: - return self.nodes.index(node), ops - else: - return self.nodes.index(node) - - - def split(self, node_id): - ''' - Split the node with index=node_id. - - Args: - node_id: float. The corresponding ID of the node. - ''' - # fetch the node to be split - node = self.nodes[node_id] - # the value that will be propagated to the parent is the middle one. - new_parent_value = node.values[len(node.values)//2] - if node.is_leaf: - # if the node is a leaf, the parent value should be a part of the new node (right) - # Important: in a b+tree, every value should appear in a leaf - right_values = node.values[len(node.values)//2:] - right_ptrs = node.ptrs[len(node.ptrs)//2:] - - # create the new node with the right half of the old nodes values and ptrs (including the middle ones) - right = Node(self.b, right_values, right_ptrs,\ - left_sibling=node_id, right_sibling=node.right_sibling, parent=node.parent, is_leaf=node.is_leaf) - # since the new node (right) will be the next one to be appended to the nodes list - # its index will be equal to the length of the nodes list. - # Thus we set the old nodes (now left) right sibling to the right nodes future index (len of nodes) - if node.right_sibling is not None: - self.nodes[node.right_sibling].left_sibling = len(self.nodes) - node.right_sibling = len(self.nodes) - - - else: - # if the node is not a leaf, the parent value shoudl NOT be part of the new node - right_values = node.values[len(node.values)//2+1:] - if self.b%2==1: - right_ptrs = node.ptrs[len(node.ptrs)//2:] - else: - right_ptrs = node.ptrs[len(node.ptrs)//2+1:] - - # if nonleafs should be connected change the following two lines and add siblings - right = Node(self.b, right_values, right_ptrs,\ - parent=node.parent, is_leaf=node.is_leaf) - # make sure that a non leaf node doesnt have a parent - node.right_sibling = None - # the right node's kids should have him as a parent (if not all nodes will have left as parent) - for ptr in right_ptrs: - self.nodes[ptr].parent = len(self.nodes) - - # old node (left) keeps only the first half of the values/ptrs - node.values = node.values[:len(node.values)//2] - if self.b%2==1: - node.ptrs = node.ptrs[:len(node.ptrs)//2] - else: - node.ptrs = node.ptrs[:len(node.ptrs)//2+1] - - # append the new node (right) to the nodes list - self.nodes.append(right) - - # If the new nodes have no parents (a new level needs to be added - if node.parent is None: - # its the root that is split - # new root contains the parent value and ptrs to the two recently split nodes - parent = Node(self.b, [new_parent_value], [node_id, len(self.nodes)-1]\ - ,parent=node.parent, is_leaf=False) - - # set root, and parent of split celss to the index of the new root node (len of nodes-1) - self.nodes.append(parent) - self.root = len(self.nodes)-1 - node.parent = len(self.nodes)-1 - right.parent = len(self.nodes)-1 - else: - # insert the parent value to the parent - - self.nodes[node.parent].insert(new_parent_value, len(self.nodes)-1) - # check whether the parent needs to be split - if len(self.nodes[node.parent].values)==self.b: - self.split(node.parent) - - - - - def show(self): - ''' - Show important info for each node (sort by level - root first, then left to right). - ''' - nds = [] - nds.append(self.root) - for ptr in nds: - if self.nodes[ptr].is_leaf: - continue - nds.extend(self.nodes[ptr].ptrs) - - for ptr in nds: - print(f'## {ptr} ##') - self.nodes[ptr].show() - print('----') - - - def plot(self): - ## arrange the nodes top to bottom left to right - nds = [] - nds.append(self.root) - for ptr in nds: - if self.nodes[ptr].is_leaf: - continue - nds.extend(self.nodes[ptr].ptrs) - - # add each node and each link - g = 'digraph G{\nforcelabels=true;\n' - - for i in nds: - node = self.nodes[i] - g+=f'{i} [label="{node.values}"]\n' - if node.is_leaf: - continue - # if node.left_sibling is not None: - # g+=f'"{node.values}"->"{self.nodes[node.left_sibling].values}" [color="blue" constraint=false];\n' - # if node.right_sibling is not None: - # g+=f'"{node.values}"->"{self.nodes[node.right_sibling].values}" [color="green" constraint=false];\n' - # - # g+=f'"{node.values}"->"{self.nodes[node.parent].values}" [color="red" constraint=false];\n' - else: - for child in node.ptrs: - g+=f'{child} [label="{self.nodes[child].values}"]\n' - g+=f'{i}->{child};\n' - g +="}" - - try: - from graphviz import Source - src = Source(g) - src.render('bplustree', view=True) - except ImportError: - print('"graphviz" package not found. Writing to graph.gv.') - with open('graph.gv','w') as f: - f.write(g) - - def find(self, operator, value): - ''' - Return ptrs of elements where btree_value"operator"value. - Important, the user supplied "value" is the right value of the operation. That is why the operation are reversed below. - The left value of the op is the btree value. - - Args: - operator: string. The provided evaluation operator. - value: float. The value being searched for. - ''' - results = [] - # find the index of the node that the element should exist in - leaf_idx, ops = self._search(value, True) - target_node = self.nodes[leaf_idx] - - if operator == '=': - # if the element exist, append to list, else pass and return - try: - results.append(target_node.ptrs[target_node.values.index(value)]) - # print('Found') - except: - # print('Not found') - pass - - # for all other ops, the code is the same, only the operations themselves and the sibling indexes change - # for > and >= (btree value is >/>= of user supplied value), we return all the right siblings (all values are larger than current cell) - # for < and <= (btree value is ': - for idx, node_value in enumerate(target_node.values): - ops+=1 - if node_value > value: - results.append(target_node.ptrs[idx]) - while target_node.right_sibling is not None: - target_node = self.nodes[target_node.right_sibling] - results.extend(target_node.ptrs) - - - if operator == '>=': - for idx, node_value in enumerate(target_node.values): - ops+=1 - if node_value >= value: - results.append(target_node.ptrs[idx]) - while target_node.right_sibling is not None: - target_node = self.nodes[target_node.right_sibling] - results.extend(target_node.ptrs) - - if operator == '<': - for idx, node_value in enumerate(target_node.values): - ops+=1 - if node_value < value: - results.append(target_node.ptrs[idx]) - while target_node.left_sibling is not None: - target_node = self.nodes[target_node.left_sibling] - results.extend(target_node.ptrs) - - if operator == '<=': - for idx, node_value in enumerate(target_node.values): - ops+=1 - if node_value <= value: - results.append(target_node.ptrs[idx]) - while target_node.left_sibling is not None: - target_node = self.nodes[target_node.left_sibling] - results.extend(target_node.ptrs) - - # print the number of operations (usefull for benchamrking) - # print(f'With BTree -> {ops} comparison operations') - return results diff --git a/miniDBold/dashboard.py b/miniDBold/dashboard.py deleted file mode 100644 index ee97b0a8..00000000 --- a/miniDBold/dashboard.py +++ /dev/null @@ -1,15 +0,0 @@ -import sys -import os - -sys.path.append(f'{os.path.dirname(os.path.dirname(os.path.abspath(__file__)))}/miniDB') - -from database import Database - - -db = Database(sys.argv[1]) - - -for name in list(db.tables): - if sys.argv[2]=='meta' and name[:4]!='meta': - continue - db.show_table(name) diff --git a/miniDBold/database.py b/miniDBold/database.py deleted file mode 100644 index a3ac6be7..00000000 --- a/miniDBold/database.py +++ /dev/null @@ -1,748 +0,0 @@ -from __future__ import annotations -import pickle -from time import sleep, localtime, strftime -import os,sys -import logging -import warnings -import readline -from tabulate import tabulate - -sys.path.append(f'{os.path.dirname(os.path.dirname(os.path.abspath(__file__)))}/miniDB') -from miniDB import table -sys.modules['table'] = table - -from joins import Inlj, Smj -from btree import Btree -from misc import split_condition -from table import Table - - -# readline.clear_history() - -class Database: - ''' - Main Database class, containing tables. - ''' - - def __init__(self, name, load=True, verbose = True): - self.tables = {} - self._name = name - self.verbose = verbose - - self.savedir = f'dbdata/{name}_db' - - if load: - try: - self.load_database() - logging.info(f'Loaded "{name}".') - return - except: - if verbose: - warnings.warn(f'Database "{name}" does not exist. Creating new.') - - # create dbdata directory if it doesnt exist - if not os.path.exists('dbdata'): - os.mkdir('dbdata') - - # create new dbs save directory - try: - os.mkdir(self.savedir) - except: - pass - - # create all the meta tables - self.create_table('meta_length', 'table_name,no_of_rows', 'str,int') - self.create_table('meta_locks', 'table_name,pid,mode', 'str,int,str') - self.create_table('meta_insert_stack', 'table_name,indexes', 'str,list') - self.create_table('meta_indexes', 'table_name,index_name', 'str,str') - self.save_database() - - def save_database(self): - ''' - Save database as a pkl file. This method saves the database object, including all tables and attributes. - ''' - for name, table in self.tables.items(): - with open(f'{self.savedir}/{name}.pkl', 'wb') as f: - pickle.dump(table, f) - - def _save_locks(self): - ''' - Stores the meta_locks table to file as meta_locks.pkl. - ''' - with open(f'{self.savedir}/meta_locks.pkl', 'wb') as f: - pickle.dump(self.tables['meta_locks'], f) - - def load_database(self): - ''' - Load all tables that are part of the database (indices noted here are loaded). - - Args: - path: string. Directory (path) of the database on the system. - ''' - path = f'dbdata/{self._name}_db' - for file in os.listdir(path): - - if file[-3:]!='pkl': # if used to load only pkl files - continue - f = open(path+'/'+file, 'rb') - tmp_dict = pickle.load(f) - f.close() - name = f'{file.split(".")[0]}' - self.tables.update({name: tmp_dict}) - # setattr(self, name, self.tables[name]) - - #### IO #### - - def _update(self): - ''' - Update all meta tables. - ''' - self._update_meta_length() - self._update_meta_insert_stack() - - - def create_table(self, name, column_names, column_types, primary_key=None, load=None): - ''' - This method create a new table. This table is saved and can be accessed via db_object.tables['table_name'] or db_object.table_name - - Args: - name: string. Name of table. - column_names: list. Names of columns. - column_types: list. Types of columns. - primary_key: string. The primary key (if it exists). - load: boolean. Defines table object parameters as the name of the table and the column names. - ''' - # print('here -> ', column_names.split(',')) - self.tables.update({name: Table(name=name, column_names=column_names.split(','), column_types=column_types.split(','), primary_key=primary_key, load=load)}) - # self._name = Table(name=name, column_names=column_names, column_types=column_types, load=load) - # check that new dynamic var doesnt exist already - # self.no_of_tables += 1 - self._update() - self.save_database() - # (self.tables[name]) - if self.verbose: - print(f'Created table "{name}".') - - - def drop_table(self, table_name): - ''' - Drop table from current database. - - Args: - table_name: string. Name of table. - ''' - self.load_database() - self.lock_table(table_name) - - self.tables.pop(table_name) - if os.path.isfile(f'{self.savedir}/{table_name}.pkl'): - os.remove(f'{self.savedir}/{table_name}.pkl') - else: - warnings.warn(f'"{self.savedir}/{table_name}.pkl" not found.') - self.delete_from('meta_locks', f'table_name={table_name}') - self.delete_from('meta_length', f'table_name={table_name}') - self.delete_from('meta_insert_stack', f'table_name={table_name}') - - if self._has_index(table_name): - to_be_deleted = [] - for key, table in enumerate(self.tables['meta_indexes'].column_by_name('table_name')): - if table == table_name: - to_be_deleted.append(key) - - for i in reversed(to_be_deleted): - self.drop_index(self.tables['meta_indexes'].data[i][1]) - - try: - delattr(self, table_name) - except AttributeError: - pass - # self._update() - self.save_database() - - - def import_table(self, table_name, filename, column_types=None, primary_key=None): - ''' - Creates table from CSV file. - - Args: - filename: string. CSV filename. If not specified, filename's name will be used. - column_types: list. Types of columns. If not specified, all will be set to type str. - primary_key: string. The primary key (if it exists). - ''' - file = open(filename, 'r') - - first_line=True - for line in file.readlines(): - if first_line: - colnames = line.strip('\n') - if column_types is None: - column_types = ",".join(['str' for _ in colnames.split(',')]) - self.create_table(name=table_name, column_names=colnames, column_types=column_types, primary_key=primary_key) - lock_ownership = self.lock_table(table_name, mode='x') - first_line = False - continue - self.tables[table_name]._insert(line.strip('\n').split(',')) - - if lock_ownership: - self.unlock_table(table_name) - self._update() - self.save_database() - - - def export(self, table_name, filename=None): - ''' - Transform table to CSV. - - Args: - table_name: string. Name of table. - filename: string. Output CSV filename. - ''' - res = '' - for row in [self.tables[table_name].column_names]+self.tables[table_name].data: - res+=str(row)[1:-1].replace('\'', '').replace('"','').replace(' ','')+'\n' - - if filename is None: - filename = f'{table_name}.csv' - - with open(filename, 'w') as file: - file.write(res) - - def table_from_object(self, new_table): - ''' - Add table object to database. - - Args: - new_table: string. Name of new table. - ''' - - self.tables.update({new_table._name: new_table}) - if new_table._name not in self.__dir__(): - setattr(self, new_table._name, new_table) - else: - raise Exception(f'"{new_table._name}" attribute already exists in class "{self.__class__.__name__}".') - self._update() - self.save_database() - - - - ##### table functions ##### - - # In every table function a load command is executed to fetch the most recent table. - # In every table function, we first check whether the table is locked. Since we have implemented - # only the X lock, if the tables is locked we always abort. - # After every table function, we update and save. Update updates all the meta tables and save saves all - # tables. - - # these function calls are named close to the ones in postgres - - def cast(self, column_name, table_name, cast_type): - ''' - Modify the type of the specified column and cast all prexisting values. - (Executes type() for every value in column and saves) - - Args: - table_name: string. Name of table (must be part of database). - column_name: string. The column that will be casted (must be part of database). - cast_type: type. Cast type (do not encapsulate in quotes). - ''' - self.load_database() - - lock_ownership = self.lock_table(table_name, mode='x') - self.tables[table_name]._cast_column(column_name, eval(cast_type)) - if lock_ownership: - self.unlock_table(table_name) - self._update() - self.save_database() - - def insert_into(self, table_name, row_str): - ''' - Inserts data to given table. - - Args: - table_name: string. Name of table (must be part of database). - row: list. A list of values to be inserted (will be casted to a predifined type automatically). - lock_load_save: boolean. If False, user needs to load, lock and save the states of the database (CAUTION). Useful for bulk-loading. - ''' - row = row_str.strip().split(',') - self.load_database() - # fetch the insert_stack. For more info on the insert_stack - # check the insert_stack meta table - lock_ownership = self.lock_table(table_name, mode='x') - insert_stack = self._get_insert_stack_for_table(table_name) - try: - self.tables[table_name]._insert(row, insert_stack) - except Exception as e: - logging.info(e) - logging.info('ABORTED') - self._update_meta_insert_stack_for_tb(table_name, insert_stack[:-1]) - - if lock_ownership: - self.unlock_table(table_name) - self._update() - self.save_database() - - - def update_table(self, table_name, set_args, condition): - ''' - Update the value of a column where a condition is met. - - Args: - table_name: string. Name of table (must be part of database). - set_value: string. New value of the predifined column name. - set_column: string. The column to be altered. - condition: string. A condition using the following format: - 'column[<,<=,==,>=,>]value' or - 'value[<,<=,==,>=,>]column'. - - Operatores supported: (<,<=,==,>=,>) - ''' - set_column, set_value = set_args.replace(' ','').split('=') - self.load_database() - - lock_ownership = self.lock_table(table_name, mode='x') - self.tables[table_name]._update_rows(set_value, set_column, condition) - if lock_ownership: - self.unlock_table(table_name) - self._update() - self.save_database() - - def delete_from(self, table_name, condition): - ''' - Delete rows of table where condition is met. - - Args: - table_name: string. Name of table (must be part of database). - condition: string. A condition using the following format: - 'column[<,<=,==,>=,>]value' or - 'value[<,<=,==,>=,>]column'. - - Operatores supported: (<,<=,==,>=,>) - ''' - self.load_database() - - lock_ownership = self.lock_table(table_name, mode='x') - deleted = self.tables[table_name]._delete_where(condition) - if lock_ownership: - self.unlock_table(table_name) - self._update() - self.save_database() - # we need the save above to avoid loading the old database that still contains the deleted elements - if table_name[:4]!='meta': - self._add_to_insert_stack(table_name, deleted) - self.save_database() - - def select(self, columns, table_name, condition, distinct=None, order_by=None, \ - limit=True, desc=None, save_as=None, return_object=True): - ''' - Selects and outputs a table's data where condtion is met. - - Args: - table_name: string. Name of table (must be part of database). - columns: list. The columns that will be part of the output table (use '*' to select all available columns) - condition: string. A condition using the following format: - 'column[<,<=,==,>=,>]value' or - 'value[<,<=,==,>=,>]column'. - - Operatores supported: (<,<=,==,>=,>) - order_by: string. A column name that signals that the resulting table should be ordered based on it (no order if None). - desc: boolean. If True, order_by will return results in descending order (True by default). - limit: int. An integer that defines the number of rows that will be returned (all rows if None). - save_as: string. The name that will be used to save the resulting table into the database (no save if None). - return_object: boolean. If True, the result will be a table object (useful for internal use - the result will be printed by default). - distinct: boolean. If True, the resulting table will contain only unique rows. - ''' - - # print(table_name) - self.load_database() - if isinstance(table_name,Table): - return table_name._select_where(columns, condition, distinct, order_by, desc, limit) - - if condition is not None: - condition_column = split_condition(condition)[0] - else: - condition_column = '' - - - # self.lock_table(table_name, mode='x') - if self.is_locked(table_name): - return - if self._has_index(table_name) and condition_column==self.tables[table_name].column_names[self.tables[table_name].pk_idx]: - index_name = self.select('*', 'meta_indexes', f'table_name={table_name}', return_object=True).column_by_name('index_name')[0] - bt = self._load_idx(index_name) - table = self.tables[table_name]._select_where_with_btree(columns, bt, condition, distinct, order_by, desc, limit) - else: - table = self.tables[table_name]._select_where(columns, condition, distinct, order_by, desc, limit) - # self.unlock_table(table_name) - if save_as is not None: - table._name = save_as - self.table_from_object(table) - else: - if return_object: - return table - else: - return table.show() - - - def show_table(self, table_name, no_of_rows=None): - ''' - Print table in a readable tabular design (using tabulate). - - Args: - table_name: string. Name of table (must be part of database). - ''' - self.load_database() - - self.tables[table_name].show(no_of_rows, self.is_locked(table_name)) - - - def sort(self, table_name, column_name, asc=False): - ''' - Sorts a table based on a column. - - Args: - table_name: string. Name of table (must be part of database). - column_name: string. the column name that will be used to sort. - asc: If True sort will return results in ascending order (False by default). - ''' - - self.load_database() - - lock_ownership = self.lock_table(table_name, mode='x') - self.tables[table_name]._sort(column_name, asc=asc) - if lock_ownership: - self.unlock_table(table_name) - self._update() - self.save_database() - - def create_view(self, table_name, table): - ''' - Create a virtual table based on the result-set of the SQL statement provided. - - Args: - table_name: string. Name of the table that will be saved. - table: table. The table that will be saved. - ''' - table._name = table_name - self.table_from_object(table) - - def join(self, mode, left_table, right_table, condition, save_as=None, return_object=True): - ''' - Join two tables that are part of the database where condition is met. - - Args: - left_table: string. Name of the left table (must be in DB) or Table obj. - right_table: string. Name of the right table (must be in DB) or Table obj. - condition: string. A condition using the following format: - 'column[<,<=,==,>=,>]value' or - 'value[<,<=,==,>=,>]column'. - - Operators supported: (<,<=,==,>=,>) - save_as: string. The output filename that will be used to save the resulting table in the database (won't save if None). - return_object: boolean. If True, the result will be a table object (useful for internal usage - the result will be printed by default). - ''' - self.load_database() - if self.is_locked(left_table) or self.is_locked(right_table): - return - - left_table = left_table if isinstance(left_table, Table) else self.tables[left_table] - right_table = right_table if isinstance(right_table, Table) else self.tables[right_table] - - - if mode=='inner': - res = left_table._inner_join(right_table, condition) - - elif mode=='left': - res = left_table._left_join(right_table, condition) - - elif mode=='right': - res = left_table._right_join(right_table, condition) - - elif mode=='full': - res = left_table._full_join(right_table, condition) - - elif mode=='inl': - # Check if there is an index of either of the two tables available, as if there isn't we can't use inlj - leftIndexExists = self._has_index(left_table._name) - rightIndexExists = self._has_index(right_table._name) - - if not leftIndexExists and not rightIndexExists: - res = None - raise Exception('Index-nested-loop join cannot be executed. Use inner join instead.\n') - elif rightIndexExists: - index_name = self.select('*', 'meta_indexes', f'table_name={right_table._name}', return_object=True).column_by_name('index_name')[0] - res = Inlj(condition, left_table, right_table, self._load_idx(index_name), 'right').join() - elif leftIndexExists: - index_name = self.select('*', 'meta_indexes', f'table_name={left_table._name}', return_object=True).column_by_name('index_name')[0] - res = Inlj(condition, left_table, right_table, self._load_idx(index_name), 'left').join() - - elif mode=='sm': - res = Smj(condition, left_table, right_table).join() - - else: - raise NotImplementedError - - if save_as is not None: - res._name = save_as - self.table_from_object(res) - else: - if return_object: - return res - else: - res.show() - - if return_object: - return res - else: - res.show() - - def lock_table(self, table_name, mode='x'): - ''' - Locks the specified table using the exclusive lock (X). - - Args: - table_name: string. Table name (must be part of database). - ''' - if table_name[:4]=='meta' or table_name not in self.tables.keys() or isinstance(table_name,Table): - return - - with open(f'{self.savedir}/meta_locks.pkl', 'rb') as f: - self.tables.update({'meta_locks': pickle.load(f)}) - - try: - pid = self.tables['meta_locks']._select_where('pid',f'table_name={table_name}').data[0][0] - if pid!=os.getpid(): - raise Exception(f'Table "{table_name}" is locked by process with pid={pid}') - else: - return False - - except IndexError: - pass - - if mode=='x': - self.tables['meta_locks']._insert([table_name, os.getpid(), mode]) - else: - raise NotImplementedError - self._save_locks() - return True - # print(f'Locking table "{table_name}"') - - def unlock_table(self, table_name, force=False): - ''' - Unlocks the specified table that is exclusively locked (X). - - Args: - table_name: string. Table name (must be part of database). - ''' - if table_name not in self.tables.keys(): - raise Exception(f'Table "{table_name}" is not in database') - - if not force: - try: - # pid = self.select('*','meta_locks', f'table_name={table_name}', return_object=True).data[0][1] - pid = self.tables['meta_locks']._select_where('pid',f'table_name={table_name}').data[0][0] - if pid!=os.getpid(): - raise Exception(f'Table "{table_name}" is locked by the process with pid={pid}') - except IndexError: - pass - self.tables['meta_locks']._delete_where(f'table_name={table_name}') - self._save_locks() - # print(f'Unlocking table "{table_name}"') - - def is_locked(self, table_name): - ''' - Check whether the specified table is exclusively locked (X). - - Args: - table_name: string. Table name (must be part of database). - ''' - if isinstance(table_name,Table) or table_name[:4]=='meta': # meta tables will never be locked (they are internal) - return False - - with open(f'{self.savedir}/meta_locks.pkl', 'rb') as f: - self.tables.update({'meta_locks': pickle.load(f)}) - - try: - pid = self.tables['meta_locks']._select_where('pid',f'table_name={table_name}').data[0][0] - if pid!=os.getpid(): - raise Exception(f'Table "{table_name}" is locked by the process with pid={pid}') - - except IndexError: - pass - return False - - - #### META #### - - # The following functions are used to update, alter, load and save the meta tables. - # Important: Meta tables contain info regarding the NON meta tables ONLY. - # i.e. meta_length will not show the number of rows in meta_locks etc. - - def _update_meta_length(self): - ''' - Updates the meta_length table. - ''' - for table in self.tables.values(): - if table._name[:4]=='meta': #skip meta tables - continue - if table._name not in self.tables['meta_length'].column_by_name('table_name'): # if new table, add record with 0 no. of rows - self.tables['meta_length']._insert([table._name, 0]) - - # the result needs to represent the rows that contain data. Since we use an insert_stack - # some rows are filled with Nones. We skip these rows. - non_none_rows = len([row for row in table.data if any(row)]) - self.tables['meta_length']._update_rows(non_none_rows, 'no_of_rows', f'table_name={table._name}') - # self.update_row('meta_length', len(table.data), 'no_of_rows', 'table_name', '==', table._name) - - def _update_meta_locks(self): - ''' - Updates the meta_locks table. - ''' - for table in self.tables.values(): - if table._name[:4]=='meta': #skip meta tables - continue - if table._name not in self.tables['meta_locks'].column_by_name('table_name'): - - self.tables['meta_locks']._insert([table._name, False]) - # self.insert('meta_locks', [table._name, False]) - - def _update_meta_insert_stack(self): - ''' - Updates the meta_insert_stack table. - ''' - for table in self.tables.values(): - if table._name[:4]=='meta': #skip meta tables - continue - if table._name not in self.tables['meta_insert_stack'].column_by_name('table_name'): - self.tables['meta_insert_stack']._insert([table._name, []]) - - - def _add_to_insert_stack(self, table_name, indexes): - ''' - Adds provided indices to the insert stack of the specified table. - - Args: - table_name: string. Table name (must be part of database). - indexes: list. The list of indices that will be added to the insert stack (the indices of the newly deleted elements). - ''' - old_lst = self._get_insert_stack_for_table(table_name) - self._update_meta_insert_stack_for_tb(table_name, old_lst+indexes) - - def _get_insert_stack_for_table(self, table_name): - ''' - Returns the insert stack of the specified table. - - Args: - table_name: string. Table name (must be part of database). - ''' - return self.tables['meta_insert_stack']._select_where('*', f'table_name={table_name}').column_by_name('indexes')[0] - # res = self.select('meta_insert_stack', '*', f'table_name={table_name}', return_object=True).indexes[0] - # return res - - def _update_meta_insert_stack_for_tb(self, table_name, new_stack): - ''' - Replaces the insert stack of a table with the one supplied by the user. - - Args: - table_name: string. Table name (must be part of database). - new_stack: string. The stack that will be used to replace the existing one. - ''' - self.tables['meta_insert_stack']._update_rows(new_stack, 'indexes', f'table_name={table_name}') - - - # indexes - def create_index(self, index_name, table_name, index_type='btree'): - ''' - Creates an index on a specified table with a given name. - Important: An index can only be created on a primary key (the user does not specify the column). - - Args: - table_name: string. Table name (must be part of database). - index_name: string. Name of the created index. - ''' - if self.tables[table_name].pk_idx is None: # if no primary key, no index - raise Exception('Cannot create index. Table has no primary key.') - if index_name not in self.tables['meta_indexes'].column_by_name('index_name'): - # currently only btree is supported. This can be changed by adding another if. - if index_type=='btree': - logging.info('Creating Btree index.') - # insert a record with the name of the index and the table on which it's created to the meta_indexes table - self.tables['meta_indexes']._insert([table_name, index_name]) - # crate the actual index - self._construct_index(table_name, index_name) - self.save_database() - else: - raise Exception('Cannot create index. Another index with the same name already exists.') - - def _construct_index(self, table_name, index_name): - ''' - Construct a btree on a table and save. - - Args: - table_name: string. Table name (must be part of database). - index_name: string. Name of the created index. - ''' - bt = Btree(3) # 3 is arbitrary - - # for each record in the primary key of the table, insert its value and index to the btree - for idx, key in enumerate(self.tables[table_name].column_by_name(self.tables[table_name].pk)): - if key is None: - continue - bt.insert(key, idx) - # save the btree - self._save_index(index_name, bt) - - - def _has_index(self, table_name): - ''' - Check whether the specified table's primary key column is indexed. - - Args: - table_name: string. Table name (must be part of database). - ''' - return table_name in self.tables['meta_indexes'].column_by_name('table_name') - - def _save_index(self, index_name, index): - ''' - Save the index object. - - Args: - index_name: string. Name of the created index. - index: obj. The actual index object (btree object). - ''' - try: - os.mkdir(f'{self.savedir}/indexes') - except: - pass - - with open(f'{self.savedir}/indexes/meta_{index_name}_index.pkl', 'wb') as f: - pickle.dump(index, f) - - def _load_idx(self, index_name): - ''' - Load and return the specified index. - - Args: - index_name: string. Name of created index. - ''' - f = open(f'{self.savedir}/indexes/meta_{index_name}_index.pkl', 'rb') - index = pickle.load(f) - f.close() - return index - - def drop_index(self, index_name): - ''' - Drop index from current database. - - Args: - index_name: string. Name of index. - ''' - if index_name in self.tables['meta_indexes'].column_by_name('index_name'): - self.delete_from('meta_indexes', f'index_name = {index_name}') - - if os.path.isfile(f'{self.savedir}/indexes/meta_{index_name}_index.pkl'): - os.remove(f'{self.savedir}/indexes/meta_{index_name}_index.pkl') - else: - warnings.warn(f'"{self.savedir}/indexes/meta_{index_name}_index.pkl" not found.') - - self.save_database() - \ No newline at end of file diff --git a/miniDBold/joins.py b/miniDBold/joins.py deleted file mode 100644 index 81fd0915..00000000 --- a/miniDBold/joins.py +++ /dev/null @@ -1,309 +0,0 @@ -import heapq -import shutil -import os -import ast -import sys - -sys.path.append(f'{os.path.dirname(os.path.dirname(os.path.abspath(__file__)))}/miniDB') - -from misc import reverse_op -from table import Table - - - -class Inlj: - def __init__(self, condition, left_table, right_table, index, index_saved): - self.left_table = left_table - self.right_table = right_table - self.condition = condition - self.join_table = None - self.results = None - self.index = index - self.index_saved = index_saved - - def join(self): - # Get the column of the left and right tables and the operator, from the condition of the join - column_name_left, operator, column_name_right = Table()._parse_condition(self.condition, join=True) - - reversed = False - # If we have the index of the left table, reverse the order of the tables - if(self.index_saved=='left'): - self.right_table, self.left_table = self.left_table, self.right_table - column_name_left, column_name_right = column_name_right, column_name_left - reversed = True - - # Try to find the left column, as even if a reverse took place, it is the only one needed - # If it fails, raise exception - try: - self.column_index_left = self.left_table.column_names.index(column_name_left) - except: - raise Exception(f'Column "{column_name_left}" doesn\'t exist in the left table. Valid columns: {self.left_table.column_names}.') - - # Create the names that appear over the tables when the final joined table is presented to the user - left_names = [f'{self.left_table._name}.{colname}' if self.left_table._name!='' else colname for colname in self.left_table.column_names] - right_names = [f'{self.right_table._name}.{colname}' if self.right_table._name!='' else colname for colname in self.right_table.column_names] - - join_table_colnames = left_names + right_names if not reversed else right_names + left_names - join_table_coltypes = self.left_table.column_types + self.right_table.column_types if not reversed else self.right_table.column_types + self.left_table.column_types - self.join_table = Table(name='', column_names=join_table_colnames, column_types=join_table_coltypes) - - # The operator needs to be reversed as we search based on the elements of the index. - # For example, if A > B and we search based on B, we need to search for B < A - operator = reverse_op(operator) if self.index_saved == 'right' else operator - - # Implementation of the index-nested-loop join - # If the tables had been reversed in the beginning, then the joined table appears - # with the tables shown in the order they appeared in the query - for row_left in self.left_table.data: - # The value that will be searched for in the index - left_value = row_left[self.column_index_left] - self.results = self.index.find(operator, left_value) - if len(self.results) > 0: - for element in self.results: - self.join_table._insert(row_left + self.right_table.data[element] if not reversed else self.right_table.data[element] + row_left) - - return self.join_table - -class Smj: - - def __init__(self, condition, left_table, right_table): - self.left_table = left_table - self.right_table = right_table - self.condition = condition - - def join(self): - # Get the column of the left and right tables and the operator, from the condition of the join - column_name_left, operator, column_name_right = Table()._parse_condition(self.condition, join=True) - column_index_left = self.left_table.column_names.index(column_name_left) - column_index_right = self.right_table.column_names.index(column_name_right) - - if(operator != "="): - raise Exception('Sort-Merge Join is used when the condition operator is "=".\n') - - # Create a temporary folder for the external sort to happen. The folder will be deleted in the end - os.makedirs('tempSMJfolder/externalSortFolder', exist_ok=True) - - # Create the names that appear over the tables when the final joined table is presented to the user - left_names = [f'{self.left_table._name}.{colname}' if self.left_table._name!='' else colname for colname in self.left_table.column_names] - right_names = [f'{self.right_table._name}.{colname}' if self.right_table._name!='' else colname for colname in self.right_table.column_names] - - # Write all the records of the right table to a local file in the following format: - # 'column_name_value [whole record]' - # Use special character '@@@' to represent spaces as spaces break the code. - with open('tempSMJfolder/externalSortFolder/rightTableFile', 'w+') as rt: - for row in self.right_table.data: - if row[column_index_right] is not None: - rt.write(f'{row[self.right_table.column_names.index(column_name_right)]} {str(row).replace(" ","@@@")}\n') - - # Same for the left lable - with open('tempSMJfolder/externalSortFolder/leftTableFile', 'w+') as lt: - for row in self.left_table.data: - if row[column_index_left] is not None: - lt.write(f'{row[self.left_table.column_names.index(column_name_left)]} {str(row).replace(" ","@@@")}\n') - - # Create an ExternalMergeSort object and sort both right table and left table local files - ems = self.ExternalMergeSort() - ems.runExternalSort('rightTableFile') - # Re-initialization of all values - ems = self.ExternalMergeSort() - ems.runExternalSort('leftTableFile') - - # Now there are sorted versions of the local files, so the initial ones can be removed - os.remove('tempSMJfolder/externalSortFolder/rightTableFile') - os.remove('tempSMJfolder/externalSortFolder/leftTableFile') - - # This does the final merge on sort-merge join - with open('tempSMJfolder/externalSortFolder/sorting of rightTableFile', 'r') as right, open('tempSMJfolder/externalSortFolder/sorting of leftTableFile', 'r') as left, open('tempSMJfolder/externalSortFolder/final', 'w+') as final: - mark = None #Used to return to previous values of the file - l = None - r = None - - # The algorithm runs until EOF of the left table file - while l != '': - try: - # If the mark is non-existent, set it equal to the current line of the right table sorted file - # While both files' column_values aren't equal, progress the current lines - if mark is None: - mark = right.tell() - l = left.readline() - r = right.readline() - while l.split()[0] < r.split()[0]: - l = left.readline() - while l.split()[0] > r.split()[0]: - mark = right.tell() - r = right.readline() - - # Now that the column_values are equal save both records to the final, joined_tables local file - # Then progress the right table's current line and continue with the procedure - if l.split()[0] == r.split()[0]: - final.write(l.replace("\n","")[l.index("["):] + " " + r.replace("\n","")[r.index("["):] + '\n') - r = right.readline() - - # Else, if left_value isn't equal to right_value after having found at least one equality of column_values - # return right table's current line to the mark, as the algorithm dictates - else: - right.seek(mark) - mark = None - # Finally, if the right file reaches EOF and IndexError happens, return right table's current line to the mark - except IndexError: - right.seek(mark) - mark = None - - # Now that the final joined file exists, the sorted files are not needed and are thus deleted - os.remove('tempSMJfolder/externalSortFolder/sorting of rightTableFile') - os.remove('tempSMJfolder/externalSortFolder/sorting of leftTableFile') - - join_table_name = '' - join_table_colnames = left_names + right_names - join_table_coltypes = self.left_table.column_types + self.right_table.column_types - join_table = Table(name=join_table_name, column_names=join_table_colnames, column_types= join_table_coltypes) - - # Save merged file first. The hypothesis is that the RAM cannot fit the file, thus we have it saved - # However we load the file to display it like this, might need to be changed in the future - with open('tempSMJfolder/externalSortFolder/final', 'r') as f: - for line in f: - records = line.split() - # ast.literal_eval creates the list [a,b,c] from the string '[a,b,c]' - join_table._insert(ast.literal_eval(records[0].replace('@@@', ' ')) + ast.literal_eval(records[1].replace('@@@', ' '))) - - # Finally, the final file and the externalSortFolder are not needed, as the joined table - # exists in a variable and can be presented to the user - os.remove(f'{os.getcwd()}/tempSMJfolder/externalSortFolder/final') - os.rmdir(f'{os.getcwd()}/tempSMJfolder/externalSortFolder') - os.rmdir(f'{os.getcwd()}/tempSMJfolder') - - return join_table - - class ExternalMergeSort: - # Total number of split files - numFiles = 0 - # Total number of numbers in the first file - sumFiles = 0 - # File name, so as to recognize the sorted file - startingFileName = '' - - # Sort the given array with the merge sort algorithm - def mergeSort(self, arr): - if len(arr) > 1: - mid = len(arr)//2 - L, R = arr[:mid], arr[mid:] - - # Recursive call of merge sort - self.mergeSort(L) - self.mergeSort(R) - - i = j = k = 0 - - while i < len(L) and j < len(R): - if L[i] < R[j]: - arr[k] = L[i] - i += 1 - else: - arr[k] = R[j] - j += 1 - k += 1 - - while i < len(L): - arr[k] = L[i] - i += 1 - k += 1 - - while j < len(R): - arr[k] = R[j] - j += 1 - k += 1 - - # Function to split the big file into smaller chunks of size specified by the user - def splitFile(self, largeFile, chunkSize:int): - # Variables used throughout the class - self.startingFileName = largeFile - self.numFiles = 1 - - # Split the file in chunks of chunkSize bytes - with open(f'tempSMJfolder/externalSortFolder/{largeFile}') as f: - chunk = f.readlines(chunkSize) - while chunk: - os.makedirs(os.path.dirname(f'tempSMJfolder/externalSortFolder/tempSplitFiles {self.startingFileName}/{self.numFiles}'), exist_ok=True) - with open(f'tempSMJfolder/externalSortFolder/tempSplitFiles {self.startingFileName}/{self.numFiles}', 'w+') as chunk_file: - for el in chunk: - chunk_file.write(el) - - chunk = f.readlines(chunkSize) - self.numFiles += 1 - - return self.numFiles - - # Function to sort a chunk of the starting file using merge sort - def sortSmallFile(self, fileToBeSorted): - arr = [] - - with open(f'tempSMJfolder/externalSortFolder/tempSplitFiles {self.startingFileName}/{fileToBeSorted}', 'r') as fts: - # If the contents of the file are integers - try: - with open(f'tempSMJfolder/externalSortFolder/tempSplitFiles {self.startingFileName}/{fileToBeSorted}', 'r') as fts: - arr = list(map(int, fts.read().splitlines())) - # If the contents are alphanumeric values - except: - with open(f'tempSMJfolder/externalSortFolder/tempSplitFiles {self.startingFileName}/{fileToBeSorted}', 'r') as fts: - arr = list(map(str, fts.read().splitlines())) - - self.sumFiles += len(arr) - self.mergeSort(arr) - - with open(f'tempSMJfolder/externalSortFolder/tempSplitFiles {self.startingFileName}/{fileToBeSorted}', 'w') as fts: - for el in arr: - fts.write(f'{el}\n') - - # K-Way Merge with priority queue implementation - def k_wayMerge(self, number): - # Create dictionary of files opened. Open all the files - # That will be merged - fileNames = {} - for i in range(1, number): - fileNames[i] = open(f'tempSMJfolder/externalSortFolder/tempSplitFiles {self.startingFileName}/{i}', 'r') - - output = [] - - # (X,Y) where - # X is the value of the element and - # Y is the key of the file in fileName - try: - pq = [(int(fileNames[i].readline().replace('\n', '')), i) for i in range(1, len(fileNames) + 1)] - except: - for i in range(1, len(fileNames) + 1): - fileNames[i].seek(0) - pq = [(fileNames[i].readline().replace('\n', ''), i) for i in range(1, len(fileNames) + 1)] - - # Create heap for the external merge sort - heapq.heapify(pq) - - while len(output) < self.sumFiles: - elem, file_key = heapq.heappop(pq) - output.append(elem) - next = fileNames[file_key].readline().replace('\n', '') - - # When on EOF, an empty string will be returned - # So if the value is not an empty string, add the integer to the heap - if next != '': - try: - heapq.heappush(pq, (int(next), file_key)) - except: - heapq.heappush(pq, (next, file_key)) - - with open(f'tempSMJfolder/externalSortFolder/sorting of {self.startingFileName}', 'w+') as sf: - for el in output: - sf.write(f'{el}\n') - - return output - - def runExternalSort(self, filename): - # 30 is just an example - self.splitFile(filename, 30) - - for i in range(1, self.numFiles): - self.sortSmallFile(i) - - self.k_wayMerge(self.numFiles) - - # After the k-way Merge is completed, remove the folder containing the temporary split files - shutil.rmtree(f'tempSMJfolder/externalSortFolder/tempSplitFiles {self.startingFileName}/') diff --git a/miniDBold/misc.py b/miniDBold/misc.py deleted file mode 100644 index 2a2ae985..00000000 --- a/miniDBold/misc.py +++ /dev/null @@ -1,65 +0,0 @@ -import operator - -def get_op(op, a, b): - ''' - Get op as a function of a and b by using a symbol - ''' - ops = {'>': operator.gt, - '<': operator.lt, - '>=': operator.ge, - '<=': operator.le, - '=': operator.eq, - 'not': operator.ne} - - try: - return ops[op](a,b) - except TypeError: # if a or b is None (deleted record), python3 raises typerror - return False - -def split_condition(condition): - ops = {'>=': operator.ge, - '<=': operator.le, - '=': operator.eq, - '>': operator.gt, - '<': operator.lt, - 'not': not_op} - - for op_key in ops.keys(): - splt=condition.split(op_key) - if len(splt)>1: - left, right = splt[0].strip(), splt[1].strip() - - if right[0] == '"' == right[-1]: # If the value has leading and trailing quotes, remove them. - right = right.strip('"') - elif ' ' in right: # If it has whitespaces but no leading and trailing double quotes, throw. - raise ValueError(f'Invalid condition: {condition}\nValue must be enclosed in double quotation marks to include whitespaces.') - - if right.find('"') != -1: # If there are any double quotes in the value, throw. (Notice we've already removed the leading and trailing ones) - raise ValueError(f'Invalid condition: {condition}\nDouble quotation marks are not allowed inside values.') - - return left, op_key, right - -def reverse_op(op): - ''' - Reverse the operator given - ''' - return { - '>' : '<', - '>=' : '<=', - '<' : '>', - '<=' : '>=', - '=' : '=' - }.get(op) - -def not_op(op): - ''' - Return opposite of the operator given - ''' - return { - '>' : '<=', - '>=' : '<', - '<' : '>=', - '<=' : '>', - '=' : '!=', - '!=' : '=' - }.get(op) diff --git a/miniDBold/table.py b/miniDBold/table.py deleted file mode 100644 index f5c7d937..00000000 --- a/miniDBold/table.py +++ /dev/null @@ -1,579 +0,0 @@ -from __future__ import annotations -from tabulate import tabulate -import pickle -import os -import sys - -sys.path.append(f'{os.path.dirname(os.path.dirname(os.path.abspath(__file__)))}/miniDB') - -from misc import get_op, split_condition - - -class Table: - ''' - Table object represents a table inside a database - - A Table object can be created either by assigning: - - a table name (string) - - column names (list of strings) - - column types (list of functions like str/int etc) - - primary (name of the primary key column) - - OR - - - by assigning a value to the variable called load. This value can be: - - a path to a Table file saved using the save function - - a dictionary that includes the appropriate info (all the attributes in __init__) - - ''' - def __init__(self, name=None, column_names=None, column_types=None, primary_key=None, load=None): - - if load is not None: - # if load is a dict, replace the object dict with it (replaces the object with the specified one) - if isinstance(load, dict): - self.__dict__.update(load) - # self._update() - # if load is str, load from a file - elif isinstance(load, str): - self._load_from_file(load) - - # if name, columns_names and column types are not none - elif (name is not None) and (column_names is not None) and (column_types is not None): - - self._name = name - - if len(column_names)!=len(column_types): - raise ValueError('Need same number of column names and types.') - - self.column_names = column_names - - self.columns = [] - - for col in self.column_names: - if col not in self.__dir__(): - # this is used in order to be able to call a column using its name as an attribute. - # example: instead of table.columns['column_name'], we do table.column_name - setattr(self, col, []) - self.columns.append([]) - else: - raise Exception(f'"{col}" attribute already exists in "{self.__class__.__name__} "class.') - - self.column_types = [eval(ct) if not isinstance(ct, type) else ct for ct in column_types] - self.data = [] # data is a list of lists, a list of rows that is. - - # if primary key is set, keep its index as an attribute - if primary_key is not None: - self.pk_idx = self.column_names.index(primary_key) - else: - self.pk_idx = None - - self.pk = primary_key - # self._update() - - # if any of the name, columns_names and column types are none. return an empty table object - - def column_by_name(self, column_name): - return [row[self.column_names.index(column_name)] for row in self.data] - - - def _update(self): - ''' - Update all the available columns with the appended rows. - ''' - self.columns = [[row[i] for row in self.data] for i in range(len(self.column_names))] - for ind, col in enumerate(self.column_names): - setattr(self, col, self.columns[ind]) - - def _cast_column(self, column_name, cast_type): - ''' - Cast all values of a column using a specified type. - - Args: - column_name: string. The column that will be casted. - cast_type: type. Cast type (do not encapsulate in quotes). - ''' - # get the column from its name - column_idx = self.column_names.index(column_name) - # for every column's value in each row, replace it with itself but casted as the specified type - for i in range(len(self.data)): - self.data[i][column_idx] = cast_type(self.data[i][column_idx]) - # change the type of the column - self.column_types[column_idx] = cast_type - # self._update() - - - def _insert(self, row, insert_stack=[]): - ''' - Insert row to table. - - Args: - row: list. A list of values to be inserted (will be casted to a predifined type automatically). - insert_stack: list. The insert stack (empty by default). - ''' - if len(row)!=len(self.column_names): - raise ValueError(f'ERROR -> Cannot insert {len(row)} values. Only {len(self.column_names)} columns exist') - - for i in range(len(row)): - # for each value, cast and replace it in row. - try: - row[i] = self.column_types[i](row[i]) - except ValueError: - if row[i] != 'NULL': - raise ValueError(f'ERROR -> Value {row[i]} of type {type(row[i])} is not of type {self.column_types[i]}.') - except TypeError as exc: - if row[i] != None: - print(exc) - - # if value is to be appended to the primary_key column, check that it doesnt alrady exist (no duplicate primary keys) - if i==self.pk_idx and row[i] in self.column_by_name(self.pk): - raise ValueError(f'## ERROR -> Value {row[i]} already exists in primary key column.') - elif i==self.pk_idx and row[i] is None: - raise ValueError(f'ERROR -> The value of the primary key cannot be None.') - - # if insert_stack is not empty, append to its last index - if insert_stack != []: - self.data[insert_stack[-1]] = row - else: # else append to the end - self.data.append(row) - # self._update() - - def _update_rows(self, set_value, set_column, condition): - ''' - Update where Condition is met. - - Args: - set_value: string. The provided set value. - set_column: string. The column to be altered. - condition: string. A condition using the following format: - 'column[<,<=,=,>=,>]value' or - 'value[<,<=,=,>=,>]column'. - - Operatores supported: (<,<=,=,>=,>) - ''' - # parse the condition - column_name, operator, value = self._parse_condition(condition) - - # get the condition and the set column - column = self.column_by_name(column_name) - set_column_idx = self.column_names.index(set_column) - - # set_columns_indx = [self.column_names.index(set_column_name) for set_column_name in set_column_names] - - # for each value in column, if condition, replace it with set_value - for row_ind, column_value in enumerate(column): - if get_op(operator, column_value, value): - self.data[row_ind][set_column_idx] = set_value - - # self._update() - # print(f"Updated {len(indexes_to_del)} rows") - - - def _delete_where(self, condition): - ''' - Deletes rows where condition is met. - - Important: delete replaces the rows to be deleted with rows filled with Nones. - These rows are then appended to the insert_stack. - - Args: - condition: string. A condition using the following format: - 'column[<,<=,==,>=,>]value' or - 'value[<,<=,==,>=,>]column'. - - Operatores supported: (<,<=,==,>=,>) - ''' - column_name, operator, value = self._parse_condition(condition) - - indexes_to_del = [] - - column = self.column_by_name(column_name) - for index, row_value in enumerate(column): - if get_op(operator, row_value, value): - indexes_to_del.append(index) - - # we pop from highest to lowest index in order to avoid removing the wrong item - # since we dont delete, we dont have to to pop in that order, but since delete is used - # to delete from meta tables too, we still implement it. - - for index in sorted(indexes_to_del, reverse=True): - if self._name[:4] != 'meta': - # if the table is not a metatable, replace the row with a row of nones - self.data[index] = [None for _ in range(len(self.column_names))] - else: - self.data.pop(index) - - # self._update() - # we have to return the deleted indexes, since they will be appended to the insert_stack - return indexes_to_del - - - def _select_where(self, return_columns, condition=None, distinct=False, order_by=None, desc=True, limit=None): - ''' - Select and return a table containing specified columns and rows where condition is met. - - Args: - return_columns: list. The columns to be returned. - condition: string. A condition using the following format: - 'column[<,<=,==,>=,>]value' or - 'value[<,<=,==,>=,>]column'. - - Operatores supported: (<,<=,==,>=,>) - distinct: boolean. If True, the resulting table will contain only unique rows (False by default). - order_by: string. A column name that signals that the resulting table should be ordered based on it (no order if None). - desc: boolean. If True, order_by will return results in descending order (False by default). - limit: int. An integer that defines the number of rows that will be returned (all rows if None). - ''' - - # if * return all columns, else find the column indexes for the columns specified - if return_columns == '*': - return_cols = [i for i in range(len(self.column_names))] - else: - return_cols = [self.column_names.index(col.strip()) for col in return_columns.split(',')] - - # if condition is None, return all rows - # if not, return the rows with values where condition is met for value - if condition is not None: - column_name, operator, value = self._parse_condition(condition) - column = self.column_by_name(column_name) - rows = [ind for ind, x in enumerate(column) if get_op(operator, x, value)] - else: - rows = [i for i in range(len(self.data))] - - # copy the old dict, but only the rows and columns of data with index in rows/columns (the indexes that we want returned) - dict = {(key):([[self.data[i][j] for j in return_cols] for i in rows] if key=="data" else value) for key,value in self.__dict__.items()} - - # we need to set the new column names/types and no of columns, since we might - # only return some columns - dict['column_names'] = [self.column_names[i] for i in return_cols] - dict['column_types'] = [self.column_types[i] for i in return_cols] - - s_table = Table(load=dict) - - s_table.data = list(set(map(lambda x: tuple(x), s_table.data))) if distinct else s_table.data - - if order_by: - s_table.order_by(order_by, desc) - - # if isinstance(limit, str): - # try: - # k = int(limit) - # except ValueError: - # raise Exception("The value following 'top' in the query should be a number.") - - # # Remove from the table's data all the None-filled rows, as they are not shown by default - # # Then, show the first k rows - # s_table.data.remove(len(s_table.column_names) * [None]) - # s_table.data = s_table.data[:k] - if isinstance(limit,str): - s_table.data = [row for row in s_table.data if any(row)][:int(limit)] - - return s_table - - - def _select_where_with_btree(self, return_columns, bt, condition, distinct=False, order_by=None, desc=True, limit=None): - - # if * return all columns, else find the column indexes for the columns specified - if return_columns == '*': - return_cols = [i for i in range(len(self.column_names))] - else: - return_cols = [self.column_names.index(colname) for colname in return_columns] - - - column_name, operator, value = self._parse_condition(condition) - - # if the column in condition is not a primary key, abort the select - if column_name != self.column_names[self.pk_idx]: - print('Column is not PK. Aborting') - - # here we run the same select twice, sequentially and using the btree. - # we then check the results match and compare performance (number of operation) - column = self.column_by_name(column_name) - - # sequential - rows1 = [] - opsseq = 0 - for ind, x in enumerate(column): - opsseq+=1 - if get_op(operator, x, value): - rows1.append(ind) - - # btree find - rows = bt.find(operator, value) - - try: - k = int(limit) - except TypeError: - k = None - # same as simple select from now on - rows = rows[:k] - # TODO: this needs to be dumbed down - dict = {(key):([[self.data[i][j] for j in return_cols] for i in rows] if key=="data" else value) for key,value in self.__dict__.items()} - - dict['column_names'] = [self.column_names[i] for i in return_cols] - dict['column_types'] = [self.column_types[i] for i in return_cols] - - s_table = Table(load=dict) - - s_table.data = list(set(map(lambda x: tuple(x), s_table.data))) if distinct else s_table.data - - if order_by: - s_table.order_by(order_by, desc) - - if isinstance(limit,str): - s_table.data = [row for row in s_table.data if row is not None][:int(limit)] - - return s_table - - def order_by(self, column_name, desc=True): - ''' - Order table based on column. - - Args: - column_name: string. Name of column. - desc: boolean. If True, order_by will return results in descending order (False by default). - ''' - column = [val if val is not None else 0 for val in self.column_by_name(column_name)] - idx = sorted(range(len(column)), key=lambda k: column[k], reverse=desc) - # print(idx) - self.data = [self.data[i] for i in idx] - # self._update() - - - def _general_join_processing(self, table_right:Table, condition, join_type): - ''' - Performs the processes all the join operations need (regardless of type) so that there is no code repetition. - - Args: - condition: string. A condition using the following format: - 'column[<,<=,==,>=,>]value' or - 'value[<,<=,==,>=,>]column'. - - Operators supported: (<,<=,==,>=,>) - ''' - # get columns and operator - column_name_left, operator, column_name_right = self._parse_condition(condition, join=True) - # try to find both columns, if you fail raise error - - if(operator != '=' and join_type in ['left','right','full']): - class CustomFailException(Exception): - pass - raise CustomFailException('Outer Joins can only be used if the condition operator is "=".\n') - - try: - column_index_left = self.column_names.index(column_name_left) - except: - raise Exception(f'Column "{column_name_left}" dont exist in left table. Valid columns: {self.column_names}.') - - try: - column_index_right = table_right.column_names.index(column_name_right) - except: - raise Exception(f'Column "{column_name_right}" dont exist in right table. Valid columns: {table_right.column_names}.') - - # get the column names of both tables with the table name in front - # ex. for left -> name becomes left_table_name_name etc - left_names = [f'{self._name}.{colname}' if self._name!='' else colname for colname in self.column_names] - right_names = [f'{table_right._name}.{colname}' if table_right._name!='' else colname for colname in table_right.column_names] - - # define the new tables name, its column names and types - join_table_name = '' - join_table_colnames = left_names+right_names - join_table_coltypes = self.column_types+table_right.column_types - join_table = Table(name=join_table_name, column_names=join_table_colnames, column_types= join_table_coltypes) - - return join_table, column_index_left, column_index_right, operator - - - def _inner_join(self, table_right: Table, condition): - ''' - Join table (left) with a supplied table (right) where condition is met. - - Args: - condition: string. A condition using the following format: - 'column[<,<=,==,>=,>]value' or - 'value[<,<=,==,>=,>]column'. - - Operators supported: (<,<=,==,>=,>) - ''' - join_table, column_index_left, column_index_right, operator = self._general_join_processing(table_right, condition, 'inner') - - # count the number of operations (<,> etc) - no_of_ops = 0 - # this code is dumb on purpose... it needs to illustrate the underline technique - # for each value in left column and right column, if condition, append the corresponding row to the new table - for row_left in self.data: - left_value = row_left[column_index_left] - for row_right in table_right.data: - right_value = row_right[column_index_right] - if(left_value is None and right_value is None): - continue - no_of_ops+=1 - if get_op(operator, left_value, right_value): #EQ_OP - join_table._insert(row_left+row_right) - - return join_table - - def _left_join(self, table_right: Table, condition): - ''' - Perform a left join on the table with the supplied table (right). - - Args: - condition: string. A condition using the following format: - 'column[<,<=,==,>=,>]value' or - 'value[<,<=,==,>=,>]column'. - - Operators supported: (<,<=,==,>=,>) - ''' - join_table, column_index_left, column_index_right, operator = self._general_join_processing(table_right, condition, 'left') - - right_column = table_right.column_by_name(table_right.column_names[column_index_right]) - right_table_row_length = len(table_right.column_names) - - for row_left in self.data: - left_value = row_left[column_index_left] - if left_value is None: - continue - elif left_value not in right_column: - join_table._insert(row_left + right_table_row_length*["NULL"]) - else: - for row_right in table_right.data: - right_value = row_right[column_index_right] - if left_value == right_value: - join_table._insert(row_left + row_right) - - return join_table - - def _right_join(self, table_right: Table, condition): - ''' - Perform a right join on the table with the supplied table (right). - - Args: - condition: string. A condition using the following format: - 'column[<,<=,==,>=,>]value' or - 'value[<,<=,==,>=,>]column'. - - Operators supported: (<,<=,==,>=,>) - ''' - join_table, column_index_left, column_index_right, operator = self._general_join_processing(table_right, condition, 'right') - - left_column = self.column_by_name(self.column_names[column_index_left]) - left_table_row_length = len(self.column_names) - - for row_right in table_right.data: - right_value = row_right[column_index_right] - if right_value is None: - continue - elif right_value not in left_column: - join_table._insert(left_table_row_length*["NULL"] + row_right) - else: - for row_left in self.data: - left_value = row_left[column_index_left] - if left_value == right_value: - join_table._insert(row_left + row_right) - - return join_table - - def _full_join(self, table_right: Table, condition): - ''' - Perform a full join on the table with the supplied table (right). - - Args: - condition: string. A condition using the following format: - 'column[<,<=,==,>=,>]value' or - 'value[<,<=,==,>=,>]column'. - - Operators supported: (<,<=,==,>=,>) - ''' - join_table, column_index_left, column_index_right, operator = self._general_join_processing(table_right, condition, 'full') - - right_column = table_right.column_by_name(table_right.column_names[column_index_right]) - left_column = self.column_by_name(self.column_names[column_index_left]) - - right_table_row_length = len(table_right.column_names) - left_table_row_length = len(self.column_names) - - for row_left in self.data: - left_value = row_left[column_index_left] - if left_value is None: - continue - if left_value not in right_column: - join_table._insert(row_left + right_table_row_length*["NULL"]) - else: - for row_right in table_right.data: - right_value = row_right[column_index_right] - if left_value == right_value: - join_table._insert(row_left + row_right) - - for row_right in table_right.data: - right_value = row_right[column_index_right] - - if right_value is None: - continue - elif right_value not in left_column: - join_table._insert(left_table_row_length*["NULL"] + row_right) - - return join_table - - def show(self, no_of_rows=None, is_locked=False): - ''' - Print the table in a nice readable format. - - Args: - no_of_rows: int. Number of rows. - is_locked: boolean. Whether it is locked (False by default). - ''' - output = "" - # if the table is locked, add locked keyword to title - if is_locked: - output += f"\n## {self._name} (locked) ##\n" - else: - output += f"\n## {self._name} ##\n" - - # headers -> "column name (column type)" - headers = [f'{col} ({tp.__name__})' for col, tp in zip(self.column_names, self.column_types)] - if self.pk_idx is not None: - # table has a primary key, add PK next to the appropriate column - headers[self.pk_idx] = headers[self.pk_idx]+' #PK#' - # detect the rows that are no tfull of nones (these rows have been deleted) - # if we dont skip these rows, the returning table has empty rows at the deleted positions - non_none_rows = [row for row in self.data if any(row)] - # print using tabulate - print(tabulate(non_none_rows[:no_of_rows], headers=headers)+'\n') - - - def _parse_condition(self, condition, join=False): - ''' - Parse the single string condition and return the value of the column and the operator. - - Args: - condition: string. A condition using the following format: - 'column[<,<=,==,>=,>]value' or - 'value[<,<=,==,>=,>]column'. - - Operatores supported: (<,<=,==,>=,>) - join: boolean. Whether to join or not (False by default). - ''' - # if both_columns (used by the join function) return the names of the names of the columns (left first) - if join: - return split_condition(condition) - - # cast the value with the specified column's type and return the column name, the operator and the casted value - left, op, right = split_condition(condition) - if left not in self.column_names: - raise ValueError(f'Condition is not valid (cant find column name)') - coltype = self.column_types[self.column_names.index(left)] - - return left, op, coltype(right) - - - def _load_from_file(self, filename): - ''' - Load table from a pkl file (not used currently). - - Args: - filename: string. Name of pkl file. - ''' - f = open(filename, 'rb') - tmp_dict = pickle.load(f) - f.close() - - self.__dict__.update(tmp_dict.__dict__)