diff --git a/Makefile b/Makefile index dd4dc9b..de4a904 100644 --- a/Makefile +++ b/Makefile @@ -7,6 +7,8 @@ $(info $(OS_SUPPORT)) server.bin: g++ server/server.cpp $(OS_SUPPORT) --std=c++1z -O3 -march=native -o server.bin +server.so: + g++ server/server.cpp -shared $(OS_SUPPORT) --std=c++1z -O3 -march=native -o server.so snippet: g++ -shared -fPIC --std=c++1z out.cpp -O3 -march=native -o dll.so clean: diff --git a/README.md b/README.md index e6cec8f..13077ce 100644 --- a/README.md +++ b/README.md @@ -41,4 +41,4 @@ Recent version of Linux, Windows or MacOS, with recent C++ compiler that has C++ - `exit`: quit the prompt #### Example: `f moving_avg.a`
- `exec` \ No newline at end of file + `exec` diff --git a/dbconn.py b/dbconn.py index d6a283a..443fd23 100644 --- a/dbconn.py +++ b/dbconn.py @@ -1,45 +1,45 @@ -import mariadb - -class dbconn: - def __init__(self) -> None: - self.db = None - self.cur = None - def clear(self): - drop_all = f''' - SET FOREIGN_KEY_CHECKS = 0; - - SET @tables = NULL; - - SELECT GROUP_CONCAT('`', table_schema, '`.`', table_name, '`') INTO @tables - FROM information_schema.tables - WHERE table_schema = '{self.db.database}'; - - SET @tables = CONCAT('DROP TABLE ', @tables); - PREPARE stmt FROM @tables; - EXECUTE stmt; - DEALLOCATE PREPARE stmt; - SET FOREIGN_KEY_CHECKS = 1; - ''' - if self.db: - if not self.cur: - self.cur = self.db.cursor() - self.cur.execute(drop_all) - - def connect(self, ip, password = '0508', user = 'root', db = 'db', port = 3306): - try: - self.db = mariadb.connect( - user = user, - password = password, - host = ip, - port = port, - database = db - ) - self.cur = self.db.cursor() - - except mariadb.Error as e: - print(e) - self.db = None - self.cur = None - - def exec(self, sql, params = None): +import mariadb + +class dbconn: + def __init__(self) -> None: + self.db = None + self.cur = None + def clear(self): + drop_all = f''' + SET FOREIGN_KEY_CHECKS = 0; + + SET @tables = NULL; + + SELECT GROUP_CONCAT('`', table_schema, '`.`', table_name, '`') INTO @tables + FROM information_schema.tables + WHERE table_schema = '{self.db.database}'; + + SET @tables = CONCAT('DROP TABLE ', @tables); + PREPARE stmt FROM @tables; + EXECUTE stmt; + DEALLOCATE PREPARE stmt; + SET FOREIGN_KEY_CHECKS = 1; + ''' + if self.db: + if not self.cur: + self.cur = self.db.cursor() + self.cur.execute(drop_all) + + def connect(self, ip, password = '0508', user = 'root', db = 'db', port = 3306): + try: + self.db = mariadb.connect( + user = user, + password = password, + host = ip, + port = port, + database = db + ) + self.cur = self.db.cursor() + + except mariadb.Error as e: + print(e) + self.db = None + self.cur = None + + def exec(self, sql, params = None): self.cur.execute(sql) \ No newline at end of file diff --git a/out.cpp b/out.cpp index 9eed642..768a22d 100644 --- a/out.cpp +++ b/out.cpp @@ -1,6 +1,6 @@ -#include "./server/libaquery.h" #include "./server/aggregations.h" #include "csv.h" +#include "./server/libaquery.h" #include "./server/hasher.h" #include @@ -18,40 +18,40 @@ test_a.init("a"); test_b.init("b"); test_c.init("c"); test_d.init("d"); -io::CSVReader<4> csv_reader_307VD4("test.csv"); -csv_reader_307VD4.read_header(io::ignore_extra_column, "a","b","c","d"); -int tmp_3LXIYQmp; -int tmp_1m5NCKR4; -int tmp_10LZcLgy; -int tmp_39pPZL8W; -while(csv_reader_307VD4.read_row(tmp_3LXIYQmp,tmp_1m5NCKR4,tmp_10LZcLgy,tmp_39pPZL8W)) { +io::CSVReader<4> csv_reader_1qh80y("test.csv"); +csv_reader_1qh80y.read_header(io::ignore_extra_column, "a","b","c","d"); +int tmp_6JfuovoZ; +int tmp_4B4ADRgW; +int tmp_3JHd3elW; +int tmp_1heR8kZw; +while(csv_reader_1qh80y.read_row(tmp_6JfuovoZ,tmp_4B4ADRgW,tmp_3JHd3elW,tmp_1heR8kZw)) { -test_a.emplace_back(tmp_3LXIYQmp); -test_b.emplace_back(tmp_1m5NCKR4); -test_c.emplace_back(tmp_10LZcLgy); -test_d.emplace_back(tmp_39pPZL8W); +test_a.emplace_back(tmp_6JfuovoZ); +test_b.emplace_back(tmp_4B4ADRgW); +test_c.emplace_back(tmp_3JHd3elW); +test_d.emplace_back(tmp_1heR8kZw); } -typedef record record_type3OMslKw; -unordered_map, transTypes> g7LNVAss; -for (uint32_t i1T = 0; i1T < test_a.size; ++i1T){ -g7LNVAss[forward_as_tuple(test_a[i1T],test_b[i1T],test_d[i1T])].emplace_back(i1T); +typedef record record_type3he5qd1; +unordered_map, transTypes> g59vWI2v; +for (uint32_t i3P = 0; i3P < test_a.size; ++i3P){ +g59vWI2v[forward_as_tuple(test_a[i3P],test_b[i3P],test_d[i3P])].emplace_back(i3P); } -auto out_HSfK = new TableInfo,value_type>,value_type>>("out_HSfK", 3); -cxt->tables.insert({"out_HSfK", out_HSfK}); -auto& out_HSfK_sumtestc = *(ColRef> *)(&out_HSfK->colrefs[0]); -auto& out_HSfK_b = *(ColRef>> *)(&out_HSfK->colrefs[1]); -auto& out_HSfK_d = *(ColRef>> *)(&out_HSfK->colrefs[2]); -out_HSfK_sumtestc.init("sumtestc"); -out_HSfK_b.init("b"); -out_HSfK_d.init("d"); -for(auto& i18 : g7LNVAss) { -auto &key_3s5slnK = i18.first; -auto &val_2nNLv0D = i18.second; -out_HSfK_sumtestc.emplace_back(sum(test_c[val_2nNLv0D])); -out_HSfK_b.emplace_back(get<1>(key_3s5slnK)); -out_HSfK_d.emplace_back(get<2>(key_3s5slnK)); +auto out_6JAp = new TableInfo,value_type>,value_type>>("out_6JAp", 3); +cxt->tables.insert({"out_6JAp", out_6JAp}); +auto& out_6JAp_sumtestc = *(ColRef> *)(&out_6JAp->colrefs[0]); +auto& out_6JAp_b = *(ColRef>> *)(&out_6JAp->colrefs[1]); +auto& out_6JAp_d = *(ColRef>> *)(&out_6JAp->colrefs[2]); +out_6JAp_sumtestc.init("sumtestc"); +out_6JAp_b.init("b"); +out_6JAp_d.init("d"); +for(auto& i2Y : g59vWI2v) { +auto &key_1yBYhdd = i2Y.first; +auto &val_61QXy6G = i2Y.second; +out_6JAp_sumtestc.emplace_back(sum(test_c[val_61QXy6G])); +out_6JAp_b.emplace_back(get<1>(key_1yBYhdd)); +out_6JAp_d.emplace_back(get<2>(key_1yBYhdd)); } -auto d5b7C95U = out_HSfK->order_by_view<-3,1>(); -print(d5b7C95U); +auto d1dyPtv0 = out_6JAp->order_by_view<-3,1>(); +print(d1dyPtv0); return 0; } \ No newline at end of file diff --git a/prompt.py b/prompt.py index f1a317b..c94d931 100644 --- a/prompt.py +++ b/prompt.py @@ -1,3 +1,4 @@ +import enum import re import time import dbconn @@ -12,14 +13,26 @@ import sys import os from engine.utils import base62uuid import atexit + +import threading +import ctypes + +class RunType(enum.Enum): + Threaded = 0 + IPC = 1 + +server_mode = RunType.Threaded + +server_bin = 'server.bin' if server_mode == RunType.IPC else 'server.so' + try: - os.remove('server.bin') + os.remove(server_bin) except Exception as e: print(type(e), e) nullstream = open(os.devnull, 'w') -subprocess.call(['make', 'server.bin'], stdout=nullstream) +subprocess.call(['make', server_bin], stdout=nullstream) cleanup = True def rm(): @@ -44,10 +57,8 @@ def rm(): mm.close() cleanup = False nullstream.close() - -atexit.register(rm) -def init(): +def init_ipc(): global shm, server, basecmd, mm shm = base62uuid() if sys.platform != 'win32': @@ -70,6 +81,100 @@ def init(): mm.flush() server = subprocess.Popen(["./server.bin", shm]) +import numpy as np + +c = lambda _ba: ctypes.cast((ctypes.c_char * len(_ba)).from_buffer(_ba), ctypes.c_char_p) + +class Config: + def __init__(self, nq = 0, mode = server_mode, n_bufs = 0, bf_szs = []) -> None: + self.int_size = 4 + self.n_attrib = 4 + self.buf = bytearray((self.n_attrib + n_bufs) * self.int_size) + self.np_buf = np.ndarray(shape=(self.n_attrib), buffer=self.buf, dtype=np.int32) + self.new_query = nq + self.server_mode = mode.value + self.running = 1 + self.n_buffers = n_bufs + + @property + def running(self): + return self.np_buf[0] + @running.setter + def running(self, rn): + self.np_buf[0] = rn + + @property + def new_query(self): + return self.np_buf[1] + @new_query.setter + def new_query(self, nq): + self.np_buf[1] = nq + + @property + def server_mode(self): + return self.np_buf[2] + @server_mode.setter + def server_mode(self, mode): + self.np_buf[2] = mode + + @property + def n_buffers(self): + return self.np_buf[3] + @n_buffers.setter + def n_buffers(self, n_bufs): + self.np_buf[3] = n_bufs + + def set_bufszs(self, buf_szs): + for i in range(min(len(buf_szs), self.n_buffers)): + self.np_buf[i+self.n_attrib] = buf_szs[i] + + @property + def c(self): + return c(self.buf) + +cfg = Config() +th = None + +def init_threaded(): + + if os.name == 'nt': + t = os.environ['PATH'].lower().split(';') + vars = re.compile('%.*%') + for e in t: + if(len(e) != 0): + if '%' in e: + try: + m_e = vars.findall(e) + for m in m_e: + e = e.replace(m, os.environ[m[1:-1]]) + # print(m, e) + except Exception: + continue + os.add_dll_directory(e) + + server_so = ctypes.CDLL('./'+server_bin) + global cfg, th + th = threading.Thread(target=server_so['main'], args=(-1, ctypes.POINTER(ctypes.c_char_p)(cfg.c)), daemon=True) + th.start() + +if server_mode == RunType.IPC: + atexit.register(rm) + init = init_ipc + set_ready = lambda : mm.seek(0,os.SEEK_SET) or mm.write(b'\x01\x01') + def __get_ready(): + mm.seek(0,os.SEEK_SET) + return mm.read(2)[1] + get_ready = __get_ready + server_status = lambda : server.poll() is not None +else: + init = init_threaded + rm = lambda: None + def __set_ready(): + global cfg + cfg.new_query = 1 + set_ready = __set_ready + get_ready = lambda:cfg.new_query + server_status = lambda : not th.is_alive() init() test_parser = True @@ -82,31 +187,16 @@ q = 'SELECT p.Name, v.Name FROM Production.Product p JOIN Purchasing.ProductVend res = parser.parse(q) -# else:f -# if subprocess.call(['make', 'snippet']) == 0: -# mm.seek(0) -# mm.write(b'\x01\x01') -# time.sleep(.1) -# mm.seek(0) -# print(mm.read(2)) - -# mm.close() -# handle.close() -# os.remove(shm) -# exit() keep = True cxt = engine.initialize() cxt.Info(res) while test_parser: try: - if server.poll() is not None: + if server_status(): init() - print("> ", end="") - ready = 1 - while ready == 1: - mm.seek(0,os.SEEK_SET) - ready = mm.read(2)[1] + while get_ready(): time.sleep(.00001) + print("> ", end="") q = input().lower() if q == 'exec': if not keep or cxt is None: @@ -123,8 +213,7 @@ while test_parser: with open('out.cpp', 'wb') as outfile: outfile.write((cxt.finalize()).encode('utf-8')) if subprocess.call(['make', 'snippet'], stdout = nullstream) == 0: - mm.seek(0,os.SEEK_SET) - mm.write(b'\x01\x01') + set_ready() continue if q == 'xexec': cxt = xengine.initialize() @@ -159,8 +248,10 @@ while test_parser: break elif q == 'r': if subprocess.call(['make', 'snippet']) == 0: - mm.seek(0,os.SEEK_SET) - mm.write(b'\x01\x01') + set_ready() + continue + elif q == 'rr': + set_ready() continue elif q.startswith('save'): filename = re.split(' |\t', q) diff --git a/reconstruct/__init__.py b/reconstruct/__init__.py index e99ac71..e103251 100644 --- a/reconstruct/__init__.py +++ b/reconstruct/__init__.py @@ -1,11 +1,11 @@ -from reconstruct.ast import Context, ast_node - -def initialize(): - return Context() - -def generate(ast, cxt): - for k in ast.keys(): - if k in ast_node.types.keys(): - ast_node.types[k](None, ast, cxt) - -__all__ = ["initialize", "generate"] +from reconstruct.ast import Context, ast_node + +def initialize(): + return Context() + +def generate(ast, cxt): + for k in ast.keys(): + if k in ast_node.types.keys(): + ast_node.types[k](None, ast, cxt) + +__all__ = ["initialize", "generate"] diff --git a/reconstruct/ast.py b/reconstruct/ast.py index d1dc0c2..b654710 100644 --- a/reconstruct/ast.py +++ b/reconstruct/ast.py @@ -1,303 +1,303 @@ -from engine.utils import enlist, base62uuid, base62alp -from reconstruct.storage import Context, TableInfo, ColRef - -class ast_node: - header = [] - types = dict() - first_order = False - - def __init__(self, parent:"ast_node", node, context:Context = None): - self.context = parent.context if context is None else context - self.parent = parent - self.sql = '' - self.datasource = None - self.init(node) - self.produce(node) - self.spawn(node) - self.consume(node) - - def emit(self, code): - self.context.emit(code) - def add(self, code): - self.sql += code + ' ' - - name = 'null' - - def init(self, _): - self.add(self.__class__.name.upper()) - def produce(self, _): - pass - def spawn(self, _): - pass - - def consume(self, _): - if self.parent is None: - self.emit(self.sql+';\n') - - -from reconstruct.expr import expr - - -class projection(ast_node): - name = 'projection' - first_order = 'select' - - def init(self, _): - pass - def produce(self, node): - p = node['select'] - self.projections = p if type(p) is list else [p] - self.add('SELECT') - - def spawn(self, node): - self.datasource = None # datasource is Join instead of TableInfo - if 'from' in node: - from_clause = node['from'] - self.datasource = join(self, from_clause) - if 'assumptions' in from_clause: - self.assumptions = enlist(from_clause['assumptions']) - - if self.datasource is not None: - self.datasource_changed = True - self.prev_datasource = self.context.datasource - self.context.datasource = self.datasource - - if 'where' in node: - self.where = filter(self, node['where']) - else: - self.where = None - - if 'groupby' in node: - self.group_node = groupby(self, node['groupby']) - else: - self.group_node = None - - def consume(self, node): - # deal with projections - self.out_table = TableInfo('out_'+base62uuid(4), [], self.context) - cols = [] - col_exprs = [] - for i, proj in enumerate(self.projections): - compound = False - self.datasource.rec = set() - name = '' - if type(proj) is dict: - - if 'value' in proj: - e = proj['value'] - name = expr(self, e).sql - disp_name = ''.join([a if a in base62alp else '' for a in name]) - compound = True # compound column - if 'name' in proj: # renaming column by AS keyword - name += ' ' + proj['name'] - col_exprs.append(name) - - elif type(proj) is str: - col = self.datasource.get_col(proj) - name = col.name - self.datasource.rec = None - # TODO: Type deduction in Python - cols.append(ColRef('unknown', self.out_table, None, disp_name, i, compound=compound)) - self.add(', '.join(col_exprs)) - - def finialize(astnode:ast_node): - if(astnode is not None): - self.add(astnode.sql) - self.add('FROM') - finialize(self.datasource) - finialize(self.where) - finialize(self.group_node) - if 'orderby' in node: - self.add(orderby(self, node['orderby']).sql) - if 'outfile' in node: - self.add(outfile(self, node['outfile']).sql) - if self.parent is None: - self.emit(self.sql+';\n') - else: - # TODO: subquery, name create tmp-table from subquery w/ alias as name - pass - - -class orderby(ast_node): - name = 'order by' - def produce(self, node): - if node is None: - self.sql = '' - return - elif type(node) is not list: - node = [node] - - o_list = [] - - for o in node: - o_str = expr(self, o['value']).sql - if 'sort' in o and f'{o["sort"]}'.lower() == 'desc': - o_str += ' ' + 'DESC' - o_list.append(o_str) - self.add(', '.join(o_list)) - - -class groupby(orderby): - name = 'group by' - - -class join(ast_node): - name = 'join' - def init(self, _): - self.joins:list = [] - self.tables = [] - self.tables_dir = dict() - # self.tmp_name = 'join_' + base62uuid(4) - # self.datasource = TableInfo(self.tmp_name, [], self.context) - def append(self, tbls, __alias = ''): - alias = lambda t : '(' + t + ') ' + __alias if len(__alias) else t - if type(tbls) is join: - self.joins.append(alias(tbls.__str__())) - self.tables += tbls.tables - self.tables_dir = {**self.tables_dir, **tbls.tables_dir} - - elif type(tbls) is TableInfo: - self.joins.append(alias(tbls.table_name)) - self.tables.append(tbls) - self.tables_dir[tbls.table_name] = tbls - for a in tbls.alias: - self.tables_dir[a] = tbls - - elif type(tbls) is projection: - self.joins.append(alias(tbls.finalize())) - - def produce(self, node): - if type(node) is list: - for d in node: - self.append(join(self, d).__str__()) - - elif type(node) is dict: - alias = '' - if 'value' in node: - table_name = node['value'] - tbl = None - if 'name' in node: - alias = node['name'] - if type(table_name) is dict: - if 'select' in table_name: - # TODO: subquery, create and register TableInfo in projection - tbl = projection(self, table_name).finalize() - else: - tbl = self.context.tables_byname[table_name] - if 'name' in node: - tbl.add_alias(node['name']) - - self.append(tbl, alias) - else: - keys = node.keys() - if keys[0].lower().endswith('join'): - j = join(self, node[keys[0]]) - tablename = f' {keys[0]} {j}' - if keys[1].lower() == 'on': - tablename += f' on {expr(self, node[keys[1]])}' - self.joins.append(tablename) - self.tables += j.tables - self.tables_dir = {**self.tables_dir, **j.tables_dir} - - elif type(node) is str: - self.append(self.context.tables_byname[node]) - - def get_cols(self, colExpr: str) -> ColRef: - for t in self.tables: - if colExpr in t.columns_byname: - return t.columns_byname[colExpr] - - def parse_col_names(self, colExpr:str) -> ColRef: - parsedColExpr = colExpr.split('.') - if len(parsedColExpr) <= 1: - return self.get_cols(colExpr) - else: - datasource = self.tables_dir[parsedColExpr[0]] - if datasource is None: - raise ValueError(f'Table name/alias not defined{parsedColExpr[0]}') - else: - return datasource.parse_col_names(parsedColExpr[1]) - - def consume(self, _): - self.sql = ', '.join(self.joins) - return super().consume(_) - def __str__(self): - return ', '.join(self.joins) - def __repr__(self): - return self.__str__() - - -class filter(ast_node): - name = 'where' - def produce(self, node): - self.add(expr(self, node).sql) - - -class create_table(ast_node): - name = 'create_table' - first_order = name - def init(self, node): - self.sql = 'CREATE TABLE ' - - def produce(self, node): - ct = node[self.name] - tbl = self.context.add_table(ct['name'], ct['columns']) - self.sql = f'CREATE TABLE {tbl.table_name}(' - columns = [] - for c in tbl.columns: - columns.append(f'{c.name} {c.type.upper()}') - self.sql += ', '.join(columns) - self.sql += ')' - - - -class insert(ast_node): - name = 'insert' - first_order = name - def produce(self, node): - values = node['query']['select'] - tbl = node['insert'] - self.sql = f'INSERT INTO {tbl} VALUES(' - # if len(values) != table.n_cols: - # raise ValueError("Column Mismatch") - list_values = [] - for i, s in enumerate(values): - if 'value' in s: - list_values.append(f"{s['value']}") - else: - # subquery, dispatch to select astnode - pass - self.sql += ', '.join(list_values) + ')' - - -class load(ast_node): - name="load" - first_order = name - def produce(self, node): - node = node['load'] - s1 = 'LOAD DATA INFILE ' - s2 = 'INTO TABLE ' - s3 = 'FIELDS TERMINATED BY ' - self.sql = f'{s1} \"{node["file"]["literal"]}\" {s2} {node["table"]}' - if 'term' in node: - self.sql += f' {s3} \"{node["term"]["literal"]}\"' - - -class outfile(ast_node): - name="_outfile" - def produce(self, node): - filename = node['loc']['literal'] if 'loc' in node else node['literal'] - self.sql = f'INTO OUTFILE "{filename}"' - if 'term' in node: - self.sql += f' FIELDS TERMINATED BY \"{node["term"]["literal"]}\"' - - -def include(objs): - import inspect - for _, cls in inspect.getmembers(objs): - if inspect.isclass(cls) and issubclass(cls, ast_node) and type(cls.first_order) is str: - ast_node.types[cls.first_order] = cls - - -import sys +from engine.utils import enlist, base62uuid, base62alp +from reconstruct.storage import Context, TableInfo, ColRef + +class ast_node: + header = [] + types = dict() + first_order = False + + def __init__(self, parent:"ast_node", node, context:Context = None): + self.context = parent.context if context is None else context + self.parent = parent + self.sql = '' + self.datasource = None + self.init(node) + self.produce(node) + self.spawn(node) + self.consume(node) + + def emit(self, code): + self.context.emit(code) + def add(self, code): + self.sql += code + ' ' + + name = 'null' + + def init(self, _): + self.add(self.__class__.name.upper()) + def produce(self, _): + pass + def spawn(self, _): + pass + + def consume(self, _): + if self.parent is None: + self.emit(self.sql+';\n') + + +from reconstruct.expr import expr + + +class projection(ast_node): + name = 'projection' + first_order = 'select' + + def init(self, _): + pass + def produce(self, node): + p = node['select'] + self.projections = p if type(p) is list else [p] + self.add('SELECT') + + def spawn(self, node): + self.datasource = None # datasource is Join instead of TableInfo + if 'from' in node: + from_clause = node['from'] + self.datasource = join(self, from_clause) + if 'assumptions' in from_clause: + self.assumptions = enlist(from_clause['assumptions']) + + if self.datasource is not None: + self.datasource_changed = True + self.prev_datasource = self.context.datasource + self.context.datasource = self.datasource + + if 'where' in node: + self.where = filter(self, node['where']) + else: + self.where = None + + if 'groupby' in node: + self.group_node = groupby(self, node['groupby']) + else: + self.group_node = None + + def consume(self, node): + # deal with projections + self.out_table = TableInfo('out_'+base62uuid(4), [], self.context) + cols = [] + col_exprs = [] + for i, proj in enumerate(self.projections): + compound = False + self.datasource.rec = set() + name = '' + if type(proj) is dict: + + if 'value' in proj: + e = proj['value'] + name = expr(self, e).sql + disp_name = ''.join([a if a in base62alp else '' for a in name]) + compound = True # compound column + if 'name' in proj: # renaming column by AS keyword + name += ' ' + proj['name'] + col_exprs.append(name) + + elif type(proj) is str: + col = self.datasource.get_col(proj) + name = col.name + self.datasource.rec = None + # TODO: Type deduction in Python + cols.append(ColRef('unknown', self.out_table, None, disp_name, i, compound=compound)) + self.add(', '.join(col_exprs)) + + def finialize(astnode:ast_node): + if(astnode is not None): + self.add(astnode.sql) + self.add('FROM') + finialize(self.datasource) + finialize(self.where) + finialize(self.group_node) + if 'orderby' in node: + self.add(orderby(self, node['orderby']).sql) + if 'outfile' in node: + self.add(outfile(self, node['outfile']).sql) + if self.parent is None: + self.emit(self.sql+';\n') + else: + # TODO: subquery, name create tmp-table from subquery w/ alias as name + pass + + +class orderby(ast_node): + name = 'order by' + def produce(self, node): + if node is None: + self.sql = '' + return + elif type(node) is not list: + node = [node] + + o_list = [] + + for o in node: + o_str = expr(self, o['value']).sql + if 'sort' in o and f'{o["sort"]}'.lower() == 'desc': + o_str += ' ' + 'DESC' + o_list.append(o_str) + self.add(', '.join(o_list)) + + +class groupby(orderby): + name = 'group by' + + +class join(ast_node): + name = 'join' + def init(self, _): + self.joins:list = [] + self.tables = [] + self.tables_dir = dict() + # self.tmp_name = 'join_' + base62uuid(4) + # self.datasource = TableInfo(self.tmp_name, [], self.context) + def append(self, tbls, __alias = ''): + alias = lambda t : '(' + t + ') ' + __alias if len(__alias) else t + if type(tbls) is join: + self.joins.append(alias(tbls.__str__())) + self.tables += tbls.tables + self.tables_dir = {**self.tables_dir, **tbls.tables_dir} + + elif type(tbls) is TableInfo: + self.joins.append(alias(tbls.table_name)) + self.tables.append(tbls) + self.tables_dir[tbls.table_name] = tbls + for a in tbls.alias: + self.tables_dir[a] = tbls + + elif type(tbls) is projection: + self.joins.append(alias(tbls.finalize())) + + def produce(self, node): + if type(node) is list: + for d in node: + self.append(join(self, d).__str__()) + + elif type(node) is dict: + alias = '' + if 'value' in node: + table_name = node['value'] + tbl = None + if 'name' in node: + alias = node['name'] + if type(table_name) is dict: + if 'select' in table_name: + # TODO: subquery, create and register TableInfo in projection + tbl = projection(self, table_name).finalize() + else: + tbl = self.context.tables_byname[table_name] + if 'name' in node: + tbl.add_alias(node['name']) + + self.append(tbl, alias) + else: + keys = node.keys() + if keys[0].lower().endswith('join'): + j = join(self, node[keys[0]]) + tablename = f' {keys[0]} {j}' + if keys[1].lower() == 'on': + tablename += f' on {expr(self, node[keys[1]])}' + self.joins.append(tablename) + self.tables += j.tables + self.tables_dir = {**self.tables_dir, **j.tables_dir} + + elif type(node) is str: + self.append(self.context.tables_byname[node]) + + def get_cols(self, colExpr: str) -> ColRef: + for t in self.tables: + if colExpr in t.columns_byname: + return t.columns_byname[colExpr] + + def parse_col_names(self, colExpr:str) -> ColRef: + parsedColExpr = colExpr.split('.') + if len(parsedColExpr) <= 1: + return self.get_cols(colExpr) + else: + datasource = self.tables_dir[parsedColExpr[0]] + if datasource is None: + raise ValueError(f'Table name/alias not defined{parsedColExpr[0]}') + else: + return datasource.parse_col_names(parsedColExpr[1]) + + def consume(self, _): + self.sql = ', '.join(self.joins) + return super().consume(_) + def __str__(self): + return ', '.join(self.joins) + def __repr__(self): + return self.__str__() + + +class filter(ast_node): + name = 'where' + def produce(self, node): + self.add(expr(self, node).sql) + + +class create_table(ast_node): + name = 'create_table' + first_order = name + def init(self, node): + self.sql = 'CREATE TABLE ' + + def produce(self, node): + ct = node[self.name] + tbl = self.context.add_table(ct['name'], ct['columns']) + self.sql = f'CREATE TABLE {tbl.table_name}(' + columns = [] + for c in tbl.columns: + columns.append(f'{c.name} {c.type.upper()}') + self.sql += ', '.join(columns) + self.sql += ')' + + + +class insert(ast_node): + name = 'insert' + first_order = name + def produce(self, node): + values = node['query']['select'] + tbl = node['insert'] + self.sql = f'INSERT INTO {tbl} VALUES(' + # if len(values) != table.n_cols: + # raise ValueError("Column Mismatch") + list_values = [] + for i, s in enumerate(values): + if 'value' in s: + list_values.append(f"{s['value']}") + else: + # subquery, dispatch to select astnode + pass + self.sql += ', '.join(list_values) + ')' + + +class load(ast_node): + name="load" + first_order = name + def produce(self, node): + node = node['load'] + s1 = 'LOAD DATA INFILE ' + s2 = 'INTO TABLE ' + s3 = 'FIELDS TERMINATED BY ' + self.sql = f'{s1} \"{node["file"]["literal"]}\" {s2} {node["table"]}' + if 'term' in node: + self.sql += f' {s3} \"{node["term"]["literal"]}\"' + + +class outfile(ast_node): + name="_outfile" + def produce(self, node): + filename = node['loc']['literal'] if 'loc' in node else node['literal'] + self.sql = f'INTO OUTFILE "{filename}"' + if 'term' in node: + self.sql += f' FIELDS TERMINATED BY \"{node["term"]["literal"]}\"' + + +def include(objs): + import inspect + for _, cls in inspect.getmembers(objs): + if inspect.isclass(cls) and issubclass(cls, ast_node) and type(cls.first_order) is str: + ast_node.types[cls.first_order] = cls + + +import sys include(sys.modules[__name__]) \ No newline at end of file diff --git a/reconstruct/expr.py b/reconstruct/expr.py index 9b24a64..4de2a24 100644 --- a/reconstruct/expr.py +++ b/reconstruct/expr.py @@ -1,128 +1,128 @@ -from reconstruct.ast import ast_node -from reconstruct.storage import ColRef, TableInfo - - -class expr(ast_node): - name='expr' - builtin_func_maps = { - 'max': 'MAX', - 'min': 'MIN', - 'avg': 'AVG', - 'sum': 'SUM', - 'count' : 'COUNT', - 'mins': ['mins', 'minw'], - 'maxs': ['maxs', 'maxw'], - 'avgs': ['avgs', 'avgw'], - 'sums': ['sums', 'sumw'], - } - - binary_ops = { - 'sub':'-', - 'add':'+', - 'mul':'*', - 'div':'/', - 'mod':'%', - 'and':' AND ', - 'or':' OR ', - 'xor' : ' XOR ', - 'gt':'>', - 'lt':'<', - 'le':'<=', - 'gt':'>=' - } - - compound_ops = { - } - - unary_ops = { - 'neg' : '-', - 'not' : ' NOT ' - } - - coumpound_generating_ops = ['avgs', 'mins', 'maxs', 'sums'] + \ - list(binary_ops.keys()) + list(compound_ops.keys()) + list(unary_ops.keys() ) - - def __init__(self, parent, node): - self.raw_col = None - self.inside_agg = False - if(type(parent) is expr): - self.inside_agg = parent.inside_agg - ast_node.__init__(self, parent, node, None) - - def init(self, _): - from engine.projection import projection - parent = self.parent - self.isvector = parent.isvector if type(parent) is expr else False - self.is_compound = parent.is_compound if type(parent) is expr else False - if type(parent) in [projection, expr]: - self.datasource = parent.datasource - else: - self.datasource = self.context.datasource - self.udf_map = parent.context.udf_map - self.func_maps = {**self.udf_map, **self.builtin_func_maps} - - def produce(self, node): - if type(node) is dict: - for key, val in node.items(): - if key in self.func_maps: - # TODO: distinguish between UDF agg functions and other UDF functions. - self.inside_agg = True - if type(val) is list and len(val) > 1: - cfunc = self.func_maps[key] - cfunc = cfunc[len(val) - 1] if type(cfunc) is list else cfunc - self.sql += f"{cfunc}(" - for i, p in enumerate(val): - self.sql += expr(self, p).sql + (',' if i < len(val) - 1 else '') - else: - funcname = self.func_maps[key] - funcname = funcname[0] if type(funcname) is list else funcname - self.sql += f"{funcname}(" - self.sql += expr(self, val).sql - self.sql += ')' - self.inside_agg = False - elif key in self.binary_ops: - l = expr(self, val[0]).sql - r = expr(self, val[1]).sql - self.sql += f'({l}{self.binary_ops[key]}{r})' - elif key in self.compound_ops: - x = [] - if type(val) is list: - for v in val: - x.append(expr(self, v).sql) - self.sql = self.compound_ops[key][1](x) - elif key in self.unary_ops: - self.sql += f'{self.unary_ops[key]}({expr(self, val).sql})' - else: - print(f'Undefined expr: {key}{val}') - - if key in self.coumpound_generating_ops and not self.is_compound: - self.is_compound = True - p = self.parent - while type(p) is expr and not p.is_compound: - p.is_compound = True - p = p.parent - - elif type(node) is str: - p = self.parent - while type(p) is expr and not p.isvector: - p.isvector = True - p = p.parent - - self.raw_col = self.datasource.parse_col_names(node) - self.raw_col = self.raw_col if type(self.raw_col) is ColRef else None - if self.raw_col is not None: - self.sql = self.raw_col.name - else: - self.sql = node - - elif type(node) is bool: - self.sql = '1' if node else '0' - else: - self.sql = f'{node}' - - def __str__(self): - return self.sql - def __repr__(self): - return self.__str__() - +from reconstruct.ast import ast_node +from reconstruct.storage import ColRef, TableInfo + + +class expr(ast_node): + name='expr' + builtin_func_maps = { + 'max': 'MAX', + 'min': 'MIN', + 'avg': 'AVG', + 'sum': 'SUM', + 'count' : 'COUNT', + 'mins': ['mins', 'minw'], + 'maxs': ['maxs', 'maxw'], + 'avgs': ['avgs', 'avgw'], + 'sums': ['sums', 'sumw'], + } + + binary_ops = { + 'sub':'-', + 'add':'+', + 'mul':'*', + 'div':'/', + 'mod':'%', + 'and':' AND ', + 'or':' OR ', + 'xor' : ' XOR ', + 'gt':'>', + 'lt':'<', + 'le':'<=', + 'gt':'>=' + } + + compound_ops = { + } + + unary_ops = { + 'neg' : '-', + 'not' : ' NOT ' + } + + coumpound_generating_ops = ['avgs', 'mins', 'maxs', 'sums'] + \ + list(binary_ops.keys()) + list(compound_ops.keys()) + list(unary_ops.keys() ) + + def __init__(self, parent, node): + self.raw_col = None + self.inside_agg = False + if(type(parent) is expr): + self.inside_agg = parent.inside_agg + ast_node.__init__(self, parent, node, None) + + def init(self, _): + from engine.projection import projection + parent = self.parent + self.isvector = parent.isvector if type(parent) is expr else False + self.is_compound = parent.is_compound if type(parent) is expr else False + if type(parent) in [projection, expr]: + self.datasource = parent.datasource + else: + self.datasource = self.context.datasource + self.udf_map = parent.context.udf_map + self.func_maps = {**self.udf_map, **self.builtin_func_maps} + + def produce(self, node): + if type(node) is dict: + for key, val in node.items(): + if key in self.func_maps: + # TODO: distinguish between UDF agg functions and other UDF functions. + self.inside_agg = True + if type(val) is list and len(val) > 1: + cfunc = self.func_maps[key] + cfunc = cfunc[len(val) - 1] if type(cfunc) is list else cfunc + self.sql += f"{cfunc}(" + for i, p in enumerate(val): + self.sql += expr(self, p).sql + (',' if i < len(val) - 1 else '') + else: + funcname = self.func_maps[key] + funcname = funcname[0] if type(funcname) is list else funcname + self.sql += f"{funcname}(" + self.sql += expr(self, val).sql + self.sql += ')' + self.inside_agg = False + elif key in self.binary_ops: + l = expr(self, val[0]).sql + r = expr(self, val[1]).sql + self.sql += f'({l}{self.binary_ops[key]}{r})' + elif key in self.compound_ops: + x = [] + if type(val) is list: + for v in val: + x.append(expr(self, v).sql) + self.sql = self.compound_ops[key][1](x) + elif key in self.unary_ops: + self.sql += f'{self.unary_ops[key]}({expr(self, val).sql})' + else: + print(f'Undefined expr: {key}{val}') + + if key in self.coumpound_generating_ops and not self.is_compound: + self.is_compound = True + p = self.parent + while type(p) is expr and not p.is_compound: + p.is_compound = True + p = p.parent + + elif type(node) is str: + p = self.parent + while type(p) is expr and not p.isvector: + p.isvector = True + p = p.parent + + self.raw_col = self.datasource.parse_col_names(node) + self.raw_col = self.raw_col if type(self.raw_col) is ColRef else None + if self.raw_col is not None: + self.sql = self.raw_col.name + else: + self.sql = node + + elif type(node) is bool: + self.sql = '1' if node else '0' + else: + self.sql = f'{node}' + + def __str__(self): + return self.sql + def __repr__(self): + return self.__str__() + \ No newline at end of file diff --git a/reconstruct/storage.py b/reconstruct/storage.py index 37d6ccd..44ed001 100644 --- a/reconstruct/storage.py +++ b/reconstruct/storage.py @@ -1,89 +1,89 @@ -class ColRef: - def __init__(self, _ty, cobj, table:'TableInfo', name, id, compound = False): - self.type = _ty - self.cobj = cobj - self.table = table - self.name = name - self.alias = set() - self.id = id # position in table - self.compound = compound # compound field (list as a field) - # e.g. order by, group by, filter by expressions - - self.__arr__ = (_ty, cobj, table, name, id) - def __getitem__(self, key): - if type(key) is str: - return getattr(self, key) - else: - return self.__arr__[key] - - def __setitem__(self, key, value): - self.__arr__[key] = value - - -class TableInfo: - def __init__(self, table_name, cols, cxt:'Context'): - # statics - self.table_name = table_name - self.alias = set([table_name]) - self.columns_byname = dict() # column_name, type - self.columns = [] - self.cxt = cxt - # keep track of temp vars - self.rec = None - self.add_cols(cols) - # runtime - self.order = [] # assumptions - - cxt.tables_byname[self.table_name] = self # construct reverse map - - def add_cols(self, cols, new = True): - for i, c in enumerate(cols): - self.add_col(c, new, i) - def add_col(self, c, new = True, i = 0): - _ty = c['type'] - if new: - _ty = _ty if type(c) is ColRef else list(_ty.keys())[0] - col_object = ColRef(_ty, c, self, c['name'], len(self.columns)) - else: - col_object = c - c.table = self - self.columns_byname[c['name']] = col_object - self.columns.append(col_object) - - def add_alias(self, alias): - if alias in self.cxt.tables_byname.keys(): - print("Error: table alias already exists") - return - self.cxt.tables_byname[alias] = self - self.alias.add(alias) - def parse_col_names(self, colExpr) -> ColRef: - parsedColExpr = colExpr.split('.') - if len(parsedColExpr) <= 1: - return self.columns_byname[colExpr] - else: - datasource = self.cxt.tables_byname[parsedColExpr[0]] - if datasource is None: - raise ValueError(f'Table name/alias not defined{parsedColExpr[0]}') - else: - return datasource.parse_col_names(parsedColExpr[1]) - - -class Context: - def __init__(self): - self.sql = '' - self.tables_byname = dict() - self.col_byname = dict() - self.tables = [] - self.cols = [] - self.datasource = None - self.udf_map = dict() - - def emit(self, sql:str): - self.sql += sql + ' ' - - def add_table(self, table_name, cols): - tbl = TableInfo(table_name, cols, self) - self.tables.append(tbl) - return tbl - +class ColRef: + def __init__(self, _ty, cobj, table:'TableInfo', name, id, compound = False): + self.type = _ty + self.cobj = cobj + self.table = table + self.name = name + self.alias = set() + self.id = id # position in table + self.compound = compound # compound field (list as a field) + # e.g. order by, group by, filter by expressions + + self.__arr__ = (_ty, cobj, table, name, id) + def __getitem__(self, key): + if type(key) is str: + return getattr(self, key) + else: + return self.__arr__[key] + + def __setitem__(self, key, value): + self.__arr__[key] = value + + +class TableInfo: + def __init__(self, table_name, cols, cxt:'Context'): + # statics + self.table_name = table_name + self.alias = set([table_name]) + self.columns_byname = dict() # column_name, type + self.columns = [] + self.cxt = cxt + # keep track of temp vars + self.rec = None + self.add_cols(cols) + # runtime + self.order = [] # assumptions + + cxt.tables_byname[self.table_name] = self # construct reverse map + + def add_cols(self, cols, new = True): + for i, c in enumerate(cols): + self.add_col(c, new, i) + def add_col(self, c, new = True, i = 0): + _ty = c['type'] + if new: + _ty = _ty if type(c) is ColRef else list(_ty.keys())[0] + col_object = ColRef(_ty, c, self, c['name'], len(self.columns)) + else: + col_object = c + c.table = self + self.columns_byname[c['name']] = col_object + self.columns.append(col_object) + + def add_alias(self, alias): + if alias in self.cxt.tables_byname.keys(): + print("Error: table alias already exists") + return + self.cxt.tables_byname[alias] = self + self.alias.add(alias) + def parse_col_names(self, colExpr) -> ColRef: + parsedColExpr = colExpr.split('.') + if len(parsedColExpr) <= 1: + return self.columns_byname[colExpr] + else: + datasource = self.cxt.tables_byname[parsedColExpr[0]] + if datasource is None: + raise ValueError(f'Table name/alias not defined{parsedColExpr[0]}') + else: + return datasource.parse_col_names(parsedColExpr[1]) + + +class Context: + def __init__(self): + self.sql = '' + self.tables_byname = dict() + self.col_byname = dict() + self.tables = [] + self.cols = [] + self.datasource = None + self.udf_map = dict() + + def emit(self, sql:str): + self.sql += sql + ' ' + + def add_table(self, table_name, cols): + tbl = TableInfo(table_name, cols, self) + self.tables.append(tbl) + return tbl + \ No newline at end of file diff --git a/server.exe b/server.exe new file mode 100644 index 0000000..8658fd9 Binary files /dev/null and b/server.exe differ diff --git a/server/.claudiaideconfig b/server/.claudiaideconfig deleted file mode 100644 index 3da1995..0000000 --- a/server/.claudiaideconfig +++ /dev/null @@ -1,27 +0,0 @@ -{ - "BackgroundImageAbsolutePath": "c:\\users\\bill\\appdata\\local\\microsoft\\visualstudio\\17.0_03c65567\\extensions\\atkxhose.05t\\Images\\background.png", - "BackgroundImagesDirectoryAbsolutePath": "c:\\users\\bill\\appdata\\local\\microsoft\\visualstudio\\17.0_03c65567\\extensions\\atkxhose.05t\\Images", - "ExpandToIDE": false, - "Extensions": ".png, .jpg, .gif, .bmp", - "ImageBackgroundType": 0, - "ImageFadeAnimationInterval": "PT5S", - "ImageStretch": 0, - "IsLimitToMainlyEditorWindow": false, - "LoopSlideshow": true, - "MaxHeight": 0, - "MaxWidth": 0, - "Opacity": 0.35, - "PositionHorizon": 1, - "PositionVertical": 1, - "ShuffleSlideshow": false, - "SoftEdgeX": 0, - "SoftEdgeY": 0, - "TileMode": 0, - "UpdateImageInterval": "PT1M", - "ViewBoxPointX": 0, - "ViewBoxPointY": 0, - "ViewPortHeight": 1, - "ViewPortPointX": 0, - "ViewPortPointY": 0, - "ViewPortWidth": 1 -} \ No newline at end of file diff --git a/server/libaquery.h b/server/libaquery.h index 57ac4e1..b40769c 100644 --- a/server/libaquery.h +++ b/server/libaquery.h @@ -10,10 +10,21 @@ enum Log_level { LOG_SILENT }; +struct Config{ + int running, new_query, server_mode, n_buffers; + int buffer_sizes[]; +}; + struct Context{ typedef int (*printf_type) (const char *format, ...); std::unordered_map tables; std::unordered_map cols; + + Config* cfg; + + int n_buffers, *sz_bufs; + void **buffers; + Log_level log_level = LOG_SILENT; printf_type print = printf; template @@ -28,8 +39,8 @@ struct Context{ } }; -#ifdef _MSC_VER -#define __DLLEXPORT__ __declspec(dllexport) __stdcall +#ifdef _WIN32 +#define __DLLEXPORT__ __declspec(dllexport) __stdcall #else #define __DLLEXPORT__ #endif diff --git a/server/server.cpp b/server/server.cpp index 8f23e66..350035c 100644 --- a/server/server.cpp +++ b/server/server.cpp @@ -36,13 +36,43 @@ void daemon(thread_context* c) { #include "aggregations.h" typedef int (*code_snippet)(void*); int _main(); -int main(int argc, char** argv) { + +int dll_main(int argc, char** argv, Context* cxt){ + Config *cfg = reinterpret_cast(argv[0]); + + auto buf_szs = cfg->buffer_sizes; + void** buffers = (void**)malloc(sizeof(void*) * cfg->n_buffers); + for (int i = 0; i < cfg->n_buffers; i++) + buffers[i] = static_cast(argv[i + 1]); + + cxt->buffers = buffers; + cxt->cfg = cfg; + cxt->n_buffers = cfg->n_buffers; + cxt->sz_bufs = buf_szs; + + while(cfg->running){ + if (cfg->new_query) { + void* handle = dlopen("d:/gg/Aquery++/dll.so", RTLD_LAZY); + code_snippet c = reinterpret_cast(dlsym(handle, "dllmain")); + c(cxt); + dlclose(handle); + cfg->new_query = 0; + } + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + } + + return 0; +} + +extern "C" int __DLLEXPORT__ main(int argc, char** argv) { Context* cxt = new Context(); cxt->log("%d %s\n", argc, argv[1]); const char* shmname; - if (argc <= 1) - return _main(); + if (argc < 0) + return dll_main(argc, argv, cxt); + else if (argc <= 1) + return test_main(); else shmname = argv[1]; SharedMemory shm = SharedMemory(shmname); @@ -68,7 +98,6 @@ int main(int argc, char** argv) { cxt->log("inner\n"); cxt->err("return: %d\n", c(cxt)); } - dlclose(handle); } ready = false; } @@ -77,7 +106,7 @@ int main(int argc, char** argv) { return 0; } #include "utils.h" -int _main() +int test_main() { //vector_type t; //t = 1;