From ae51d8c622aa2ae1eb7f5561302c252a66b76724 Mon Sep 17 00:00:00 2001 From: bill sun Date: Thu, 14 Apr 2022 03:16:29 +0800 Subject: [PATCH] bug fix --- README.md | 4 +- engine/ast.py | 3 - engine/scan.py | 69 +---------- out.cpp | 26 ++--- prompt.py | 286 +++++++++++++++++++++++----------------------- q1.sql | 20 ++-- server/server.cpp | 2 +- 7 files changed, 171 insertions(+), 239 deletions(-) diff --git a/README.md b/README.md index 099ba68..f1c753b 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ -# AQuery Compiler +# AQuery++ -AQuery Compiler that compiles AQuery into C++11. +AQuery++ Compiler that compiles AQuery into C++11. Frontend built on top of [mo-sql-parsing](https://github.com/klahnakoski/mo-sql-parsing). ## Roadmap diff --git a/engine/ast.py b/engine/ast.py index 0c2d257..85cdd24 100644 --- a/engine/ast.py +++ b/engine/ast.py @@ -111,9 +111,6 @@ class TableInfo: self.columns_byname[c['name']] = col_object self.columns.append(col_object) - def construct(self): - for c in self.columns: - self.cxt.emit(f'{c.cname}:()') @property def n_cols(self): return len(self.columns) diff --git a/engine/scan.py b/engine/scan.py index f567829..4766d7a 100644 --- a/engine/scan.py +++ b/engine/scan.py @@ -48,6 +48,7 @@ class filter(ast_node): def spawn(self, node): # TODO: deal with subqueries + self.modified_node = node return super().spawn(node) def __materialize__(self): if self.materialize: @@ -66,72 +67,6 @@ class filter(ast_node): def consume(self, node): # TODO: optimizations after converting expr to cnf - if type(node) is bool and node and self.materialize: - self.output = self.context.datasource if node else None - self.value = '1' if node else '0' - else: - if type(node) is dict: - def short_circuit(op, idx, inv = True): - v = filter(self, node[op][idx]).value - inv_filter = lambda x: not x if inv else x - if type(v) is bool and inv_filter(v): - self.value = inv_filter(v) - self.__materialize__() - return None - return v - def binary(l, r, _ty = '&'): - if type(l) is bool: - self.value = r - elif type(r) is bool: - self.value = l - elif type(l) is View: - if type(r) is View: - self.emit(f"{l.name}: {l.name} {_ty} {r.name if type(r) is View else f'({r})'}") - self.value = l - elif type(l) is str: - if type(r) is str: - self.value = f'({l}){_ty}({r})' - else: - self.emit(f'{r.name}:{r.name} {_ty} ({l})') - self.value = r - if 'and' in node: - l = short_circuit('and', 0) - if l is not None: - r = short_circuit('and', 1) - if r is not None: - binary(l, r) - - elif 'or' in node: - l = short_circuit('or', 0, False) - if l is not None: - r = short_circuit('or', 1, False) - if r is not None: - binary(l, r, '|') - - elif 'not' in node: - v = filter(self, node['not']).value - if type(v) is bool: - self.value = not v - self.__materialize__() - elif type(v) is View: - if len(v.table.columns) > 0: - all_rows = View(self.context, v.table) - self.emit(f'{all_rows.name}:(#{v.table.columns[0].cname})#1') - self.emit(f'{v.name}:{all_rows.name}-{v.name}') - self.value = v - else: - self.value = '~(' + v + ')' - # TODO: arithmetic ops connecting logical ops. - else: - e = expr(self, node) - if e.isvector: - v = View(self.context, self.datasource) - v.construct() - self.emit(f'{v.name}:{e.cexpr}') - self.value = v - else: - self.value = e.cexpr - self.__materialize__() - + self.expr = expr(self, self.modified_node) print(node) \ No newline at end of file diff --git a/out.cpp b/out.cpp index 6c5b5f5..8e4f407 100644 --- a/out.cpp +++ b/out.cpp @@ -1,5 +1,5 @@ -#include "./server/libaquery.h" #include "./server/aggregations.h" +#include "./server/libaquery.h" extern "C" int __DLLEXPORT__ dllmain(Context* cxt) { using namespace std; @@ -42,19 +42,19 @@ stocks_timestamp.emplace_back(15); stocks_price.emplace_back(2); stocks_timestamp.emplace_back(16); stocks_price.emplace_back(5); -auto out_6gPn = new TableInfo>("out_6gPn", 1); -cxt->tables.insert({"out_6gPn", out_6gPn}); -auto& out_6gPn_maxstockspriceminstockstimestamp = *(ColRef> *)(&out_6gPn->colrefs[0]); -out_6gPn_maxstockspriceminstockstimestamp.init(); -out_6gPn_maxstockspriceminstockstimestamp = max((stocks_price-min(stocks_timestamp))); -print(*out_6gPn); +auto out_ZPYh = new TableInfo>("out_ZPYh", 1); +cxt->tables.insert({"out_ZPYh", out_ZPYh}); +auto& out_ZPYh_maxstockspriceminstockstimestamp = *(ColRef> *)(&out_ZPYh->colrefs[0]); +out_ZPYh_maxstockspriceminstockstimestamp.init(); +out_ZPYh_maxstockspriceminstockstimestamp = max((stocks_price-min(stocks_timestamp))); +print(*out_ZPYh); -auto out_7a2d = new TableInfo>("out_7a2d", 1); -cxt->tables.insert({"out_7a2d", out_7a2d}); -auto& out_7a2d_maxstockspriceminsstocksprice = *(ColRef> *)(&out_7a2d->colrefs[0]); -out_7a2d_maxstockspriceminsstocksprice.init(); -out_7a2d_maxstockspriceminsstocksprice = max((stocks_price-mins(stocks_price))); -print(*out_7a2d); +auto out_1ac3 = new TableInfo>("out_1ac3", 1); +cxt->tables.insert({"out_1ac3", out_1ac3}); +auto& out_1ac3_maxstockspriceminsstocksprice = *(ColRef> *)(&out_1ac3->colrefs[0]); +out_1ac3_maxstockspriceminsstocksprice.init(); +out_1ac3_maxstockspriceminsstocksprice = max((stocks_price-mins(stocks_price))); +print(*out_1ac3); return 0; } \ No newline at end of file diff --git a/prompt.py b/prompt.py index 84607ae..9333997 100644 --- a/prompt.py +++ b/prompt.py @@ -1,144 +1,144 @@ -from concurrent.futures import thread -import re -import time -import aquery_parser as parser -import engine -import subprocess -import mmap -import sys -import os -from engine.utils import base62uuid -import atexit -try: - os.remove('server.bin') -except Exception as e: - print(type(e), e) - -subprocess.call(['make', 'server.bin']) -cleanup = True -def rm(): - if cleanup: - mm.seek(0,os.SEEK_SET) - mm.write(b'\x00\x00') - mm.flush() - time.sleep(.001) - server.kill() - files = os.listdir('.') - for f in files: - if f.endswith('.shm'): - os.remove(f) - mm.close() - cleanup = False - -def proc_alive(pid): - try: - os.kill(pid, 0) - except OSError: - return False - else: - return True - -atexit.register(rm) - -shm = base62uuid() -if sys.platform != 'win32': - import readline - shm += '.shm' - basecmd = ['bash', '-c', 'rlwrap k'] - mm = None - if not os.path.isfile(shm): - # create initial file - with open(shm, "w+b") as handle: - handle.write(b'\x01\x00') # [running, new job] - handle.flush() - mm = mmap.mmap(handle.fileno(), 2, access=mmap.ACCESS_WRITE, offset=0) - if mm is None: - exit(1) -else: - basecmd = ['bash.exe', '-c', 'rlwrap ./k'] - mm = mmap.mmap(0, 2, shm) - mm.write(b'\x01\x00') - mm.flush() -server = subprocess.Popen(["./server.bin", shm]) - -test_parser = True - -# code to test parser -ws = re.compile(r'\s+') - -q = 'SELECT p.Name, v.Name FROM Production.Product p JOIN Purchasing.ProductVendor pv ON p.ProductID = pv.ProductID JOIN Purchasing.Vendor v ON pv.BusinessEntityID = v.BusinessEntityID WHERE ProductSubcategoryID = 15 ORDER BY v.Name;' - -res = parser.parse(q) - -print(res) - - -# else:f -# if subprocess.call(['make', 'snippet']) == 0: -# mm.seek(0) -# mm.write(b'\x01\x01') -# time.sleep(.1) -# mm.seek(0) -# print(mm.read(2)) - -# mm.close() -# handle.close() -# os.remove(shm) -# exit() -keep = False -cxt = None -while test_parser: - try: - if not proc_alive(server.pid): - server = subprocess.Popen(["./server.bin", shm]) - q = input() - if q == 'exec': - if not keep or cxt is None: - cxt = engine.initialize() - stmts_stmts = stmts['stmts'] - if type(stmts_stmts) is list: - for s in stmts_stmts: - engine.generate(s, cxt) - else: - engine.generate(stmts_stmts, cxt) - print(cxt.ccode) - with open('out.cpp', 'wb') as outfile: - outfile.write((cxt.finalize()).encode('utf-8')) - if subprocess.call(['make', 'snippet']) == 0: - mm.seek(0,os.SEEK_SET) - mm.write(b'\x01\x01') - continue - elif q == 'k': - subprocess.call(basecmd) - continue - elif q == 'print': - print(stmts) - continue - elif q == 'keep': - keep = not keep - continue - elif q=='format' or q == 'fmt': - subprocess.call(['clang-format', 'out.cpp']) - elif q == 'exit': - break - elif q == 'r': - if subprocess.call(['make', 'snippet']) == 0: - mm.seek(0,os.SEEK_SET) - mm.write(b'\x01\x01') - continue - trimed = ws.sub(' ', q.lower()).split(' ') - if trimed[0].startswith('f'): - fn = 'stock.a' if len(trimed) <= 1 or len(trimed[1]) == 0 \ - else trimed[1] - - with open(fn, 'r') as file: - contents = file.read() - stmts = parser.parse(contents) - continue - stmts = parser.parse(q) - print(stmts) - except (ValueError, FileNotFoundError) as e: - rm() - print(type(e), e) - +from concurrent.futures import thread +import re +import time +import aquery_parser as parser +import engine +import subprocess +import mmap +import sys +import os +from engine.utils import base62uuid +import atexit +try: + os.remove('server.bin') +except Exception as e: + print(type(e), e) + +subprocess.call(['make', 'server.bin']) +cleanup = True + +def rm(): + global cleanup + if cleanup: + mm.seek(0,os.SEEK_SET) + mm.write(b'\x00\x00') + mm.flush() + + try: + time.sleep(.001) + server.kill() + time.sleep(.001) + server.terminate() + except OSError: + pass + + files = os.listdir('.') + for f in files: + if f.endswith('.shm'): + os.remove(f) + mm.close() + cleanup = False + +atexit.register(rm) + +shm = base62uuid() +if sys.platform != 'win32': + import readline + shm += '.shm' + basecmd = ['bash', '-c', 'rlwrap k'] + mm = None + if not os.path.isfile(shm): + # create initial file + with open(shm, "w+b") as handle: + handle.write(b'\x01\x00') # [running, new job] + handle.flush() + mm = mmap.mmap(handle.fileno(), 2, access=mmap.ACCESS_WRITE, offset=0) + if mm is None: + exit(1) +else: + basecmd = ['bash.exe', '-c', 'rlwrap ./k'] + mm = mmap.mmap(0, 2, shm) + mm.write(b'\x01\x00') + mm.flush() +server = subprocess.Popen(["./server.bin", shm]) + +test_parser = True + +# code to test parser +ws = re.compile(r'\s+') + +q = 'SELECT p.Name, v.Name FROM Production.Product p JOIN Purchasing.ProductVendor pv ON p.ProductID = pv.ProductID JOIN Purchasing.Vendor v ON pv.BusinessEntityID = v.BusinessEntityID WHERE ProductSubcategoryID = 15 ORDER BY v.Name;' + +res = parser.parse(q) + +print(res) + +# else:f +# if subprocess.call(['make', 'snippet']) == 0: +# mm.seek(0) +# mm.write(b'\x01\x01') +# time.sleep(.1) +# mm.seek(0) +# print(mm.read(2)) + +# mm.close() +# handle.close() +# os.remove(shm) +# exit() +keep = False +cxt = None +while test_parser: + try: + if server.poll() is not None: + server = subprocess.Popen(["./server.bin", shm]) + q = input() + if q == 'exec': + if not keep or cxt is None: + cxt = engine.initialize() + stmts_stmts = stmts['stmts'] + if type(stmts_stmts) is list: + for s in stmts_stmts: + engine.generate(s, cxt) + else: + engine.generate(stmts_stmts, cxt) + print(cxt.ccode) + with open('out.cpp', 'wb') as outfile: + outfile.write((cxt.finalize()).encode('utf-8')) + if subprocess.call(['make', 'snippet']) == 0: + mm.seek(0,os.SEEK_SET) + mm.write(b'\x01\x01') + continue + elif q == 'k': + subprocess.call(basecmd) + continue + elif q == 'print': + print(stmts) + continue + elif q == 'keep': + keep = not keep + continue + elif q=='format' or q == 'fmt': + subprocess.call(['clang-format', 'out.cpp']) + elif q == 'exit': + break + elif q == 'r': + if subprocess.call(['make', 'snippet']) == 0: + mm.seek(0,os.SEEK_SET) + mm.write(b'\x01\x01') + continue + trimed = ws.sub(' ', q.lower()).split(' ') + if trimed[0].startswith('f'): + fn = 'stock.a' if len(trimed) <= 1 or len(trimed[1]) == 0 \ + else trimed[1] + + with open(fn, 'r') as file: + contents = file.read() + stmts = parser.parse(contents) + continue + stmts = parser.parse(q) + print(stmts) + except (ValueError, FileNotFoundError) as e: + rm() + print(type(e), e) + rm() \ No newline at end of file diff --git a/q1.sql b/q1.sql index bb55cd6..bd6d5fd 100644 --- a/q1.sql +++ b/q1.sql @@ -1,10 +1,10 @@ -CREATE TABLE test(a INT, b INT, c INT, d INT) - -LOAD DATA INFILE "test.csv" -INTO TABLE test -FIELDS TERMINATED BY "," - -SELECT sum(c), b, d -FROM test -group by a,b,d --- order by d DESC, b ASC +CREATE TABLE test(a INT, b INT, c INT, d INT) + +LOAD DATA INFILE "test.csv" +INTO TABLE test +FIELDS TERMINATED BY "," + +SELECT sum(c), b, d +FROM test +group by a,b,d +-- order by d DESC, b ASC diff --git a/server/server.cpp b/server/server.cpp index 5b40ab8..5ff0738 100644 --- a/server/server.cpp +++ b/server/server.cpp @@ -53,7 +53,7 @@ int main(int argc, char** argv) { printf("running: %s\n", running? "true":"false"); printf("ready: %s\n", ready? "true":"false"); while (running) { - std::this_thread::sleep_for(100us); + std::this_thread::sleep_for(1ms); if(ready){ printf("running: %s\n", running? "true":"false"); printf("ready: %s\n", ready? "true":"false");