bill/AQuery
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

bug fix

Browse Source
This commit is contained in:
bill
2022-04-14 03:16:29 +08:00
parent c46ce578d4
commit ae51d8c622
7 changed files with 171 additions and 239 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+2 -2
View File
@@ -1,6 +1,6 @@
# AQuery Compiler # AQuery++
AQuery Compiler that compiles AQuery into C++11. AQuery++ Compiler that compiles AQuery into C++11.
Frontend built on top of [mo-sql-parsing](https://github.com/klahnakoski/mo-sql-parsing). Frontend built on top of [mo-sql-parsing](https://github.com/klahnakoski/mo-sql-parsing).
## Roadmap ## Roadmap
engine/ast.py
-3
View File
@@ -111,9 +111,6 @@ class TableInfo:
self.columns_byname[c['name']] = col_object self.columns_byname[c['name']] = col_object
self.columns.append(col_object) self.columns.append(col_object)
def construct(self):
for c in self.columns:
self.cxt.emit(f'{c.cname}:()')
@property @property
def n_cols(self): def n_cols(self):
return len(self.columns) return len(self.columns)
engine/scan.py
+2 -67
View File
@@ -48,6 +48,7 @@ class filter(ast_node):
def spawn(self, node): def spawn(self, node):
# TODO: deal with subqueries # TODO: deal with subqueries
self.modified_node = node
return super().spawn(node) return super().spawn(node)
def __materialize__(self): def __materialize__(self):
if self.materialize: if self.materialize:
@@ -66,72 +67,6 @@ class filter(ast_node):
def consume(self, node): def consume(self, node):
# TODO: optimizations after converting expr to cnf # TODO: optimizations after converting expr to cnf
if type(node) is bool and node and self.materialize: self.expr = expr(self, self.modified_node)
self.output = self.context.datasource if node else None
self.value = '1' if node else '0'
else:
if type(node) is dict:
def short_circuit(op, idx, inv = True):
v = filter(self, node[op][idx]).value
inv_filter = lambda x: not x if inv else x
if type(v) is bool and inv_filter(v):
self.value = inv_filter(v)
self.__materialize__()
return None
return v
def binary(l, r, _ty = '&'):
if type(l) is bool:
self.value = r
elif type(r) is bool:
self.value = l
elif type(l) is View:
if type(r) is View:
self.emit(f"{l.name}: {l.name} {_ty} {r.name if type(r) is View else f'({r})'}")
self.value = l
elif type(l) is str:
if type(r) is str:
self.value = f'({l}){_ty}({r})'
else:
self.emit(f'{r.name}:{r.name} {_ty} ({l})')
self.value = r
if 'and' in node:
l = short_circuit('and', 0)
if l is not None:
r = short_circuit('and', 1)
if r is not None:
binary(l, r)
elif 'or' in node:
l = short_circuit('or', 0, False)
if l is not None:
r = short_circuit('or', 1, False)
if r is not None:
binary(l, r, '|')
elif 'not' in node:
v = filter(self, node['not']).value
if type(v) is bool:
self.value = not v
self.__materialize__()
elif type(v) is View:
if len(v.table.columns) > 0:
all_rows = View(self.context, v.table)
self.emit(f'{all_rows.name}:(#{v.table.columns[0].cname})#1')
self.emit(f'{v.name}:{all_rows.name}-{v.name}')
self.value = v
else:
self.value = '~(' + v + ')'
# TODO: arithmetic ops connecting logical ops.
else:
e = expr(self, node)
if e.isvector:
v = View(self.context, self.datasource)
v.construct()
self.emit(f'{v.name}:{e.cexpr}')
self.value = v
else:
self.value = e.cexpr
self.__materialize__()
print(node) print(node)
out.cpp
+13 -13
View File
@@ -1,5 +1,5 @@
#include "./server/libaquery.h"
#include "./server/aggregations.h" #include "./server/aggregations.h"
#include "./server/libaquery.h"
extern "C" int __DLLEXPORT__ dllmain(Context* cxt) { extern "C" int __DLLEXPORT__ dllmain(Context* cxt) {
using namespace std; using namespace std;
@@ -42,19 +42,19 @@ stocks_timestamp.emplace_back(15);
stocks_price.emplace_back(2); stocks_price.emplace_back(2);
stocks_timestamp.emplace_back(16); stocks_timestamp.emplace_back(16);
stocks_price.emplace_back(5); stocks_price.emplace_back(5);
auto out_6gPn = new TableInfo<decays<decltype(max((stocks_price[0]-min(stocks_timestamp[0]))))>>("out_6gPn", 1); auto out_ZPYh = new TableInfo<decays<decltype(max((stocks_price[0]-min(stocks_timestamp[0]))))>>("out_ZPYh", 1);
cxt->tables.insert({"out_6gPn", out_6gPn}); cxt->tables.insert({"out_ZPYh", out_ZPYh});
auto& out_6gPn_maxstockspriceminstockstimestamp = *(ColRef<decays<decltype(max((stocks_price[0]-min(stocks_timestamp[0]))))>> *)(&out_6gPn->colrefs[0]); auto& out_ZPYh_maxstockspriceminstockstimestamp = *(ColRef<decays<decltype(max((stocks_price[0]-min(stocks_timestamp[0]))))>> *)(&out_ZPYh->colrefs[0]);
out_6gPn_maxstockspriceminstockstimestamp.init(); out_ZPYh_maxstockspriceminstockstimestamp.init();
out_6gPn_maxstockspriceminstockstimestamp = max((stocks_price-min(stocks_timestamp))); out_ZPYh_maxstockspriceminstockstimestamp = max((stocks_price-min(stocks_timestamp)));
print(*out_6gPn); print(*out_ZPYh);
auto out_7a2d = new TableInfo<decays<decltype(max((stocks_price[0]-mins(stocks_price[0]))))>>("out_7a2d", 1); auto out_1ac3 = new TableInfo<decays<decltype(max((stocks_price[0]-mins(stocks_price[0]))))>>("out_1ac3", 1);
cxt->tables.insert({"out_7a2d", out_7a2d}); cxt->tables.insert({"out_1ac3", out_1ac3});
auto& out_7a2d_maxstockspriceminsstocksprice = *(ColRef<decays<decltype(max((stocks_price[0]-mins(stocks_price[0]))))>> *)(&out_7a2d->colrefs[0]); auto& out_1ac3_maxstockspriceminsstocksprice = *(ColRef<decays<decltype(max((stocks_price[0]-mins(stocks_price[0]))))>> *)(&out_1ac3->colrefs[0]);
out_7a2d_maxstockspriceminsstocksprice.init(); out_1ac3_maxstockspriceminsstocksprice.init();
out_7a2d_maxstockspriceminsstocksprice = max((stocks_price-mins(stocks_price))); out_1ac3_maxstockspriceminsstocksprice = max((stocks_price-mins(stocks_price)));
print(*out_7a2d); print(*out_1ac3);
return 0; return 0;
} }
prompt.py
+143 -143
View File
@@ -1,144 +1,144 @@
from concurrent.futures import thread from concurrent.futures import thread
import re import re
import time import time
import aquery_parser as parser import aquery_parser as parser
import engine import engine
import subprocess import subprocess
import mmap import mmap
import sys import sys
import os import os
from engine.utils import base62uuid from engine.utils import base62uuid
import atexit import atexit
try: try:
os.remove('server.bin') os.remove('server.bin')
except Exception as e: except Exception as e:
print(type(e), e) print(type(e), e)
subprocess.call(['make', 'server.bin']) subprocess.call(['make', 'server.bin'])
cleanup = True cleanup = True
def rm():
if cleanup: def rm():
mm.seek(0,os.SEEK_SET) global cleanup
mm.write(b'\x00\x00') if cleanup:
mm.flush() mm.seek(0,os.SEEK_SET)
time.sleep(.001) mm.write(b'\x00\x00')
server.kill() mm.flush()
files = os.listdir('.')
for f in files: try:
if f.endswith('.shm'): time.sleep(.001)
os.remove(f) server.kill()
mm.close() time.sleep(.001)
cleanup = False server.terminate()
except OSError:
def proc_alive(pid): pass
try:
os.kill(pid, 0) files = os.listdir('.')
except OSError: for f in files:
return False if f.endswith('.shm'):
else: os.remove(f)
return True mm.close()
cleanup = False
atexit.register(rm)
atexit.register(rm)
shm = base62uuid()
if sys.platform != 'win32': shm = base62uuid()
import readline if sys.platform != 'win32':
shm += '.shm' import readline
basecmd = ['bash', '-c', 'rlwrap k'] shm += '.shm'
mm = None basecmd = ['bash', '-c', 'rlwrap k']
if not os.path.isfile(shm): mm = None
# create initial file if not os.path.isfile(shm):
with open(shm, "w+b") as handle: # create initial file
handle.write(b'\x01\x00') # [running, new job] with open(shm, "w+b") as handle:
handle.flush() handle.write(b'\x01\x00') # [running, new job]
mm = mmap.mmap(handle.fileno(), 2, access=mmap.ACCESS_WRITE, offset=0) handle.flush()
if mm is None: mm = mmap.mmap(handle.fileno(), 2, access=mmap.ACCESS_WRITE, offset=0)
exit(1) if mm is None:
else: exit(1)
basecmd = ['bash.exe', '-c', 'rlwrap ./k'] else:
mm = mmap.mmap(0, 2, shm) basecmd = ['bash.exe', '-c', 'rlwrap ./k']
mm.write(b'\x01\x00') mm = mmap.mmap(0, 2, shm)
mm.flush() mm.write(b'\x01\x00')
server = subprocess.Popen(["./server.bin", shm]) mm.flush()
server = subprocess.Popen(["./server.bin", shm])
test_parser = True
test_parser = True
# code to test parser
ws = re.compile(r'\s+') # code to test parser
ws = re.compile(r'\s+')
q = 'SELECT p.Name, v.Name FROM Production.Product p JOIN Purchasing.ProductVendor pv ON p.ProductID = pv.ProductID JOIN Purchasing.Vendor v ON pv.BusinessEntityID = v.BusinessEntityID WHERE ProductSubcategoryID = 15 ORDER BY v.Name;'
q = 'SELECT p.Name, v.Name FROM Production.Product p JOIN Purchasing.ProductVendor pv ON p.ProductID = pv.ProductID JOIN Purchasing.Vendor v ON pv.BusinessEntityID = v.BusinessEntityID WHERE ProductSubcategoryID = 15 ORDER BY v.Name;'
res = parser.parse(q)
res = parser.parse(q)
print(res)
print(res)
# else:f # else:f
# if subprocess.call(['make', 'snippet']) == 0: # if subprocess.call(['make', 'snippet']) == 0:
# mm.seek(0) # mm.seek(0)
# mm.write(b'\x01\x01') # mm.write(b'\x01\x01')
# time.sleep(.1) # time.sleep(.1)
# mm.seek(0) # mm.seek(0)
# print(mm.read(2)) # print(mm.read(2))
# mm.close() # mm.close()
# handle.close() # handle.close()
# os.remove(shm) # os.remove(shm)
# exit() # exit()
keep = False keep = False
cxt = None cxt = None
while test_parser: while test_parser:
try: try:
if not proc_alive(server.pid): if server.poll() is not None:
server = subprocess.Popen(["./server.bin", shm]) server = subprocess.Popen(["./server.bin", shm])
q = input() q = input()
if q == 'exec': if q == 'exec':
if not keep or cxt is None: if not keep or cxt is None:
cxt = engine.initialize() cxt = engine.initialize()
stmts_stmts = stmts['stmts'] stmts_stmts = stmts['stmts']
if type(stmts_stmts) is list: if type(stmts_stmts) is list:
for s in stmts_stmts: for s in stmts_stmts:
engine.generate(s, cxt) engine.generate(s, cxt)
else: else:
engine.generate(stmts_stmts, cxt) engine.generate(stmts_stmts, cxt)
print(cxt.ccode) print(cxt.ccode)
with open('out.cpp', 'wb') as outfile: with open('out.cpp', 'wb') as outfile:
outfile.write((cxt.finalize()).encode('utf-8')) outfile.write((cxt.finalize()).encode('utf-8'))
if subprocess.call(['make', 'snippet']) == 0: if subprocess.call(['make', 'snippet']) == 0:
mm.seek(0,os.SEEK_SET) mm.seek(0,os.SEEK_SET)
mm.write(b'\x01\x01') mm.write(b'\x01\x01')
continue continue
elif q == 'k': elif q == 'k':
subprocess.call(basecmd) subprocess.call(basecmd)
continue continue
elif q == 'print': elif q == 'print':
print(stmts) print(stmts)
continue continue
elif q == 'keep': elif q == 'keep':
keep = not keep keep = not keep
continue continue
elif q=='format' or q == 'fmt': elif q=='format' or q == 'fmt':
subprocess.call(['clang-format', 'out.cpp']) subprocess.call(['clang-format', 'out.cpp'])
elif q == 'exit': elif q == 'exit':
break break
elif q == 'r': elif q == 'r':
if subprocess.call(['make', 'snippet']) == 0: if subprocess.call(['make', 'snippet']) == 0:
mm.seek(0,os.SEEK_SET) mm.seek(0,os.SEEK_SET)
mm.write(b'\x01\x01') mm.write(b'\x01\x01')
continue continue
trimed = ws.sub(' ', q.lower()).split(' ') trimed = ws.sub(' ', q.lower()).split(' ')
if trimed[0].startswith('f'): if trimed[0].startswith('f'):
fn = 'stock.a' if len(trimed) <= 1 or len(trimed[1]) == 0 \ fn = 'stock.a' if len(trimed) <= 1 or len(trimed[1]) == 0 \
else trimed[1] else trimed[1]
with open(fn, 'r') as file: with open(fn, 'r') as file:
contents = file.read() contents = file.read()
stmts = parser.parse(contents) stmts = parser.parse(contents)
continue continue
stmts = parser.parse(q) stmts = parser.parse(q)
print(stmts) print(stmts)
except (ValueError, FileNotFoundError) as e: except (ValueError, FileNotFoundError) as e:
rm() rm()
print(type(e), e) print(type(e), e)
rm() rm()
q1.sql
+10 -10
View File
@@ -1,10 +1,10 @@
CREATE TABLE test(a INT, b INT, c INT, d INT) CREATE TABLE test(a INT, b INT, c INT, d INT)
LOAD DATA INFILE "test.csv" LOAD DATA INFILE "test.csv"
INTO TABLE test INTO TABLE test
FIELDS TERMINATED BY "," FIELDS TERMINATED BY ","
SELECT sum(c), b, d SELECT sum(c), b, d
FROM test FROM test
group by a,b,d group by a,b,d
-- order by d DESC, b ASC -- order by d DESC, b ASC
server/server.cpp
+1 -1
View File
@@ -53,7 +53,7 @@ int main(int argc, char** argv) {
printf("running: %s\n", running? "true":"false"); printf("running: %s\n", running? "true":"false");
printf("ready: %s\n", ready? "true":"false"); printf("ready: %s\n", ready? "true":"false");
while (running) { while (running) {
std::this_thread::sleep_for(100us); std::this_thread::sleep_for(1ms);
if(ready){ if(ready){
printf("running: %s\n", running? "true":"false"); printf("running: %s\n", running? "true":"false");
printf("ready: %s\n", ready? "true":"false"); printf("ready: %s\n", ready? "true":"false");