bill/AQuery
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

server 2

Browse Source
This commit is contained in:
bill
2022-05-28 03:38:47 +08:00
parent 634ed382f5
commit 824db2a43b
13 changed files with 772 additions and 666 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Makefile
+2
View File
@@ -7,6 +7,8 @@ $(info $(OS_SUPPORT))
server.bin:
g++ server/server.cpp $(OS_SUPPORT) --std=c++1z -O3 -march=native -o server.bin
server.so:
g++ server/server.cpp -shared $(OS_SUPPORT) --std=c++1z -O3 -march=native -o server.so
snippet:
g++ -shared -fPIC --std=c++1z out.cpp -O3 -march=native -o dll.so
clean:
README.md
+1 -1
View File
@@ -41,4 +41,4 @@ Recent version of Linux, Windows or MacOS, with recent C++ compiler that has C++
- `exit`: quit the prompt
#### Example:
`f moving_avg.a` <br>
`exec`
`exec`
dbconn.py
+44 -44
View File
@@ -1,45 +1,45 @@
import mariadb
class dbconn:
def __init__(self) -> None:
self.db = None
self.cur = None
def clear(self):
drop_all = f'''
SET FOREIGN_KEY_CHECKS = 0;
SET @tables = NULL;
SELECT GROUP_CONCAT('`', table_schema, '`.`', table_name, '`') INTO @tables
FROM information_schema.tables
WHERE table_schema = '{self.db.database}';
SET @tables = CONCAT('DROP TABLE ', @tables);
PREPARE stmt FROM @tables;
EXECUTE stmt;
DEALLOCATE PREPARE stmt;
SET FOREIGN_KEY_CHECKS = 1;
'''
if self.db:
if not self.cur:
self.cur = self.db.cursor()
self.cur.execute(drop_all)
def connect(self, ip, password = '0508', user = 'root', db = 'db', port = 3306):
try:
self.db = mariadb.connect(
user = user,
password = password,
host = ip,
port = port,
database = db
)
self.cur = self.db.cursor()
except mariadb.Error as e:
print(e)
self.db = None
self.cur = None
def exec(self, sql, params = None):
import mariadb
class dbconn:
def __init__(self) -> None:
self.db = None
self.cur = None
def clear(self):
drop_all = f'''
SET FOREIGN_KEY_CHECKS = 0;
SET @tables = NULL;
SELECT GROUP_CONCAT('`', table_schema, '`.`', table_name, '`') INTO @tables
FROM information_schema.tables
WHERE table_schema = '{self.db.database}';
SET @tables = CONCAT('DROP TABLE ', @tables);
PREPARE stmt FROM @tables;
EXECUTE stmt;
DEALLOCATE PREPARE stmt;
SET FOREIGN_KEY_CHECKS = 1;
'''
if self.db:
if not self.cur:
self.cur = self.db.cursor()
self.cur.execute(drop_all)
def connect(self, ip, password = '0508', user = 'root', db = 'db', port = 3306):
try:
self.db = mariadb.connect(
user = user,
password = password,
host = ip,
port = port,
database = db
)
self.cur = self.db.cursor()
except mariadb.Error as e:
print(e)
self.db = None
self.cur = None
def exec(self, sql, params = None):
self.cur.execute(sql)
out.cpp
+32 -32
View File
@@ -1,6 +1,6 @@
#include "./server/libaquery.h"
#include "./server/aggregations.h"
#include "csv.h"
#include "./server/libaquery.h"
#include "./server/hasher.h"
#include <unordered_map>
@@ -18,40 +18,40 @@ test_a.init("a");
test_b.init("b");
test_c.init("c");
test_d.init("d");
io::CSVReader<4> csv_reader_307VD4("test.csv");
csv_reader_307VD4.read_header(io::ignore_extra_column, "a","b","c","d");
int tmp_3LXIYQmp;
int tmp_1m5NCKR4;
int tmp_10LZcLgy;
int tmp_39pPZL8W;
while(csv_reader_307VD4.read_row(tmp_3LXIYQmp,tmp_1m5NCKR4,tmp_10LZcLgy,tmp_39pPZL8W)) {
io::CSVReader<4> csv_reader_1qh80y("test.csv");
csv_reader_1qh80y.read_header(io::ignore_extra_column, "a","b","c","d");
int tmp_6JfuovoZ;
int tmp_4B4ADRgW;
int tmp_3JHd3elW;
int tmp_1heR8kZw;
while(csv_reader_1qh80y.read_row(tmp_6JfuovoZ,tmp_4B4ADRgW,tmp_3JHd3elW,tmp_1heR8kZw)) {
test_a.emplace_back(tmp_3LXIYQmp);
test_b.emplace_back(tmp_1m5NCKR4);
test_c.emplace_back(tmp_10LZcLgy);
test_d.emplace_back(tmp_39pPZL8W);
test_a.emplace_back(tmp_6JfuovoZ);
test_b.emplace_back(tmp_4B4ADRgW);
test_c.emplace_back(tmp_3JHd3elW);
test_d.emplace_back(tmp_1heR8kZw);
}
typedef record<decltype(test_a[0]),decltype(test_b[0]),decltype(test_d[0])> record_type3OMslKw;
unordered_map<record_type3OMslKw, vector_type<uint32_t>, transTypes<record_type3OMslKw, hasher>> g7LNVAss;
for (uint32_t i1T = 0; i1T < test_a.size; ++i1T){
g7LNVAss[forward_as_tuple(test_a[i1T],test_b[i1T],test_d[i1T])].emplace_back(i1T);
typedef record<decltype(test_a[0]),decltype(test_b[0]),decltype(test_d[0])> record_type3he5qd1;
unordered_map<record_type3he5qd1, vector_type<uint32_t>, transTypes<record_type3he5qd1, hasher>> g59vWI2v;
for (uint32_t i3P = 0; i3P < test_a.size; ++i3P){
g59vWI2v[forward_as_tuple(test_a[i3P],test_b[i3P],test_d[i3P])].emplace_back(i3P);
}
auto out_HSfK = new TableInfo<decays<decltype(sum(test_c))>,value_type<decays<decltype(test_b)>>,value_type<decays<decltype(test_d)>>>("out_HSfK", 3);
cxt->tables.insert({"out_HSfK", out_HSfK});
auto& out_HSfK_sumtestc = *(ColRef<decays<decltype(sum(test_c))>> *)(&out_HSfK->colrefs[0]);
auto& out_HSfK_b = *(ColRef<value_type<decays<decltype(test_b)>>> *)(&out_HSfK->colrefs[1]);
auto& out_HSfK_d = *(ColRef<value_type<decays<decltype(test_d)>>> *)(&out_HSfK->colrefs[2]);
out_HSfK_sumtestc.init("sumtestc");
out_HSfK_b.init("b");
out_HSfK_d.init("d");
for(auto& i18 : g7LNVAss) {
auto &key_3s5slnK = i18.first;
auto &val_2nNLv0D = i18.second;
out_HSfK_sumtestc.emplace_back(sum(test_c[val_2nNLv0D]));
out_HSfK_b.emplace_back(get<1>(key_3s5slnK));
out_HSfK_d.emplace_back(get<2>(key_3s5slnK));
auto out_6JAp = new TableInfo<decays<decltype(sum(test_c))>,value_type<decays<decltype(test_b)>>,value_type<decays<decltype(test_d)>>>("out_6JAp", 3);
cxt->tables.insert({"out_6JAp", out_6JAp});
auto& out_6JAp_sumtestc = *(ColRef<decays<decltype(sum(test_c))>> *)(&out_6JAp->colrefs[0]);
auto& out_6JAp_b = *(ColRef<value_type<decays<decltype(test_b)>>> *)(&out_6JAp->colrefs[1]);
auto& out_6JAp_d = *(ColRef<value_type<decays<decltype(test_d)>>> *)(&out_6JAp->colrefs[2]);
out_6JAp_sumtestc.init("sumtestc");
out_6JAp_b.init("b");
out_6JAp_d.init("d");
for(auto& i2Y : g59vWI2v) {
auto &key_1yBYhdd = i2Y.first;
auto &val_61QXy6G = i2Y.second;
out_6JAp_sumtestc.emplace_back(sum(test_c[val_61QXy6G]));
out_6JAp_b.emplace_back(get<1>(key_1yBYhdd));
out_6JAp_d.emplace_back(get<2>(key_1yBYhdd));
}
auto d5b7C95U = out_HSfK->order_by_view<-3,1>();
print(d5b7C95U);
auto d1dyPtv0 = out_6JAp->order_by_view<-3,1>();
print(d1dyPtv0);
return 0;
}
prompt.py
+118 -27
View File
@@ -1,3 +1,4 @@
import enum
import re
import time
import dbconn
@@ -12,14 +13,26 @@ import sys
import os
from engine.utils import base62uuid
import atexit
import threading
import ctypes
class RunType(enum.Enum):
Threaded = 0
IPC = 1
server_mode = RunType.Threaded
server_bin = 'server.bin' if server_mode == RunType.IPC else 'server.so'
try:
os.remove('server.bin')
os.remove(server_bin)
except Exception as e:
print(type(e), e)
nullstream = open(os.devnull, 'w')
subprocess.call(['make', 'server.bin'], stdout=nullstream)
subprocess.call(['make', server_bin], stdout=nullstream)
cleanup = True
def rm():
@@ -44,10 +57,8 @@ def rm():
mm.close()
cleanup = False
nullstream.close()
atexit.register(rm)
def init():
def init_ipc():
global shm, server, basecmd, mm
shm = base62uuid()
if sys.platform != 'win32':
@@ -70,6 +81,100 @@ def init():
mm.flush()
server = subprocess.Popen(["./server.bin", shm])
import numpy as np
c = lambda _ba: ctypes.cast((ctypes.c_char * len(_ba)).from_buffer(_ba), ctypes.c_char_p)
class Config:
def __init__(self, nq = 0, mode = server_mode, n_bufs = 0, bf_szs = []) -> None:
self.int_size = 4
self.n_attrib = 4
self.buf = bytearray((self.n_attrib + n_bufs) * self.int_size)
self.np_buf = np.ndarray(shape=(self.n_attrib), buffer=self.buf, dtype=np.int32)
self.new_query = nq
self.server_mode = mode.value
self.running = 1
self.n_buffers = n_bufs
@property
def running(self):
return self.np_buf[0]
@running.setter
def running(self, rn):
self.np_buf[0] = rn
@property
def new_query(self):
return self.np_buf[1]
@new_query.setter
def new_query(self, nq):
self.np_buf[1] = nq
@property
def server_mode(self):
return self.np_buf[2]
@server_mode.setter
def server_mode(self, mode):
self.np_buf[2] = mode
@property
def n_buffers(self):
return self.np_buf[3]
@n_buffers.setter
def n_buffers(self, n_bufs):
self.np_buf[3] = n_bufs
def set_bufszs(self, buf_szs):
for i in range(min(len(buf_szs), self.n_buffers)):
self.np_buf[i+self.n_attrib] = buf_szs[i]
@property
def c(self):
return c(self.buf)
cfg = Config()
th = None
def init_threaded():
if os.name == 'nt':
t = os.environ['PATH'].lower().split(';')
vars = re.compile('%.*%')
for e in t:
if(len(e) != 0):
if '%' in e:
try:
m_e = vars.findall(e)
for m in m_e:
e = e.replace(m, os.environ[m[1:-1]])
# print(m, e)
except Exception:
continue
os.add_dll_directory(e)
server_so = ctypes.CDLL('./'+server_bin)
global cfg, th
th = threading.Thread(target=server_so['main'], args=(-1, ctypes.POINTER(ctypes.c_char_p)(cfg.c)), daemon=True)
th.start()
if server_mode == RunType.IPC:
atexit.register(rm)
init = init_ipc
set_ready = lambda : mm.seek(0,os.SEEK_SET) or mm.write(b'\x01\x01')
def __get_ready():
mm.seek(0,os.SEEK_SET)
return mm.read(2)[1]
get_ready = __get_ready
server_status = lambda : server.poll() is not None
else:
init = init_threaded
rm = lambda: None
def __set_ready():
global cfg
cfg.new_query = 1
set_ready = __set_ready
get_ready = lambda:cfg.new_query
server_status = lambda : not th.is_alive()
init()
test_parser = True
@@ -82,31 +187,16 @@ q = 'SELECT p.Name, v.Name FROM Production.Product p JOIN Purchasing.ProductVend
res = parser.parse(q)
# else:f
# if subprocess.call(['make', 'snippet']) == 0:
# mm.seek(0)
# mm.write(b'\x01\x01')
# time.sleep(.1)
# mm.seek(0)
# print(mm.read(2))
# mm.close()
# handle.close()
# os.remove(shm)
# exit()
keep = True
cxt = engine.initialize()
cxt.Info(res)
while test_parser:
try:
if server.poll() is not None:
if server_status():
init()
print("> ", end="")
ready = 1
while ready == 1:
mm.seek(0,os.SEEK_SET)
ready = mm.read(2)[1]
while get_ready():
time.sleep(.00001)
print("> ", end="")
q = input().lower()
if q == 'exec':
if not keep or cxt is None:
@@ -123,8 +213,7 @@ while test_parser:
with open('out.cpp', 'wb') as outfile:
outfile.write((cxt.finalize()).encode('utf-8'))
if subprocess.call(['make', 'snippet'], stdout = nullstream) == 0:
mm.seek(0,os.SEEK_SET)
mm.write(b'\x01\x01')
set_ready()
continue
if q == 'xexec':
cxt = xengine.initialize()
@@ -159,8 +248,10 @@ while test_parser:
break
elif q == 'r':
if subprocess.call(['make', 'snippet']) == 0:
mm.seek(0,os.SEEK_SET)
mm.write(b'\x01\x01')
set_ready()
continue
elif q == 'rr':
set_ready()
continue
elif q.startswith('save'):
filename = re.split(' |\t', q)
reconstruct/__init__.py
+11 -11
View File
@@ -1,11 +1,11 @@
from reconstruct.ast import Context, ast_node
def initialize():
return Context()
def generate(ast, cxt):
for k in ast.keys():
if k in ast_node.types.keys():
ast_node.types[k](None, ast, cxt)
__all__ = ["initialize", "generate"]
from reconstruct.ast import Context, ast_node
def initialize():
return Context()
def generate(ast, cxt):
for k in ast.keys():
if k in ast_node.types.keys():
ast_node.types[k](None, ast, cxt)
__all__ = ["initialize", "generate"]
reconstruct/ast.py
+302 -302
View File
@@ -1,303 +1,303 @@
from engine.utils import enlist, base62uuid, base62alp
from reconstruct.storage import Context, TableInfo, ColRef
class ast_node:
header = []
types = dict()
first_order = False
def __init__(self, parent:"ast_node", node, context:Context = None):
self.context = parent.context if context is None else context
self.parent = parent
self.sql = ''
self.datasource = None
self.init(node)
self.produce(node)
self.spawn(node)
self.consume(node)
def emit(self, code):
self.context.emit(code)
def add(self, code):
self.sql += code + ' '
name = 'null'
def init(self, _):
self.add(self.__class__.name.upper())
def produce(self, _):
pass
def spawn(self, _):
pass
def consume(self, _):
if self.parent is None:
self.emit(self.sql+';\n')
from reconstruct.expr import expr
class projection(ast_node):
name = 'projection'
first_order = 'select'
def init(self, _):
pass
def produce(self, node):
p = node['select']
self.projections = p if type(p) is list else [p]
self.add('SELECT')
def spawn(self, node):
self.datasource = None # datasource is Join instead of TableInfo
if 'from' in node:
from_clause = node['from']
self.datasource = join(self, from_clause)
if 'assumptions' in from_clause:
self.assumptions = enlist(from_clause['assumptions'])
if self.datasource is not None:
self.datasource_changed = True
self.prev_datasource = self.context.datasource
self.context.datasource = self.datasource
if 'where' in node:
self.where = filter(self, node['where'])
else:
self.where = None
if 'groupby' in node:
self.group_node = groupby(self, node['groupby'])
else:
self.group_node = None
def consume(self, node):
# deal with projections
self.out_table = TableInfo('out_'+base62uuid(4), [], self.context)
cols = []
col_exprs = []
for i, proj in enumerate(self.projections):
compound = False
self.datasource.rec = set()
name = ''
if type(proj) is dict:
if 'value' in proj:
e = proj['value']
name = expr(self, e).sql
disp_name = ''.join([a if a in base62alp else '' for a in name])
compound = True # compound column
if 'name' in proj: # renaming column by AS keyword
name += ' ' + proj['name']
col_exprs.append(name)
elif type(proj) is str:
col = self.datasource.get_col(proj)
name = col.name
self.datasource.rec = None
# TODO: Type deduction in Python
cols.append(ColRef('unknown', self.out_table, None, disp_name, i, compound=compound))
self.add(', '.join(col_exprs))
def finialize(astnode:ast_node):
if(astnode is not None):
self.add(astnode.sql)
self.add('FROM')
finialize(self.datasource)
finialize(self.where)
finialize(self.group_node)
if 'orderby' in node:
self.add(orderby(self, node['orderby']).sql)
if 'outfile' in node:
self.add(outfile(self, node['outfile']).sql)
if self.parent is None:
self.emit(self.sql+';\n')
else:
# TODO: subquery, name create tmp-table from subquery w/ alias as name
pass
class orderby(ast_node):
name = 'order by'
def produce(self, node):
if node is None:
self.sql = ''
return
elif type(node) is not list:
node = [node]
o_list = []
for o in node:
o_str = expr(self, o['value']).sql
if 'sort' in o and f'{o["sort"]}'.lower() == 'desc':
o_str += ' ' + 'DESC'
o_list.append(o_str)
self.add(', '.join(o_list))
class groupby(orderby):
name = 'group by'
class join(ast_node):
name = 'join'
def init(self, _):
self.joins:list = []
self.tables = []
self.tables_dir = dict()
# self.tmp_name = 'join_' + base62uuid(4)
# self.datasource = TableInfo(self.tmp_name, [], self.context)
def append(self, tbls, __alias = ''):
alias = lambda t : '(' + t + ') ' + __alias if len(__alias) else t
if type(tbls) is join:
self.joins.append(alias(tbls.__str__()))
self.tables += tbls.tables
self.tables_dir = {**self.tables_dir, **tbls.tables_dir}
elif type(tbls) is TableInfo:
self.joins.append(alias(tbls.table_name))
self.tables.append(tbls)
self.tables_dir[tbls.table_name] = tbls
for a in tbls.alias:
self.tables_dir[a] = tbls
elif type(tbls) is projection:
self.joins.append(alias(tbls.finalize()))
def produce(self, node):
if type(node) is list:
for d in node:
self.append(join(self, d).__str__())
elif type(node) is dict:
alias = ''
if 'value' in node:
table_name = node['value']
tbl = None
if 'name' in node:
alias = node['name']
if type(table_name) is dict:
if 'select' in table_name:
# TODO: subquery, create and register TableInfo in projection
tbl = projection(self, table_name).finalize()
else:
tbl = self.context.tables_byname[table_name]
if 'name' in node:
tbl.add_alias(node['name'])
self.append(tbl, alias)
else:
keys = node.keys()
if keys[0].lower().endswith('join'):
j = join(self, node[keys[0]])
tablename = f' {keys[0]} {j}'
if keys[1].lower() == 'on':
tablename += f' on {expr(self, node[keys[1]])}'
self.joins.append(tablename)
self.tables += j.tables
self.tables_dir = {**self.tables_dir, **j.tables_dir}
elif type(node) is str:
self.append(self.context.tables_byname[node])
def get_cols(self, colExpr: str) -> ColRef:
for t in self.tables:
if colExpr in t.columns_byname:
return t.columns_byname[colExpr]
def parse_col_names(self, colExpr:str) -> ColRef:
parsedColExpr = colExpr.split('.')
if len(parsedColExpr) <= 1:
return self.get_cols(colExpr)
else:
datasource = self.tables_dir[parsedColExpr[0]]
if datasource is None:
raise ValueError(f'Table name/alias not defined{parsedColExpr[0]}')
else:
return datasource.parse_col_names(parsedColExpr[1])
def consume(self, _):
self.sql = ', '.join(self.joins)
return super().consume(_)
def __str__(self):
return ', '.join(self.joins)
def __repr__(self):
return self.__str__()
class filter(ast_node):
name = 'where'
def produce(self, node):
self.add(expr(self, node).sql)
class create_table(ast_node):
name = 'create_table'
first_order = name
def init(self, node):
self.sql = 'CREATE TABLE '
def produce(self, node):
ct = node[self.name]
tbl = self.context.add_table(ct['name'], ct['columns'])
self.sql = f'CREATE TABLE {tbl.table_name}('
columns = []
for c in tbl.columns:
columns.append(f'{c.name} {c.type.upper()}')
self.sql += ', '.join(columns)
self.sql += ')'
class insert(ast_node):
name = 'insert'
first_order = name
def produce(self, node):
values = node['query']['select']
tbl = node['insert']
self.sql = f'INSERT INTO {tbl} VALUES('
# if len(values) != table.n_cols:
# raise ValueError("Column Mismatch")
list_values = []
for i, s in enumerate(values):
if 'value' in s:
list_values.append(f"{s['value']}")
else:
# subquery, dispatch to select astnode
pass
self.sql += ', '.join(list_values) + ')'
class load(ast_node):
name="load"
first_order = name
def produce(self, node):
node = node['load']
s1 = 'LOAD DATA INFILE '
s2 = 'INTO TABLE '
s3 = 'FIELDS TERMINATED BY '
self.sql = f'{s1} \"{node["file"]["literal"]}\" {s2} {node["table"]}'
if 'term' in node:
self.sql += f' {s3} \"{node["term"]["literal"]}\"'
class outfile(ast_node):
name="_outfile"
def produce(self, node):
filename = node['loc']['literal'] if 'loc' in node else node['literal']
self.sql = f'INTO OUTFILE "{filename}"'
if 'term' in node:
self.sql += f' FIELDS TERMINATED BY \"{node["term"]["literal"]}\"'
def include(objs):
import inspect
for _, cls in inspect.getmembers(objs):
if inspect.isclass(cls) and issubclass(cls, ast_node) and type(cls.first_order) is str:
ast_node.types[cls.first_order] = cls
import sys
from engine.utils import enlist, base62uuid, base62alp
from reconstruct.storage import Context, TableInfo, ColRef
class ast_node:
header = []
types = dict()
first_order = False
def __init__(self, parent:"ast_node", node, context:Context = None):
self.context = parent.context if context is None else context
self.parent = parent
self.sql = ''
self.datasource = None
self.init(node)
self.produce(node)
self.spawn(node)
self.consume(node)
def emit(self, code):
self.context.emit(code)
def add(self, code):
self.sql += code + ' '
name = 'null'
def init(self, _):
self.add(self.__class__.name.upper())
def produce(self, _):
pass
def spawn(self, _):
pass
def consume(self, _):
if self.parent is None:
self.emit(self.sql+';\n')
from reconstruct.expr import expr
class projection(ast_node):
name = 'projection'
first_order = 'select'
def init(self, _):
pass
def produce(self, node):
p = node['select']
self.projections = p if type(p) is list else [p]
self.add('SELECT')
def spawn(self, node):
self.datasource = None # datasource is Join instead of TableInfo
if 'from' in node:
from_clause = node['from']
self.datasource = join(self, from_clause)
if 'assumptions' in from_clause:
self.assumptions = enlist(from_clause['assumptions'])
if self.datasource is not None:
self.datasource_changed = True
self.prev_datasource = self.context.datasource
self.context.datasource = self.datasource
if 'where' in node:
self.where = filter(self, node['where'])
else:
self.where = None
if 'groupby' in node:
self.group_node = groupby(self, node['groupby'])
else:
self.group_node = None
def consume(self, node):
# deal with projections
self.out_table = TableInfo('out_'+base62uuid(4), [], self.context)
cols = []
col_exprs = []
for i, proj in enumerate(self.projections):
compound = False
self.datasource.rec = set()
name = ''
if type(proj) is dict:
if 'value' in proj:
e = proj['value']
name = expr(self, e).sql
disp_name = ''.join([a if a in base62alp else '' for a in name])
compound = True # compound column
if 'name' in proj: # renaming column by AS keyword
name += ' ' + proj['name']
col_exprs.append(name)
elif type(proj) is str:
col = self.datasource.get_col(proj)
name = col.name
self.datasource.rec = None
# TODO: Type deduction in Python
cols.append(ColRef('unknown', self.out_table, None, disp_name, i, compound=compound))
self.add(', '.join(col_exprs))
def finialize(astnode:ast_node):
if(astnode is not None):
self.add(astnode.sql)
self.add('FROM')
finialize(self.datasource)
finialize(self.where)
finialize(self.group_node)
if 'orderby' in node:
self.add(orderby(self, node['orderby']).sql)
if 'outfile' in node:
self.add(outfile(self, node['outfile']).sql)
if self.parent is None:
self.emit(self.sql+';\n')
else:
# TODO: subquery, name create tmp-table from subquery w/ alias as name
pass
class orderby(ast_node):
name = 'order by'
def produce(self, node):
if node is None:
self.sql = ''
return
elif type(node) is not list:
node = [node]
o_list = []
for o in node:
o_str = expr(self, o['value']).sql
if 'sort' in o and f'{o["sort"]}'.lower() == 'desc':
o_str += ' ' + 'DESC'
o_list.append(o_str)
self.add(', '.join(o_list))
class groupby(orderby):
name = 'group by'
class join(ast_node):
name = 'join'
def init(self, _):
self.joins:list = []
self.tables = []
self.tables_dir = dict()
# self.tmp_name = 'join_' + base62uuid(4)
# self.datasource = TableInfo(self.tmp_name, [], self.context)
def append(self, tbls, __alias = ''):
alias = lambda t : '(' + t + ') ' + __alias if len(__alias) else t
if type(tbls) is join:
self.joins.append(alias(tbls.__str__()))
self.tables += tbls.tables
self.tables_dir = {**self.tables_dir, **tbls.tables_dir}
elif type(tbls) is TableInfo:
self.joins.append(alias(tbls.table_name))
self.tables.append(tbls)
self.tables_dir[tbls.table_name] = tbls
for a in tbls.alias:
self.tables_dir[a] = tbls
elif type(tbls) is projection:
self.joins.append(alias(tbls.finalize()))
def produce(self, node):
if type(node) is list:
for d in node:
self.append(join(self, d).__str__())
elif type(node) is dict:
alias = ''
if 'value' in node:
table_name = node['value']
tbl = None
if 'name' in node:
alias = node['name']
if type(table_name) is dict:
if 'select' in table_name:
# TODO: subquery, create and register TableInfo in projection
tbl = projection(self, table_name).finalize()
else:
tbl = self.context.tables_byname[table_name]
if 'name' in node:
tbl.add_alias(node['name'])
self.append(tbl, alias)
else:
keys = node.keys()
if keys[0].lower().endswith('join'):
j = join(self, node[keys[0]])
tablename = f' {keys[0]} {j}'
if keys[1].lower() == 'on':
tablename += f' on {expr(self, node[keys[1]])}'
self.joins.append(tablename)
self.tables += j.tables
self.tables_dir = {**self.tables_dir, **j.tables_dir}
elif type(node) is str:
self.append(self.context.tables_byname[node])
def get_cols(self, colExpr: str) -> ColRef:
for t in self.tables:
if colExpr in t.columns_byname:
return t.columns_byname[colExpr]
def parse_col_names(self, colExpr:str) -> ColRef:
parsedColExpr = colExpr.split('.')
if len(parsedColExpr) <= 1:
return self.get_cols(colExpr)
else:
datasource = self.tables_dir[parsedColExpr[0]]
if datasource is None:
raise ValueError(f'Table name/alias not defined{parsedColExpr[0]}')
else:
return datasource.parse_col_names(parsedColExpr[1])
def consume(self, _):
self.sql = ', '.join(self.joins)
return super().consume(_)
def __str__(self):
return ', '.join(self.joins)
def __repr__(self):
return self.__str__()
class filter(ast_node):
name = 'where'
def produce(self, node):
self.add(expr(self, node).sql)
class create_table(ast_node):
name = 'create_table'
first_order = name
def init(self, node):
self.sql = 'CREATE TABLE '
def produce(self, node):
ct = node[self.name]
tbl = self.context.add_table(ct['name'], ct['columns'])
self.sql = f'CREATE TABLE {tbl.table_name}('
columns = []
for c in tbl.columns:
columns.append(f'{c.name} {c.type.upper()}')
self.sql += ', '.join(columns)
self.sql += ')'
class insert(ast_node):
name = 'insert'
first_order = name
def produce(self, node):
values = node['query']['select']
tbl = node['insert']
self.sql = f'INSERT INTO {tbl} VALUES('
# if len(values) != table.n_cols:
# raise ValueError("Column Mismatch")
list_values = []
for i, s in enumerate(values):
if 'value' in s:
list_values.append(f"{s['value']}")
else:
# subquery, dispatch to select astnode
pass
self.sql += ', '.join(list_values) + ')'
class load(ast_node):
name="load"
first_order = name
def produce(self, node):
node = node['load']
s1 = 'LOAD DATA INFILE '
s2 = 'INTO TABLE '
s3 = 'FIELDS TERMINATED BY '
self.sql = f'{s1} \"{node["file"]["literal"]}\" {s2} {node["table"]}'
if 'term' in node:
self.sql += f' {s3} \"{node["term"]["literal"]}\"'
class outfile(ast_node):
name="_outfile"
def produce(self, node):
filename = node['loc']['literal'] if 'loc' in node else node['literal']
self.sql = f'INTO OUTFILE "{filename}"'
if 'term' in node:
self.sql += f' FIELDS TERMINATED BY \"{node["term"]["literal"]}\"'
def include(objs):
import inspect
for _, cls in inspect.getmembers(objs):
if inspect.isclass(cls) and issubclass(cls, ast_node) and type(cls.first_order) is str:
ast_node.types[cls.first_order] = cls
import sys
include(sys.modules[__name__])
reconstruct/expr.py
+127 -127
View File
@@ -1,128 +1,128 @@
from reconstruct.ast import ast_node
from reconstruct.storage import ColRef, TableInfo
class expr(ast_node):
name='expr'
builtin_func_maps = {
'max': 'MAX',
'min': 'MIN',
'avg': 'AVG',
'sum': 'SUM',
'count' : 'COUNT',
'mins': ['mins', 'minw'],
'maxs': ['maxs', 'maxw'],
'avgs': ['avgs', 'avgw'],
'sums': ['sums', 'sumw'],
}
binary_ops = {
'sub':'-',
'add':'+',
'mul':'*',
'div':'/',
'mod':'%',
'and':' AND ',
'or':' OR ',
'xor' : ' XOR ',
'gt':'>',
'lt':'<',
'le':'<=',
'gt':'>='
}
compound_ops = {
}
unary_ops = {
'neg' : '-',
'not' : ' NOT '
}
coumpound_generating_ops = ['avgs', 'mins', 'maxs', 'sums'] + \
list(binary_ops.keys()) + list(compound_ops.keys()) + list(unary_ops.keys() )
def __init__(self, parent, node):
self.raw_col = None
self.inside_agg = False
if(type(parent) is expr):
self.inside_agg = parent.inside_agg
ast_node.__init__(self, parent, node, None)
def init(self, _):
from engine.projection import projection
parent = self.parent
self.isvector = parent.isvector if type(parent) is expr else False
self.is_compound = parent.is_compound if type(parent) is expr else False
if type(parent) in [projection, expr]:
self.datasource = parent.datasource
else:
self.datasource = self.context.datasource
self.udf_map = parent.context.udf_map
self.func_maps = {**self.udf_map, **self.builtin_func_maps}
def produce(self, node):
if type(node) is dict:
for key, val in node.items():
if key in self.func_maps:
# TODO: distinguish between UDF agg functions and other UDF functions.
self.inside_agg = True
if type(val) is list and len(val) > 1:
cfunc = self.func_maps[key]
cfunc = cfunc[len(val) - 1] if type(cfunc) is list else cfunc
self.sql += f"{cfunc}("
for i, p in enumerate(val):
self.sql += expr(self, p).sql + (',' if i < len(val) - 1 else '')
else:
funcname = self.func_maps[key]
funcname = funcname[0] if type(funcname) is list else funcname
self.sql += f"{funcname}("
self.sql += expr(self, val).sql
self.sql += ')'
self.inside_agg = False
elif key in self.binary_ops:
l = expr(self, val[0]).sql
r = expr(self, val[1]).sql
self.sql += f'({l}{self.binary_ops[key]}{r})'
elif key in self.compound_ops:
x = []
if type(val) is list:
for v in val:
x.append(expr(self, v).sql)
self.sql = self.compound_ops[key][1](x)
elif key in self.unary_ops:
self.sql += f'{self.unary_ops[key]}({expr(self, val).sql})'
else:
print(f'Undefined expr: {key}{val}')
if key in self.coumpound_generating_ops and not self.is_compound:
self.is_compound = True
p = self.parent
while type(p) is expr and not p.is_compound:
p.is_compound = True
p = p.parent
elif type(node) is str:
p = self.parent
while type(p) is expr and not p.isvector:
p.isvector = True
p = p.parent
self.raw_col = self.datasource.parse_col_names(node)
self.raw_col = self.raw_col if type(self.raw_col) is ColRef else None
if self.raw_col is not None:
self.sql = self.raw_col.name
else:
self.sql = node
elif type(node) is bool:
self.sql = '1' if node else '0'
else:
self.sql = f'{node}'
def __str__(self):
return self.sql
def __repr__(self):
return self.__str__()
from reconstruct.ast import ast_node
from reconstruct.storage import ColRef, TableInfo
class expr(ast_node):
name='expr'
builtin_func_maps = {
'max': 'MAX',
'min': 'MIN',
'avg': 'AVG',
'sum': 'SUM',
'count' : 'COUNT',
'mins': ['mins', 'minw'],
'maxs': ['maxs', 'maxw'],
'avgs': ['avgs', 'avgw'],
'sums': ['sums', 'sumw'],
}
binary_ops = {
'sub':'-',
'add':'+',
'mul':'*',
'div':'/',
'mod':'%',
'and':' AND ',
'or':' OR ',
'xor' : ' XOR ',
'gt':'>',
'lt':'<',
'le':'<=',
'gt':'>='
}
compound_ops = {
}
unary_ops = {
'neg' : '-',
'not' : ' NOT '
}
coumpound_generating_ops = ['avgs', 'mins', 'maxs', 'sums'] + \
list(binary_ops.keys()) + list(compound_ops.keys()) + list(unary_ops.keys() )
def __init__(self, parent, node):
self.raw_col = None
self.inside_agg = False
if(type(parent) is expr):
self.inside_agg = parent.inside_agg
ast_node.__init__(self, parent, node, None)
def init(self, _):
from engine.projection import projection
parent = self.parent
self.isvector = parent.isvector if type(parent) is expr else False
self.is_compound = parent.is_compound if type(parent) is expr else False
if type(parent) in [projection, expr]:
self.datasource = parent.datasource
else:
self.datasource = self.context.datasource
self.udf_map = parent.context.udf_map
self.func_maps = {**self.udf_map, **self.builtin_func_maps}
def produce(self, node):
if type(node) is dict:
for key, val in node.items():
if key in self.func_maps:
# TODO: distinguish between UDF agg functions and other UDF functions.
self.inside_agg = True
if type(val) is list and len(val) > 1:
cfunc = self.func_maps[key]
cfunc = cfunc[len(val) - 1] if type(cfunc) is list else cfunc
self.sql += f"{cfunc}("
for i, p in enumerate(val):
self.sql += expr(self, p).sql + (',' if i < len(val) - 1 else '')
else:
funcname = self.func_maps[key]
funcname = funcname[0] if type(funcname) is list else funcname
self.sql += f"{funcname}("
self.sql += expr(self, val).sql
self.sql += ')'
self.inside_agg = False
elif key in self.binary_ops:
l = expr(self, val[0]).sql
r = expr(self, val[1]).sql
self.sql += f'({l}{self.binary_ops[key]}{r})'
elif key in self.compound_ops:
x = []
if type(val) is list:
for v in val:
x.append(expr(self, v).sql)
self.sql = self.compound_ops[key][1](x)
elif key in self.unary_ops:
self.sql += f'{self.unary_ops[key]}({expr(self, val).sql})'
else:
print(f'Undefined expr: {key}{val}')
if key in self.coumpound_generating_ops and not self.is_compound:
self.is_compound = True
p = self.parent
while type(p) is expr and not p.is_compound:
p.is_compound = True
p = p.parent
elif type(node) is str:
p = self.parent
while type(p) is expr and not p.isvector:
p.isvector = True
p = p.parent
self.raw_col = self.datasource.parse_col_names(node)
self.raw_col = self.raw_col if type(self.raw_col) is ColRef else None
if self.raw_col is not None:
self.sql = self.raw_col.name
else:
self.sql = node
elif type(node) is bool:
self.sql = '1' if node else '0'
else:
self.sql = f'{node}'
def __str__(self):
return self.sql
def __repr__(self):
return self.__str__()
reconstruct/storage.py
+88 -88
View File
@@ -1,89 +1,89 @@
class ColRef:
def __init__(self, _ty, cobj, table:'TableInfo', name, id, compound = False):
self.type = _ty
self.cobj = cobj
self.table = table
self.name = name
self.alias = set()
self.id = id # position in table
self.compound = compound # compound field (list as a field)
# e.g. order by, group by, filter by expressions
self.__arr__ = (_ty, cobj, table, name, id)
def __getitem__(self, key):
if type(key) is str:
return getattr(self, key)
else:
return self.__arr__[key]
def __setitem__(self, key, value):
self.__arr__[key] = value
class TableInfo:
def __init__(self, table_name, cols, cxt:'Context'):
# statics
self.table_name = table_name
self.alias = set([table_name])
self.columns_byname = dict() # column_name, type
self.columns = []
self.cxt = cxt
# keep track of temp vars
self.rec = None
self.add_cols(cols)
# runtime
self.order = [] # assumptions
cxt.tables_byname[self.table_name] = self # construct reverse map
def add_cols(self, cols, new = True):
for i, c in enumerate(cols):
self.add_col(c, new, i)
def add_col(self, c, new = True, i = 0):
_ty = c['type']
if new:
_ty = _ty if type(c) is ColRef else list(_ty.keys())[0]
col_object = ColRef(_ty, c, self, c['name'], len(self.columns))
else:
col_object = c
c.table = self
self.columns_byname[c['name']] = col_object
self.columns.append(col_object)
def add_alias(self, alias):
if alias in self.cxt.tables_byname.keys():
print("Error: table alias already exists")
return
self.cxt.tables_byname[alias] = self
self.alias.add(alias)
def parse_col_names(self, colExpr) -> ColRef:
parsedColExpr = colExpr.split('.')
if len(parsedColExpr) <= 1:
return self.columns_byname[colExpr]
else:
datasource = self.cxt.tables_byname[parsedColExpr[0]]
if datasource is None:
raise ValueError(f'Table name/alias not defined{parsedColExpr[0]}')
else:
return datasource.parse_col_names(parsedColExpr[1])
class Context:
def __init__(self):
self.sql = ''
self.tables_byname = dict()
self.col_byname = dict()
self.tables = []
self.cols = []
self.datasource = None
self.udf_map = dict()
def emit(self, sql:str):
self.sql += sql + ' '
def add_table(self, table_name, cols):
tbl = TableInfo(table_name, cols, self)
self.tables.append(tbl)
return tbl
class ColRef:
def __init__(self, _ty, cobj, table:'TableInfo', name, id, compound = False):
self.type = _ty
self.cobj = cobj
self.table = table
self.name = name
self.alias = set()
self.id = id # position in table
self.compound = compound # compound field (list as a field)
# e.g. order by, group by, filter by expressions
self.__arr__ = (_ty, cobj, table, name, id)
def __getitem__(self, key):
if type(key) is str:
return getattr(self, key)
else:
return self.__arr__[key]
def __setitem__(self, key, value):
self.__arr__[key] = value
class TableInfo:
def __init__(self, table_name, cols, cxt:'Context'):
# statics
self.table_name = table_name
self.alias = set([table_name])
self.columns_byname = dict() # column_name, type
self.columns = []
self.cxt = cxt
# keep track of temp vars
self.rec = None
self.add_cols(cols)
# runtime
self.order = [] # assumptions
cxt.tables_byname[self.table_name] = self # construct reverse map
def add_cols(self, cols, new = True):
for i, c in enumerate(cols):
self.add_col(c, new, i)
def add_col(self, c, new = True, i = 0):
_ty = c['type']
if new:
_ty = _ty if type(c) is ColRef else list(_ty.keys())[0]
col_object = ColRef(_ty, c, self, c['name'], len(self.columns))
else:
col_object = c
c.table = self
self.columns_byname[c['name']] = col_object
self.columns.append(col_object)
def add_alias(self, alias):
if alias in self.cxt.tables_byname.keys():
print("Error: table alias already exists")
return
self.cxt.tables_byname[alias] = self
self.alias.add(alias)
def parse_col_names(self, colExpr) -> ColRef:
parsedColExpr = colExpr.split('.')
if len(parsedColExpr) <= 1:
return self.columns_byname[colExpr]
else:
datasource = self.cxt.tables_byname[parsedColExpr[0]]
if datasource is None:
raise ValueError(f'Table name/alias not defined{parsedColExpr[0]}')
else:
return datasource.parse_col_names(parsedColExpr[1])
class Context:
def __init__(self):
self.sql = ''
self.tables_byname = dict()
self.col_byname = dict()
self.tables = []
self.cols = []
self.datasource = None
self.udf_map = dict()
def emit(self, sql:str):
self.sql += sql + ' '
def add_table(self, table_name, cols):
tbl = TableInfo(table_name, cols, self)
self.tables.append(tbl)
return tbl
server.exe
BIN
View File
Binary file not shown.
server/.claudiaideconfig
-27
View File
@@ -1,27 +0,0 @@
{
"BackgroundImageAbsolutePath": "c:\\users\\bill\\appdata\\local\\microsoft\\visualstudio\\17.0_03c65567\\extensions\\atkxhose.05t\\Images\\background.png",
"BackgroundImagesDirectoryAbsolutePath": "c:\\users\\bill\\appdata\\local\\microsoft\\visualstudio\\17.0_03c65567\\extensions\\atkxhose.05t\\Images",
"ExpandToIDE": false,
"Extensions": ".png, .jpg, .gif, .bmp",
"ImageBackgroundType": 0,
"ImageFadeAnimationInterval": "PT5S",
"ImageStretch": 0,
"IsLimitToMainlyEditorWindow": false,
"LoopSlideshow": true,
"MaxHeight": 0,
"MaxWidth": 0,
"Opacity": 0.35,
"PositionHorizon": 1,
"PositionVertical": 1,
"ShuffleSlideshow": false,
"SoftEdgeX": 0,
"SoftEdgeY": 0,
"TileMode": 0,
"UpdateImageInterval": "PT1M",
"ViewBoxPointX": 0,
"ViewBoxPointY": 0,
"ViewPortHeight": 1,
"ViewPortPointX": 0,
"ViewPortPointY": 0,
"ViewPortWidth": 1
}
server/libaquery.h
+13 -2
View File
@@ -10,10 +10,21 @@ enum Log_level {
LOG_SILENT
};
struct Config{
int running, new_query, server_mode, n_buffers;
int buffer_sizes[];
};
struct Context{
typedef int (*printf_type) (const char *format, ...);
std::unordered_map<const char*, void*> tables;
std::unordered_map<const char*, uColRef *> cols;
Config* cfg;
int n_buffers, *sz_bufs;
void **buffers;
Log_level log_level = LOG_SILENT;
printf_type print = printf;
template <class ...Types>
@@ -28,8 +39,8 @@ struct Context{
}
};
#ifdef _MSC_VER
#define __DLLEXPORT__ __declspec(dllexport) __stdcall
#ifdef _WIN32
#define __DLLEXPORT__ __declspec(dllexport) __stdcall
#else
#define __DLLEXPORT__
#endif
server/server.cpp
+34 -5
View File
@@ -36,13 +36,43 @@ void daemon(thread_context* c) {
#include "aggregations.h"
typedef int (*code_snippet)(void*);
int _main();
int main(int argc, char** argv) {
int dll_main(int argc, char** argv, Context* cxt){
Config *cfg = reinterpret_cast<Config *>(argv[0]);
auto buf_szs = cfg->buffer_sizes;
void** buffers = (void**)malloc(sizeof(void*) * cfg->n_buffers);
for (int i = 0; i < cfg->n_buffers; i++)
buffers[i] = static_cast<void *>(argv[i + 1]);
cxt->buffers = buffers;
cxt->cfg = cfg;
cxt->n_buffers = cfg->n_buffers;
cxt->sz_bufs = buf_szs;
while(cfg->running){
if (cfg->new_query) {
void* handle = dlopen("d:/gg/Aquery++/dll.so", RTLD_LAZY);
code_snippet c = reinterpret_cast<code_snippet>(dlsym(handle, "dllmain"));
c(cxt);
dlclose(handle);
cfg->new_query = 0;
}
std::this_thread::sleep_for(std::chrono::milliseconds(100));
}
return 0;
}
extern "C" int __DLLEXPORT__ main(int argc, char** argv) {
Context* cxt = new Context();
cxt->log("%d %s\n", argc, argv[1]);
const char* shmname;
if (argc <= 1)
return _main();
if (argc < 0)
return dll_main(argc, argv, cxt);
else if (argc <= 1)
return test_main();
else
shmname = argv[1];
SharedMemory shm = SharedMemory(shmname);
@@ -68,7 +98,6 @@ int main(int argc, char** argv) {
cxt->log("inner\n");
cxt->err("return: %d\n", c(cxt));
}
dlclose(handle);
}
ready = false;
}
@@ -77,7 +106,7 @@ int main(int argc, char** argv) {
return 0;
}
#include "utils.h"
int _main()
int test_main()
{
//vector_type<int> t;
//t = 1;