bill/AQuery
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

update

Browse Source
This commit is contained in:
bill
2022-04-29 04:53:23 +08:00
parent d4d11286d5
commit 15b124e2e6
12 changed files with 231 additions and 133 deletions
Download Patch File Download Diff File Expand all files Collapse all files
engine/ast.py
+1 -1
View File
@@ -92,7 +92,7 @@ class TableInfo:
type_tags = type_tags[:-1]
type_tags += '>'
self.cxt.emit(f'auto& {base_name} = *(TableInfo{type_tags} *)(cxt->tables[{self.table_name}]);')
self.cxt.emit(f'auto& {base_name} = *(TableInfo{type_tags} *)(cxt->tables["{self.table_name}"]);')
return self.cxt_name
def refer_all(self):
self.reference()
engine/ddl.py
+22 -4
View File
@@ -1,15 +1,17 @@
# code-gen for data decl languages
from engine.orderby import orderby
from engine.ast import ColRef, TableInfo, ast_node, Context, include
from engine.scan import scan
from engine.utils import base62uuid
class create_table(ast_node):
name = 'create_table'
def __init__(self, parent: "ast_node", node, context: Context = None, cexprs = None):
def __init__(self, parent: "ast_node", node, context: Context = None, cexprs = None, lineage = False):
self.cexprs = cexprs
self.lineage = lineage
super().__init__(parent, node, context)
def produce(self, node):
def produce(self, node):
if type(node) is not TableInfo:
ct = node[self.name]
tbl = self.context.add_table(ct['name'], ct['columns'])
@@ -29,16 +31,32 @@ class create_table(ast_node):
self.emit(f"{c.cxt_name}.init();")
# create an output table
else:
# 1 to 1 lineage.
if len(self.context.scans) == 0:
if self.lineage:
order = 'order_' + base62uuid(6)
self.emit(f'auto {order} = {self.parent.datasource.cxt_name}->order_by<{orderby(self.parent, self.parent.assumptions).result()}>();')
self.lineage = '*' + order
else:
self.lineage = None
for i, c in enumerate(tbl.columns):
self.emit(f"{c.cxt_name}.init();")
self.emit(f"{c.cxt_name} = {self.cexprs[i]()};")
self.emit(f"{c.cxt_name} = {self.cexprs[i](self.lineage)};")
self.lineage = None
self.parent.assumptions = None
else:
scanner:scan = self.context.scans[-1]
if self.lineage:
lineage_var = 'lineage_' + base62uuid(6)
counter_var = 'counter_' + base62uuid(6)
scanner.add(f'auto {lineage_var} = {self.datasource.cxt_name}->bind({tbl.cxt_name});', "init")
scanner.add(f'auto {counter_var} = 0;', "init")
scanner.add(f"{lineage_var}.emplace_back({counter_var}++);", "front")
self.lineage = f"{lineage_var}.rid"
for i, c in enumerate(tbl.columns):
scanner.add(f"{c.cxt_name}.init();", "init")
scanner.add(f"{c.cxt_name} = {self.cexprs[i](scanner.it_ver)};")
class insert(ast_node):
name = 'insert'
def produce(self, node):
engine/groupby.py
+16 -5
View File
@@ -1,8 +1,9 @@
from engine.ast import TableInfo, ast_node
from engine.ast import ColRef, TableInfo, ast_node
from engine.orderby import assumption
from engine.scan import scan
from engine.utils import base62uuid
from engine.expr import expr
import engine.types
class groupby(ast_node):
name = '_groupby'
def init(self, _):
@@ -24,7 +25,8 @@ class groupby(ast_node):
for i, g in enumerate(node):
v = g['value']
e = expr(self, v)
self.raw_groups.append(e.raw_col)
if type(e.raw_col) is ColRef:
self.raw_groups.append(e.raw_col)
e = e._expr
# if v is compound expr, create tmp cols
if type(v) is not str:
@@ -48,7 +50,14 @@ class groupby(ast_node):
self.referenced = self.datasource.rec
self.datasource.rec = None
self.scanner.finalize()
def deal_with_assumptions(self, assumption:assumption, out:TableInfo):
gscanner = scan(self, self.group)
val_var = 'val_'+base62uuid(7)
gscanner.add(f'auto &{val_var} = {gscanner.it_ver}.second;')
gscanner.add(f'{out.cxt_name}->order_by<{assumption.result()}>(&{val_var});')
gscanner.finalize()
def finalize(self, cexprs, out:TableInfo):
gscanner = scan(self, self.group)
key_var = 'key_'+base62uuid(7)
@@ -58,4 +67,6 @@ class groupby(ast_node):
gscanner.add(f'auto &{val_var} = {gscanner.it_ver}.second;')
gscanner.add(';\n'.join([f'{out.columns[i].reference()}.emplace_back({ce(x=val_var, y=key_var)})' for i, ce in enumerate(cexprs)])+';')
gscanner.finalize()
gscanner.finalize()
self.datasource.groupinfo = None
engine/orderby.py
+27 -3
View File
@@ -35,7 +35,31 @@ class orderby(ast_node):
for n in node:
order = not ('sort' in n and n['sort'] == 'desc')
col_id = self.datasource.columns_byname[n['value']].id
self.col_list.append(col_id if order else -col_id-1)
self.order.append(order_item(n['value'], self, order))
col_id = col_id if order else -col_id-1
if col_id not in self.col_list:
self.col_list.append(col_id)
self.order.append(order_item(n['value'], self, order))
def merge(self, node):
self.produce(node)
def finialize(self, references):
self.order = [ o for o in self.order if o.name in references ]
self.order = [ o for o in self.order if o.name in references ]
def result(self, sep:str = ','):
return sep.join([f"{c}" for c in self.col_list])
class assumption(orderby):
name = '_assumption'
def __init__(self, parent: "ast_node", node, context: Context = None, exclude = []):
self.exclude = exclude
super().__init__(parent, node, context)
def produce(self, node):
if type(node) is not list:
node = [node]
[n for n in node if n not in self.exclude]
return super().produce(node)
def empty(self):
return len(self.col_list) == 0
engine/projection.py
+43 -18
View File
@@ -2,7 +2,7 @@ from engine.ast import ColRef, TableInfo, ast_node, Context, include
from engine.groupby import groupby
from engine.join import join
from engine.expr import expr
from engine.orderby import orderby
from engine.orderby import assumption, orderby
from engine.scan import filter
from engine.utils import base62uuid, enlist, base62alp, has_other
from engine.ddl import create_table, outfile
@@ -14,7 +14,7 @@ class projection(ast_node):
self.disp = disp
self.outname = outname
self.group_node = None
self.assumption = None
self.assumptions = None
self.where = None
ast_node.__init__(self, parent, node, context)
def init(self, _):
@@ -46,7 +46,7 @@ class projection(ast_node):
elif type(value) is str:
self.datasource = self.context.tables_byname[value]
if 'assumptions' in from_clause:
self.assumption = enlist(from_clause['assumptions'])
self.assumptions = enlist(from_clause['assumptions'])
elif type(from_clause) is str:
self.datasource = self.context.tables_byname[from_clause]
@@ -61,7 +61,7 @@ class projection(ast_node):
if 'where' in node:
self.where = filter(self, node['where'], True)
# self.datasource = filter(self, node['where'], True).output
#self.context.datasource = self.datasource
# self.context.datasource = self.datasource
if 'groupby' in node:
self.group_node = groupby(self, node['groupby'])
@@ -73,10 +73,7 @@ class projection(ast_node):
def consume(self, node):
self.inv = True
disp_varname = 'd'+base62uuid(7)
has_groupby = False
if self.group_node is not None:
# There is group by;
has_groupby = True
has_groupby = self.group_node is not None
cexprs = []
flatten = False
cols = []
@@ -85,6 +82,7 @@ class projection(ast_node):
flatten = True
new_names = []
proj_raw_cols = []
for i, proj in enumerate(self.projections):
cname = ''
compound = False
@@ -92,7 +90,10 @@ class projection(ast_node):
if type(proj) is dict:
if 'value' in proj:
e = proj['value']
sname = expr(self, e)._expr
sname = expr(self, e)
if type(sname.raw_col) is ColRef:
proj_raw_cols.append(sname.raw_col)
sname = sname._expr
fname = expr.toCExpr(sname) # fastest access method at innermost context
absname = expr(self, e, abs_col=True)._expr # absolute name at function scope
# TODO: Make it single pass here.
@@ -118,26 +119,50 @@ class projection(ast_node):
self.out_table.add_cols(cols, False)
lineage = None
if has_groupby:
create_table(self, self.out_table) # creates empty out_table.
if self.assumptions is not None:
self.assumptions = assumption(self, self.assumptions, exclude=self.group_node.raw_groups)
if not self.assumptions.empty():
self.group_node.deal_with_assumptions(self.assumptions, self.out_table)
self.assumptions = None
self.group_node.finalize(cexprs, self.out_table)
else:
create_table(self, self.out_table, cexprs = cexprs) # create and populate out_table.
self.datasource.group_node = None
# if all assumptions in projections, treat as orderby
lineage = self.assumptions is not None and has_other(self.assumptions, proj_raw_cols)
spawn = create_table(self, self.out_table, cexprs = cexprs, lineage = lineage) # create and populate out_table.
if lineage and type(spawn.lineage) is str:
lineage = spawn.lineage
self.assumptions = orderby(self, self.assumptions) # do not exclude proj_raw_cols
else:
lineage = None
if self.where is not None:
self.where.finalize()
has_orderby = 'orderby' in node
if has_orderby:
if type(lineage) is str:
order = 'order_' + base62uuid(6)
self.emit(f'auto {order} = {self.datasource.cxt_name}->order_by<{self.assumptions.result()}>({lineage});')
self.emit(f'{self.out_table.cxt_name}->materialize(*{order});')
self.assumptions = None
if self.assumptions is not None:
orderby_node = orderby(self, self.assumptions)
else:
orderby_node = None
if 'orderby' in node:
self.datasource = self.out_table
self.context.datasource = self.out_table # discard current ds
orderby_node = orderby(self, node['orderby'])
self.emit(f'auto {disp_varname} = {self.out_table.reference()}->order_by_view<{",".join([f"{c}" for c in orderby_node.col_list])}>();')
orderbys = node['orderby']
orderby_node = orderby(self, orderbys) if orderby_node is None else orderby_node.merge(orderbys)
if orderby_node is not None:
self.emit(f'auto {disp_varname} = {self.out_table.reference()}->order_by_view<{orderby_node.result()}>();')
else:
disp_varname = f'*{self.out_table.cxt_name}'
if self.disp:
self.emit(f'print({disp_varname});')
engine/scan.py
+10 -5
View File
@@ -5,14 +5,16 @@ from engine.expr import expr
class scan(ast_node):
name = 'scan'
def __init__(self, parent: "ast_node", node, size = None, context: Context = None):
def __init__(self, parent: "ast_node", node, size = None, context: Context = None, const = False):
self.type = type
self.size = size
self.const = "const " if const else ""
super().__init__(parent, node, context)
def init(self, _):
self.datasource = self.context.datasource
self.initializers = ''
self.start = ''
self.front = ''
self.body = ''
self.end = '}'
self.mode = None
@@ -27,13 +29,13 @@ class scan(ast_node):
self.colref = node
if self.size is None:
self.mode = ["col", node.table]
self.start += f'for (auto& {self.it_ver} : {node.reference()}) {{\n'
self.start += f'for ({self.const}auto& {self.it_ver} : {node.reference()}) {{\n'
else:
self.mode = ["idx", node.table]
self.start += f"for (uint32_t {self.it_ver} = 0; {self.it_ver} < {node.reference()}.size; ++{self.it_ver}){{\\n"
elif type(node) is str:
self.mode = ["idx", None]
self.start+= f'for(auto& {self.it_ver} : {node}) {{\n'
self.start+= f'for({self.const}auto& {self.it_ver} : {node}) {{\n'
else:
self.mode = ["idx", node] # Node is the TableInfo
self.start += f"for (uint32_t {self.it_ver} = 0; {self.it_ver} < {self.size}; ++{self.it_ver}){{\n"
@@ -41,13 +43,16 @@ class scan(ast_node):
def add(self, stmt, position = "body"):
if position == "body":
self.body += stmt + '\n'
else:
elif position == "init":
self.initializers += stmt + '\n'
else:
self.front += stmt + '\n'
def finalize(self):
for f in self.filters:
self.start += f
self.end += '}'
self.context.remove_scan(self, self.initializers + self.start + self.body + self.end)
self.context.remove_scan(self, self.initializers + self.start + self.front + self.body + self.end)
class filter(ast_node):
name = 'filter'