bill/AQuery
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Group by

Browse Source 
Flatten
Rewritten Agg funcs and csv I/O
bug fixes
This commit is contained in:
BillSun
2022-02-11 08:06:42 -05:00
parent 7eac7837a3
commit b9a8ad3ac7
16 changed files with 285 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files
engine/ast.py
+17 -4
View File
@@ -1,5 +1,7 @@
from typing import List
from pyparsing import col
from engine.utils import base62uuid
# replace column info with this later.
@@ -23,6 +25,9 @@ class ColRef:
def __setitem__(self, key, value):
self.__arr__[key] = value
def __str__(self):
return self.k9name
class TableInfo:
def __init__(self, table_name, cols, cxt:'Context'):
@@ -61,11 +66,13 @@ class TableInfo:
def n_cols(self):
return len(self.columns)
def get_k9colname(self, col_name):
def get_col(self, col_name):
col = self.columns_byname[col_name]
if type(self.rec) is list:
self.rec.append(col)
return col.k9name
return col
def get_k9colname(self, col_name):
return self.get_col(col_name).k9name
def add_alias(self, alias):
# TODO: Exception when alias already defined.
@@ -75,14 +82,20 @@ class TableInfo:
def parse_tablenames(self, colExpr):
parsedColExpr = colExpr.split('.')
ret = None
if len(parsedColExpr) <= 1:
return self.get_k9colname(colExpr)
ret = self.get_col(colExpr)
else:
datasource = self.cxt.tables_byname[parsedColExpr[0]]
if datasource is None:
raise ValueError(f'Table name/alias not defined{parsedColExpr[0]}')
else:
return datasource.get_k9colname(parsedColExpr[1])
ret = datasource.get_col(parsedColExpr[1])
if self.groupinfo is not None and ret:
ret = f"{ret.k9name}[{'start' if ret in self.groupinfo.referenced else 'range'}]"
else:
ret = ret.k9name
return ret
class View:
def __init__(self, context, table = None, tmp = True):
engine/ddl.py
+28 -6
View File
@@ -40,24 +40,46 @@ class load(ast_node):
table:TableInfo = self.context.tables_byname[node['table']]
n_keys = len(table.columns)
keys = ''
for _ in n_keys:
for _ in range(n_keys):
keys+='`tk'+base62uuid(6)
tablename = 'l'+base62uuid(7)
self.emit(f"{tablename}:[{keys}!+(`csv ? 1:\"{node['file']['literal']}\")][{keys}]")
self.emit(f"{tablename}:({keys}!(+(`csv ? 1:\"{node['file']['literal']}\")))[{keys}]")
for i, c in enumerate(table.columns):
c:ColRef
self.emit(f'{c.k9name}:{tablename}[{i}]')
class outfile(ast_node):
name="_outfile"
def produce(self, node):
out_table:TableInfo = self.parent.out_table
self.emit_no_ln(f"\"{node['loc']['literal']}\"1:`csv@[[]")
filename = node['loc']['literal'] if 'loc' in node else node['literal']
self.emit_no_ln(f"\"{filename}\"1:`csv@(+(")
l_compound = False
l_cols = ''
l_keys = ''
ending = lambda x: x[:-1] if len(x) > 0 and x[-1]==';' else x
for i, c in enumerate(out_table.columns):
self.emit_no_ln(f"{c.name}:{c.k9name}{';' if i < len(out_table.columns) - 1 else ''}")
self.emit(']')
c:ColRef
l_keys += '`' + c.name
if c.compound:
if l_compound:
l_cols=f'flatBOTH\'+(({ending(l_cols)});{c.k9name})'
else:
l_compound = True
if i >= 1:
l_cols = f'flatRO\'+(({ending(l_cols)});{c.k9name})'
else:
l_cols = c.k9name + ';'
elif l_compound:
l_cols = f'flatLO\'+(({ending(l_cols)});{c.k9name})'
else:
l_cols += f"{c.k9name};"
if not l_compound:
self.emit_no_ln(l_keys + '!(' + ending(l_cols) + ')')
else:
self.emit_no_ln(f'{l_keys}!+,/({ending(l_cols)})')
self.emit('))')
import sys
include(sys.modules[__name__])
engine/expr.py
+15 -2
View File
@@ -32,6 +32,9 @@ class expr(ast_node):
'neg' : '-',
'not' : '~'
}
coumpound_generating_ops = ['mod', 'mins', 'maxs', 'sums'] + \
list( binary_ops.keys()) + list(compound_ops.keys()) + list(unary_ops.keys() )
def __init__(self, parent, node):
ast_node.__init__(self, parent, node, None)
@@ -39,6 +42,7 @@ class expr(ast_node):
from engine.projection import projection
parent = self.parent
self.isvector = parent.isvector if type(parent) is expr else False
self.is_compound = parent.is_compound if type(parent) is expr else False
if type(parent) in [projection, expr]:
self.datasource = parent.datasource
else:
@@ -59,7 +63,9 @@ class expr(ast_node):
for i, p in enumerate(val):
self.k9expr += expr(self, p).k9expr + (';'if i<len(val)-1 else '')
else:
self.k9expr += f"{self.func_maps[key]}["
funcname = self.func_maps[key]
funcname = funcname[0] if type(funcname) is list else funcname
self.k9expr += f"{funcname}["
self.k9expr += expr(self, val).k9expr
self.k9expr += ']'
elif key in self.binary_ops:
@@ -76,7 +82,14 @@ class expr(ast_node):
self.k9expr += f'({expr(self, val).k9expr}{self.unary_ops[key]})'
else:
print(f'Undefined expr: {key}{val}')
if key in self.coumpound_generating_ops and not self.is_compound:
self.is_compound = True
p = self.parent
while type(p) is expr and not p.is_compound:
p.is_compound = True
p = p.parent
elif type(node) is str:
p = self.parent
while type(p) is expr and not p.isvector:
engine/groupby.py
+21 -4
View File
@@ -26,13 +26,30 @@ class groupby(ast_node):
g_contents += e + (';'if i < len(node)-1 else '')
self.emit(f'{self.group}:'+g_contents+')')
if len(node) <= 1:
self.n_grps = len(node)
if self.n_grps <= 1:
self.emit(f'{self.group}:={self.group}')
else:
self.emit(f'{self.group}:groupby[+({self.group},(,!(#({first_col}))))]')
self.emit(f'{self.group}:groupby[({self.group},(,!(#({first_col}))))]')
def consume(self, _):
self.referenced = self.datasource.rec
self.datasource.rec = None
return super().consume(_)
return super().consume(_)
def finalize(self, ret, out):
self.groupby_function = 'fgrp'+base62uuid(4)
grp = self.group
if self.n_grps <= 1:
k9fn = "{[range] start:*range;"+ ret + "}"
self.emit(f'{out}:(({k9fn}\'{grp})[!{grp}])')
self.parent.inv = False
else:
k9fn = "{[ids;grps;ll;dim;x] " + \
"start:$[x=ll;ll;grps[x+1][dim-1]];" + \
"end: grps[x][dim-1];" + \
"range:(end-start)#(((start-ll))#ids);" + \
ret + '}'
self.emit(f'{self.groupby_function}:{k9fn}')
self.emit(f'{out}:+({self.groupby_function}' + \
f'[{grp}[1];{grp}[0];(#{grp}[0])-1;#({grp}[0][0])]\'!((#({grp}[0]))-1))')
engine/orderby.py
+38
View File
@@ -0,0 +1,38 @@
from engine.ast import ColRef, TableInfo, ast_node
from engine.utils import base62uuid
from engine.expr import expr
class orderby(ast_node):
name = '_orderby'
def init(self, _):
self.group = 'g' + base62uuid(7)
self.datasource = self.parent.datasource
self.datasource.rec = []
def produce(self, node):
if type(node) is not list:
node = [node]
g_contents = '('
first_col = ''
for i, g in enumerate(node):
v = g['value']
e = expr(self, v).k9expr
# if v is compound expr, create tmp cols
if type(v) is not str:
tmpcol = 't' + base62uuid(7)
self.emit(f'{tmpcol}:{e}')
e = tmpcol
if i == 0:
first_col = e
g_contents += e + (';'if i < len(node)-1 else '')
self.emit(f'{self.group}:'+g_contents+')')
self.n_grps = len(node)
if self.n_grps <= 1:
self.emit(f'{self.group}:={self.group}')
else:
self.emit(f'{self.group}:groupby[+({self.group},(,!(#({first_col}))))]')
def consume(self, _):
self.referenced = self.datasource.rec
self.datasource.rec = None
return super().consume(_)
engine/projection.py
+30 -33
View File
@@ -12,6 +12,7 @@ class projection(ast_node):
def __init__(self, parent:ast_node, node, context:Context = None, outname = None, disp = True):
self.disp = disp
self.outname = outname
self.group_node = None
ast_node.__init__(self, parent, node, context)
def init(self, _):
if self.outname is None:
@@ -64,36 +65,19 @@ class projection(ast_node):
if 'groupby' in node:
self.group_node = groupby(self, node['groupby'])
self.datasource = copy(self.datasource) # shallow copy
self.datasource = copy.copy(self.datasource) # shallow copy
self.datasource.groupinfo = self.group_node
else:
self.group_node = None
def consume(self, node):
self.inv = True
disp_varname = 'd'+base62uuid(7)
pcolrefs = []
if type(self.group_node) is groupby:
grp_table = self.group_node.group
grp_refs = self.group_node.referenced
for i, proj in enumerate(self.projections):
self.datasource.rec = []
cname = ''
if type(proj) is dict:
if 'value' in proj:
e = proj['value']
if type(e) is str:
cname = self.datasource.parse_tablenames(proj['value'])
elif type(e) is dict:
cname = expr(self, e).k9expr
cname = ''.join([a if a in base62alp else '' for a in cname])
pcolrefs.append(self.datasource.rec)
self.datasource.rec = None
keys = 'k'+base62uuid(7)
self.emit(f'{keys}:!{grp_table}')
fn = 'fn' + base62uuid(6)
# self.emit
self.emit_no_ln(f'{disp_varname}:(')
has_groupby = False
if self.group_node is not None:
# There is group by;
has_groupby = True
k9expr = f'('
flatten = False
cols = []
self.out_table = TableInfo('out_'+base62uuid(4), [], self.context)
@@ -102,27 +86,40 @@ class projection(ast_node):
for i, proj in enumerate(self.projections):
cname = ''
compound = False
self.datasource.rec = []
if type(proj) is dict:
if 'value' in proj:
e = proj['value']
if type(e) is str:
cname = self.datasource.parse_tablenames(proj['value'])
self.emit_no_ln(f"{cname}")
k9expr += (f"{cname}")
elif type(e) is dict:
cname = expr(self, e).k9expr
self.emit_no_ln(f"{cname}")
cname = ''.join([a if a in base62alp else '' for a in cname])
self.emit_no_ln(';'if i < len(self.projections)-1 else '')
cols.append(ColRef(f'(+{disp_varname})[{i}]', 'generic', self.out_table, 0, None, cname, i))
self.emit(')')
p_expr = expr(self, e)
cname = p_expr.k9expr
compound = True
k9expr += f"{cname}"
cname = ''.join([a if a in base62alp else '' for a in cname])
k9expr += ';'if i < len(self.projections)-1 else ''
compound = compound and has_groupby and self.datasource.rec not in self.group_node.referenced
cols.append(ColRef(f'(+{disp_varname})[{i}]', 'generic', self.out_table, 0, None, cname, i, compound=compound))
k9expr += ')'
if has_groupby:
self.group_node.finalize(k9expr, disp_varname)
else:
self.emit(f'{disp_varname}:{k9expr}')
self.datasource.group_node = None
if flatten:
self.emit_no_ln(f'{disp_varname}:' if flatten else '')
if flatten or self.disp:
if len(self.projections) > 1:
self.emit(f"+{disp_varname}")
self.emit(f"{'+' if self.inv else ''}{disp_varname}")
else:
self.emit(f'+,(,{disp_varname})')
self.emit(f'$[(#{disp_varname})>1;+,({disp_varname});+,(,{disp_varname})]')
if flatten:
self.emit(f'{disp_varname}')
if flatten: