update

engine/ast.py

@@ -4,13 +4,15 @@ from engine.utils import base62uuid
 
 # replace column info with this later.
 class ColRef:
-    def __init__(self, k9name, type, cobj, cnt, table):
+    def __init__(self, k9name, _ty, cobj, cnt, table, name, id):
         self.k9name = k9name
-        self.type = type
+        self.type = _ty
         self.cobj = cobj
         self.cnt = cnt
         self.table = table
-        self.__arr__ = (k9name, type, cobj, cnt, table)
+        self.name = name
+        self.id = id
+        self.__arr__ = (k9name, _ty, cobj, cnt, table, name, id)
         
     def __getitem__(self, key):
         return self.__arr__[key]
@@ -28,6 +30,7 @@ class TableInfo:
         self.columns = []
         self.cxt = cxt
         self.views = set()
+        self.rec = None 
         for c in cols:
             self.add_col(c)
 
@@ -48,7 +51,7 @@ class TableInfo:
         #     root.cnt += 1
 
         # column: (k9name, type, original col_object, dup_count)
-        col_object =  ColRef(k9name, (list(c['type'].keys()))[0], c, 1, self)
+        col_object =  ColRef(k9name, (list(c['type'].keys()))[0], c, 1, self,c['name'], len(self.columns))
 
         self.cxt.k9cols_byname[k9name] = col_object
         self.columns_byname[c['name']] = col_object
@@ -62,7 +65,11 @@ class TableInfo:
         return len(self.columns)
 
     def get_k9colname(self, col_name):
-        return self.columns_byname[col_name].k9name
+        col = self.columns_byname[col_name]
+        if type(self.rec) is list:
+            self.rec.append(col)
+        return col.k9name
+        
     def add_alias(self, alias):
         # TODO: Exception when alias already defined.
         # TODO: Scoping of alias should be constrainted in the query.
@@ -158,5 +165,5 @@ class ast_node:
 def include(objs):
     import inspect
     for _, cls in inspect.getmembers(objs):
-        if inspect.isclass(cls) and issubclass(cls, ast_node):
+        if inspect.isclass(cls) and issubclass(cls, ast_node) and not cls.name.startswith('_'):
             ast_node.types[cls.name] = cls

engine/ddl.py

@@ -1,7 +1,7 @@
 # code-gen for data decl languages
 
-from engine.ast import TableInfo, ast_node, include
-
+from engine.ast import ColRef, TableInfo, ast_node, include
+from engine.utils import base62uuid
 class create_table(ast_node):
     name = 'create_table'
     def produce(self, node):
@@ -27,6 +27,34 @@ class insert(ast_node):
             else:
                 # subquery, dispatch to select astnode
                 pass
+            
+class k9(ast_node):
+    name='k9'
+    def produce(self, node):
+        self.emit(node[self.name])
         
+class load(ast_node):
+    name="load"
+    def produce(self, node):
+        node = node[self.name]
+        tablename = 'l'+base62uuid(7)
+        keys = 'k'+base62uuid(7)
+        self.emit(f"{tablename}:`csv ? 1:\"{node['file']['literal']}\"")
+        self.emit(f"{keys}:!{tablename}")
+        table:TableInfo = self.context.tables_byname[node['table']]
+        
+        for i, c in enumerate(table.columns):
+            c:ColRef
+            self.emit(f'{c.k9name}:{tablename}[({keys})[{i}]]')
+            
+class outfile(ast_node):
+    name="_outfile"
+    def produce(self, node):
+        out_table:TableInfo = self.parent.out_table
+        self.emit_no_ln(f"\"{node['loc']['literal']}\"1:`csv@[[]")
+        for i, c in enumerate(out_table.columns):
+            self.emit_no_ln(f"{c.name}:{c.k9name}{';' if i < len(out_table.columns) - 1 else ''}")
+        self.emit(']')
+            
 import sys
 include(sys.modules[__name__])

engine/expr.py

@@ -6,10 +6,12 @@ class expr(ast_node):
     builtin_func_maps = {
         'max': 'max',
         'min': 'min', 
-        'avg':'avg',
-        'sum':'sum',
-        'mins': 'mins',
-        'maxs': 'maxs'
+        'avg': 'avg',
+        'sum': 'sum',
+        'mins': ['mins', 'minsw'],
+        'maxs': ['maxs', 'maxsw'],
+        'avgs': ['avgs', 'avgsw'],
+        'sums': ['sums', 'sumsw'],
     }
     binary_ops = {
         'sub':'-', 
@@ -22,6 +24,10 @@ class expr(ast_node):
         'gt':'>',
         'lt':'<',
     }
+    compound_ops = {
+        'ge' : [2, lambda x: f'~({x[0]}<{x[1]})'],
+        'le' : [2, lambda x: f'~({x[0]}>{x[1]})'],
+    }
     unary_ops = {
         'neg' : '-',
         'not' : '~'
@@ -45,19 +51,32 @@ class expr(ast_node):
         if type(node) is dict:
             for key, val in node.items():
                 if key in self.func_maps:
-                    self.k9expr += f"{self.func_maps[key]}(" 
                     # if type(val) in [dict, str]:
-                    self.k9expr += expr(self, val).k9expr
-                    self.k9expr += ')'
+                    if type(val) is list and len(val) > 1:
+                        k9func = self.func_maps[key]
+                        k9func = k9func[len(val) - 1] if type(k9func) is list else k9func
+                        self.k9expr += f"{k9func}[" 
+                        for i, p in enumerate(val):
+                            self.k9expr += expr(self, p).k9expr + (';'if i<len(val)-1 else '')
+                    else:
+                        self.k9expr += f"{self.func_maps[key]}[" 
+                        self.k9expr += expr(self, val).k9expr
+                    self.k9expr += ']'
                 elif key in self.binary_ops:
                     l = expr(self, val[0]).k9expr
                     r = expr(self, val[1]).k9expr
                     self.k9expr += f'({l}{self.binary_ops[key]}{r})'
-                    
+                elif key in self.compound_ops:
+                    x = []
+                    if type(val) is list:
+                        for v in val:
+                            x.append(expr(self, v).k9expr)
+                    self.k9expr = self.compound_ops[key][1](x)
                 elif key in self.unary_ops:
                     self.k9expr += f'({expr(self, val).k9expr}{self.unary_ops[key]})'
                 else:
                     print(f'Undefined expr: {key}{val}')
+                    
         elif type(node) is str:
             p = self.parent
             while type(p) is expr and not p.isvector:

engine/groupby.py

@@ -0,0 +1,37 @@
+from engine.ast import ast_node
+from engine.utils import base62uuid
+from engine.expr import expr
+
+class groupby(ast_node):
+    name = '_groupby'
+    def init(self, _):
+        self.group = 'g' + base62uuid(7)
+        self.datasource = self.parent.datasource
+        self.datasource.rec = []
+    def produce(self, node):
+        if type(node) is not list:
+            node = [node]
+        g_contents = '('
+        
+        for i, g in enumerate(node):
+            v = g['value']
+            e = expr(self, v).k9expr
+            # if v is compound expr, create tmp cols
+            if type(v) is not str:
+                tmpcol = 't' + base62uuid(7)
+                self.emit(f'{tmpcol}:{e}')
+                e = tmpcol
+
+            g_contents += e + (';'if i < len(node)-1 else '')
+            
+        self.emit(f'{self.group}:'+g_contents+')')
+        
+        if len(node) <= 1:
+            self.emit(f'{self.group}:={self.group}')
+        else:
+            self.emit(f'{self.group}:groupby[{self.group}[0];+{self.group}]')
+    
+    def consume(self, _):
+        self.referenced = self.datasource.rec
+        self.datasource.rec = None
+        return super().consume(_)

engine/projection.py

@@ -1,9 +1,10 @@
-from engine.ast import TableInfo, ast_node, Context, include
+from engine.ast import ColRef, TableInfo, ast_node, Context, include
+from engine.groupby import groupby
 from engine.join import join
 from engine.expr import expr
 from engine.scan import filter
-from engine.utils import base62uuid
-
+from engine.utils import base62uuid, enlist, base62alp
+from engine.ddl import outfile
 class projection(ast_node):
     name='select'
     def __init__(self, parent:ast_node, node, context:Context = None, outname = None, disp = True):
@@ -35,12 +36,14 @@ class projection(ast_node):
                             projection(self, from_clause, disp = False)
                         else:
                             # TODO: from func over table
-                            print(f"from func over table{node}")
+                            print(f'from func over table{node}')
                     elif type(value) is str:
                         self.datasource = self.context.tables_byname[value]
                 if 'assumptions' in from_clause:
-                    ord = from_clause['assumptions']['ord'] == 'asc'
-                    ord = '^' if ord else '|^'
+                    for assumption in enlist(from_clause['assumptions']):
+                        ord = assumption['ord'] == 'asc'
+                        attrib = assumption['attrib']
+                        ord = '^' if ord else '|^'
                     # TODO: generate view of table by order
 
             elif type(from_clause) is str:
@@ -57,26 +60,70 @@ class projection(ast_node):
             self.datasource = filter(self, node['where'], True).output
             self.context.datasource = self.datasource            
 
-
-    def consume(self, _):
+        if 'groupby' in node:
+            self.group_node = groupby(self, node['groupby'])
+        else:
+            self.group_node = None
+            
+    def consume(self, node):
         disp_varname = 'd'+base62uuid(7)
+        pcolrefs = []
+        if type(self.group_node) is groupby:
+            grp_table = self.group_node.group
+            grp_refs = self.group_node.referenced
+            for i, proj in enumerate(self.projections):
+                self.datasource.rec = []
+                cname = ''
+                if type(proj) is dict:
+                    if 'value' in proj:
+                        e = proj['value']
+                        if type(e) is str:
+                            cname = self.datasource.parse_tablenames(proj['value'])
+                        elif type(e) is dict:
+                            cname = expr(self, e).k9expr
+                            cname = ''.join([a if a in base62alp else '' for a in cname])
+                pcolrefs.append(self.datasource.rec)
+                self.datasource.rec = None
+            keys = 'k'+base62uuid(7)
+            self.emit(f'{keys}:!{grp_table}')
+            fn = 'fn' + base62uuid(6)
+            # self.emit
+        
         self.emit_no_ln(f'{disp_varname}:(')
+        flatten = False
+        cols = []
+        self.out_table = TableInfo('out_'+base62uuid(4), [], self.context)
+        if 'outfile' in node:
+            flatten = True
+            
         for i, proj in enumerate(self.projections):
+            cname = ''
             if type(proj) is dict:
                 if 'value' in proj:
                     e = proj['value']
                     if type(e) is str:
-                        self.emit_no_ln(f"{self.datasource.parse_tablenames(proj['value'])}")
+                        cname = self.datasource.parse_tablenames(proj['value'])
+                        self.emit_no_ln(f"{cname}")
                     elif type(e) is dict:
-                        self.emit_no_ln(f"{expr(self, e).k9expr}")
+                        cname = expr(self, e).k9expr
+                        self.emit_no_ln(f"{cname}")
+                        cname = ''.join([a if a in base62alp else '' for a in cname])
                     self.emit_no_ln(';'if i < len(self.projections)-1 else '')
-
+            cols.append(ColRef(f'(+{disp_varname})[{i}]', 'generic', self.out_table, 0, None, cname, i))
         self.emit(')')
-        if self.disp:
+        if flatten:
+            self.emit_no_ln(f'{disp_varname}:' if flatten else '')
+            
+        if flatten or self.disp:
             if len(self.projections) > 1:
-                self.emit(f'+{disp_varname}')
+                self.emit(f"+{disp_varname}")
             else:
                 self.emit(f'+,(,{disp_varname})')
+            if flatten:
+                self.emit(f'{disp_varname}')
+        if flatten:
+            self.out_table.columns = cols
+            outfile(self, node['outfile'])
         if self.datasource_changed:
             self.context.datasource = self.prev_datasource
 

engine/utils.py

@@ -1,12 +1,19 @@
 import uuid
 
+base62alp = '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
+
 def base62uuid(crop=8):
-    alp = '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
     id = uuid.uuid4().int
     ret = ''
     
     while id:
-        ret = alp[id % 62] + ret
+        ret = base62alp[id % 62] + ret
         id //= 62
     
-    return ret[:crop] if len(ret) else '0'
+    return ret[:crop] if len(ret) else '0'
+
+def enlist(l): 
+    return l if type(l) is list else [l]
+
+def seps(s, i, l):
+    return s if i < len(l) - 1 else ''