updated code for groupby and windowed aggregations

engine/ast.py

@@ -4,7 +4,7 @@ from engine.utils import base62uuid
 
 # replace column info with this later.
 class ColRef:
-    def __init__(self, k9name, _ty, cobj, cnt, table, name, id):
+    def __init__(self, k9name, _ty, cobj, cnt, table, name, id, order = None, compound = False):
         self.k9name = k9name
         self.type = _ty
         self.cobj = cobj
@@ -12,6 +12,9 @@ class ColRef:
         self.table = table
         self.name = name
         self.id = id
+        self.order = order # True -> asc, False -> dsc; None -> unordered
+        self.compound = compound # compound field (list as a field) 
+        self.views = []
         self.__arr__ = (k9name, _ty, cobj, cnt, table, name, id)
         
     def __getitem__(self, key):
@@ -31,6 +34,7 @@ class TableInfo:
         self.cxt = cxt
         self.views = set()
         self.rec = None 
+        self.groupinfo = None
         for c in cols:
             self.add_col(c)
 
@@ -44,13 +48,6 @@ class TableInfo:
         if type(c) is ColRef:
             c = c.cobj
         k9name = 'c' + base62uuid(7)
-        # k9name = self.table_name + c['name']
-        # if k9name in self.cxt.k9cols_byname: # duplicate names?
-        #     root = self.cxt.k9cols_byname[k9name] 
-        #     k9name = k9name + root.cnt
-        #     root.cnt += 1
-
-        # column: (k9name, type, original col_object, dup_count)
         col_object =  ColRef(k9name, (list(c['type'].keys()))[0], c, 1, self,c['name'], len(self.columns))
 
         self.cxt.k9cols_byname[k9name] = col_object

engine/ddl.py

@@ -37,15 +37,18 @@ class load(ast_node):
     name="load"
     def produce(self, node):
         node = node[self.name]
-        tablename = 'l'+base62uuid(7)
-        keys = 'k'+base62uuid(7)
-        self.emit(f"{tablename}:`csv ? 1:\"{node['file']['literal']}\"")
-        self.emit(f"{keys}:!{tablename}")
         table:TableInfo = self.context.tables_byname[node['table']]
-        
+        n_keys = len(table.columns)
+        keys = ''
+        for _ in n_keys:
+            keys+='`tk'+base62uuid(6)
+        tablename = 'l'+base62uuid(7)        
+
+        self.emit(f"{tablename}:[{keys}!+(`csv ? 1:\"{node['file']['literal']}\")][{keys}]")
+
         for i, c in enumerate(table.columns):
             c:ColRef
-            self.emit(f'{c.k9name}:{tablename}[({keys})[{i}]]')
+            self.emit(f'{c.k9name}:{tablename}[{i}]')
             
 class outfile(ast_node):
     name="_outfile"

engine/expr.py

@@ -8,17 +8,17 @@ class expr(ast_node):
         'min': 'min', 
         'avg': 'avg',
         'sum': 'sum',
+        'mod':'mod',
         'mins': ['mins', 'minsw'],
         'maxs': ['maxs', 'maxsw'],
         'avgs': ['avgs', 'avgsw'],
         'sums': ['sums', 'sumsw'],
     }
     binary_ops = {
-        'sub':'-', 
+        'sub':'-',  
         'add':'+', 
         'mul':'*', 
         'div':'%',
-        'mod':'mod',
         'and':'&',
         'or':'|',
         'gt':'>',

engine/groupby.py

@@ -12,7 +12,7 @@ class groupby(ast_node):
         if type(node) is not list:
             node = [node]
         g_contents = '('
-        
+        first_col = ''
         for i, g in enumerate(node):
             v = g['value']
             e = expr(self, v).k9expr
@@ -21,7 +21,8 @@ class groupby(ast_node):
                 tmpcol = 't' + base62uuid(7)
                 self.emit(f'{tmpcol}:{e}')
                 e = tmpcol
-
+            if i == 0:
+                first_col = e
             g_contents += e + (';'if i < len(node)-1 else '')
             
         self.emit(f'{self.group}:'+g_contents+')')
@@ -29,8 +30,8 @@ class groupby(ast_node):
         if len(node) <= 1:
             self.emit(f'{self.group}:={self.group}')
         else:
-            self.emit(f'{self.group}:groupby[{self.group}[0];+{self.group}]')
-    
+            self.emit(f'{self.group}:groupby[+({self.group},(,!(#({first_col}))))]')
+        
     def consume(self, _):
         self.referenced = self.datasource.rec
         self.datasource.rec = None

engine/projection.py

@@ -5,6 +5,8 @@ from engine.expr import expr
 from engine.scan import filter
 from engine.utils import base62uuid, enlist, base62alp
 from engine.ddl import outfile
+import copy
+
 class projection(ast_node):
     name='select'
     def __init__(self, parent:ast_node, node, context:Context = None, outname = None, disp = True):
@@ -62,6 +64,8 @@ class projection(ast_node):
 
         if 'groupby' in node:
             self.group_node = groupby(self, node['groupby'])
+            self.datasource = copy(self.datasource) # shallow copy
+            self.datasource.groupinfo = self.group_node
         else:
             self.group_node = None