Bug fixes on assumptions.

Other improvements.

engine/ast.py

@@ -15,6 +15,9 @@ class ColRef:
         self.order_pending = None # order_pending
         self.compound = compound # compound field (list as a field) 
         self.views = []
+        self.aux_columns = [] # columns for temperary calculations 
+        # e.g. order by, group by, filter by expressions
+        
         self.__arr__ = (cname, _ty, cobj, cnt, table, name, id)
     
     def reference(self):
@@ -89,7 +92,8 @@ class TableInfo:
                 type_tags = type_tags[:-1]
             type_tags += '>'
             
-            self.cxt.emit(f'auto& {base_name} = *(TableInfo{type_tags} *)(cxt->tables[{self.table_name}]);')
+            self.cxt.emit(f'auto& {base_name} = *(TableInfo{type_tags} *)(cxt->tables["{self.table_name}"]);')
+        return self.cxt_name
     def refer_all(self):
         self.reference()
         for c in self.columns:
@@ -110,7 +114,10 @@ class TableInfo:
         self.cxt.ccols_byname[cname] = col_object
         self.columns_byname[c['name']] = col_object
         self.columns.append(col_object)
-        
+    def get_size(self):
+        size_tmp = 'tmp_sz_'+base62uuid(6)
+        self.cxt.emit(f'const auto& {size_tmp} = {self.columns[0].reference()}.size;')
+        return size_tmp
     @property
     def n_cols(self):
         return len(self.columns)
@@ -136,8 +143,8 @@ class TableInfo:
 
     def get_col_d(self, col_name):
         col = self.columns_byname[col_name]
-        if type(self.rec) is list:
-            self.rec.append(col)
+        if type(self.rec) is set:
+            self.rec.add(col)
         return col
 
     def get_ccolname_d(self, col_name):
@@ -160,12 +167,12 @@ class TableInfo:
         self.alias.add(alias)
         
     def parse_tablenames(self, colExpr, materialize = True, raw = False):
-        self.get_col = self.get_col if materialize else self.get_col_d
+        # get_col = self.get_col if materialize else self.get_col_d
 
         parsedColExpr = colExpr.split('.')
         ret = None
         if len(parsedColExpr) <= 1:
-            ret = self.get_col(colExpr)
+            ret = self.get_col_d(colExpr)
         else: 
             datasource = self.cxt.tables_byname[parsedColExpr[0]]
             if datasource is None:
@@ -177,6 +184,7 @@ class TableInfo:
         if self.groupinfo is not None and ret and ret in self.groupinfo.raw_groups:
             string = f'get<{self.groupinfo.raw_groups.index(ret)}>({{y}})'
         return string, ret if raw else string
+
 class View:
     def __init__(self, context, table = None, tmp = True):
         self.table: TableInfo = table
@@ -193,6 +201,7 @@ class Context:
     extern "C" int __DLLEXPORT__ dllmain(Context* cxt) { 
         using namespace std;
         using namespace types;
+        
     '''
     def __init__(self): 
         self.tables:List[TableInfo] = []

engine/ddl.py

@@ -1,14 +1,17 @@
 # code-gen for data decl languages
 
+from engine.orderby import orderby
 from engine.ast import ColRef, TableInfo, ast_node, Context, include
+from engine.scan import scan
 from engine.utils import base62uuid
 
 class create_table(ast_node):
     name = 'create_table'
-    def __init__(self, parent: "ast_node", node, context: Context = None, cexpr = None):
-        self.cexpr = cexpr
+    def __init__(self, parent: "ast_node", node, context: Context = None, cexprs = None, lineage = False):
+        self.cexprs = cexprs
+        self.lineage = lineage
         super().__init__(parent, node, context)
-    def produce(self, node):
+    def produce(self, node): 
         if type(node) is not TableInfo:
             ct = node[self.name]
             tbl = self.context.add_table(ct['name'], ct['columns'])
@@ -22,14 +25,38 @@ class create_table(ast_node):
         self.context.tables_in_context[tbl] = tbl.table_name
         tbl.cxt_name = tbl.table_name
         tbl.refer_all()
-        if self.cexpr is None:
+        # create an empty new table
+        if self.cexprs is None:
             for c in tbl.columns:
                 self.emit(f"{c.cxt_name}.init();")
+        # create an output table
         else:
-            for i, c in enumerate(tbl.columns):
-                self.emit(f"{c.cxt_name}.init();")
-                self.emit(f"{c.cxt_name} = {self.cexpr[i]()};")
-
+            # 1 to 1 lineage.
+            if len(self.context.scans) == 0:
+                if self.lineage:
+                    order = 'order_' + base62uuid(6)
+                    self.emit(f'auto {order} = {self.parent.datasource.cxt_name}->order_by<{orderby(self.parent, self.parent.assumptions).result()}>();')
+                    self.lineage = '*' + order
+                else:
+                    self.lineage = None
+                for i, c in enumerate(tbl.columns):
+                    self.emit(f"{c.cxt_name}.init();")
+                    self.emit(f"{c.cxt_name} = {self.cexprs[i](self.lineage)};")
+                self.lineage = None
+                self.parent.assumptions = None
+            else:
+                scanner:scan = self.context.scans[-1]
+                if self.lineage:
+                    lineage_var = 'lineage_' + base62uuid(6)
+                    counter_var = 'counter_' + base62uuid(6)
+                    scanner.add(f'auto {lineage_var} = {self.datasource.cxt_name}->bind({tbl.cxt_name});', "init")
+                    scanner.add(f'auto {counter_var} = 0;', "init")
+                    scanner.add(f"{lineage_var}.emplace_back({counter_var}++);", "front")
+                    self.lineage = f"{lineage_var}.rid"
+                for i, c in enumerate(tbl.columns):
+                    scanner.add(f"{c.cxt_name}.init();", "init")
+                    scanner.add(f"{c.cxt_name} = {self.cexprs[i](scanner.it_ver)};")
+                    
 class insert(ast_node):
     name = 'insert'
     def produce(self, node):
@@ -81,32 +108,21 @@ class outfile(ast_node):
     def produce(self, node):
         out_table:TableInfo = self.parent.out_table
         filename = node['loc']['literal'] if 'loc' in node else node['literal']
-        self.emit_no_ln(f"\"{filename}\"1:`csv@(+(")
-        l_compound = False
-        l_cols = ''
-        l_keys = ''
-        ending = lambda x: x[:-1] if len(x) > 0 and x[-1]==';' else x
-        for i, c in enumerate(out_table.columns):
-            c:ColRef
-            l_keys += '`' + c.name
-            if c.compound:
-                if l_compound:
-                    l_cols=f'flatBOTH\'+(({ending(l_cols)});{c.cname})'
-                else:
-                    l_compound = True
-                    if i >= 1:
-                        l_cols = f'flatRO\'+(({ending(l_cols)});{c.cname})'
-                    else:
-                        l_cols = c.cname + ';'
-            elif l_compound:
-                l_cols = f'flatLO\'+(({ending(l_cols)});{c.cname})'
-            else:
-                l_cols += f"{c.cname};"
-        if not l_compound:
-            self.emit_no_ln(l_keys + '!(' + ending(l_cols) + ')')
-        else:
-            self.emit_no_ln(f'{l_keys}!+,/({ending(l_cols)})')
-        self.emit('))')
-            
+        sep = ',' if 'term' not in node else node['term']['literal']
+        file_pointer = 'fp_' + base62uuid(6)
+        self.emit(f'FILE* {file_pointer} = fopen("{filename}", "w");')
+        self.emit(f'{out_table.cxt_name}->printall("{sep}", "\\n", nullptr, {file_pointer});')
+        self.emit(f'fclose({file_pointer});')
+        # self.context.headers.add('fstream')
+        # cout_backup_buffer = 'stdout_' + base62uuid(4)
+        # ofstream = 'ofstream_' + base62uuid(6)
+        # self.emit(f'auto {cout_backup_buffer} = cout.rdbuf();')
+        # self.emit(f'auto {ofstream} = ofstream("{filename}");')
+        # self.emit(f'cout.rdbuf({ofstream}.rdbuf());')
+        # TODO: ADD STMTS.
+        # self.emit(f'cout.rdbuf({cout_backup_buffer});')
+        # self.emit(f'{ofstream}.close();')
+        
+        
 import sys
 include(sys.modules[__name__])

engine/expr.py

@@ -11,10 +11,10 @@ class expr(ast_node):
         'avg': 'avg',
         'sum': 'sum',
         'count' : 'count',
-        'mins': ['mins', 'minsw'],
-        'maxs': ['maxs', 'maxsw'],
-        'avgs': ['avgs', 'avgsw'],
-        'sums': ['sums', 'sumsw'],
+        'mins': ['mins', 'minw'],
+        'maxs': ['maxs', 'maxw'],
+        'avgs': ['avgs', 'avgw'],
+        'sums': ['sums', 'sumw'],
     }
     
     binary_ops = {
@@ -23,8 +23,8 @@ class expr(ast_node):
         'mul':'*', 
         'div':'/',
         'mod':'%',
-        'and':'&',
-        'or':'|',
+        'and':'&&',
+        'or':'||',
         'xor' : '^',
         'gt':'>',
         'lt':'<',
@@ -40,13 +40,16 @@ class expr(ast_node):
         'not' : '!'
     }
     
-    coumpound_generating_ops = ['mod', 'mins', 'maxs', 'sums'] + \
+    coumpound_generating_ops = ['avgs', 'mins', 'maxs', 'sums'] + \
        list( binary_ops.keys()) + list(compound_ops.keys()) + list(unary_ops.keys() )
 
     def __init__(self, parent, node, materialize_cols = True, abs_col = False):
         self.materialize_cols = materialize_cols
         self.raw_col = None
         self.__abs = abs_col
+        self.inside_agg = False
+        if(type(parent) is expr):
+            self.inside_agg = parent.inside_agg
         ast_node.__init__(self, parent, node, None)
 
     def init(self, _):
@@ -67,7 +70,8 @@ class expr(ast_node):
         if type(node) is dict:
             for key, val in node.items():
                 if key in self.func_maps:
-                    # if type(val) in [dict, str]:
+                    # TODO: distinguish between UDF agg functions and other UDF functions.
+                    self.inside_agg = True
                     self.context.headers.add('"./server/aggregations.h"')
                     if type(val) is list and len(val) > 1:
                         cfunc = self.func_maps[key]
@@ -81,6 +85,7 @@ class expr(ast_node):
                         self._expr += f"{funcname}(" 
                         self._expr += expr(self, val)._expr
                     self._expr += ')'
+                    self.inside_agg = False
                 elif key in self.binary_ops:
                     l = expr(self, val[0])._expr
                     r = expr(self, val[1])._expr
@@ -92,7 +97,7 @@ class expr(ast_node):
                             x.append(expr(self, v)._expr)
                     self._expr = self.compound_ops[key][1](x)
                 elif key in self.unary_ops:
-                    self._expr += f'({expr(self, val)._expr}{self.unary_ops[key]})'
+                    self._expr += f'{self.unary_ops[key]}({expr(self, val)._expr})'
                 else:
                     print(f'Undefined expr: {key}{val}')
 
@@ -112,7 +117,7 @@ class expr(ast_node):
             self._expr, self.raw_col = self.datasource.parse_tablenames(node, self.materialize_cols, True)
             self.raw_col = self.raw_col if type(self.raw_col) is ColRef else None
             if self.__abs and self.raw_col:
-                self._expr = self.raw_col.reference() + index_expr
+                self._expr = self.raw_col.reference() + ("" if self.inside_agg else index_expr)
         elif type(node) is bool:
             self._expr = '1' if node else '0'
         else:

engine/groupby.py

@@ -1,8 +1,9 @@
-from engine.ast import TableInfo, ast_node
+from engine.ast import ColRef, TableInfo, ast_node
+from engine.orderby import assumption
 from engine.scan import scan
 from engine.utils import base62uuid
 from engine.expr import expr
-import engine.types
+
 class groupby(ast_node):
     name = '_groupby'
     def init(self, _):
@@ -12,7 +13,7 @@ class groupby(ast_node):
         self.group_type = 'record_type' + base62uuid(7)
         self.datasource = self.parent.datasource
         self.scanner = None
-        self.datasource.rec = []
+        self.datasource.rec = set()
         self.raw_groups = []
     def produce(self, node):
         
@@ -24,7 +25,8 @@ class groupby(ast_node):
         for i, g in enumerate(node):
             v = g['value']
             e = expr(self, v)
-            self.raw_groups.append(e.raw_col)
+            if type(e.raw_col) is ColRef:
+                self.raw_groups.append(e.raw_col)
             e = e._expr
             # if v is compound expr, create tmp cols
             if type(v) is not str:
@@ -40,7 +42,7 @@ class groupby(ast_node):
         self.emit(f'unordered_map<{self.group_type}, vector_type<uint32_t>, '
             f'transTypes<{self.group_type}, hasher>> {self.group};')
         self.n_grps = len(node)
-        self.scanner = scan(self, None, expr.toCExpr(first_col)()+'.size')
+        self.scanner = scan(self, self.datasource, expr.toCExpr(first_col)()+'.size')
         self.scanner.add(f'{self.group}[forward_as_tuple({g_contents(self.scanner.it_ver)})].emplace_back({self.scanner.it_ver});')
 
         
@@ -48,7 +50,14 @@ class groupby(ast_node):
         self.referenced = self.datasource.rec
         self.datasource.rec = None
         self.scanner.finalize()
-
+        
+    def deal_with_assumptions(self, assumption:assumption, out:TableInfo):
+        gscanner = scan(self, self.group)
+        val_var = 'val_'+base62uuid(7)
+        gscanner.add(f'auto &{val_var} = {gscanner.it_ver}.second;')
+        gscanner.add(f'{out.cxt_name}->order_by<{assumption.result()}>(&{val_var});')
+        gscanner.finalize()
+        
     def finalize(self, cexprs, out:TableInfo):
         gscanner = scan(self, self.group)
         key_var = 'key_'+base62uuid(7)
@@ -58,4 +67,6 @@ class groupby(ast_node):
         gscanner.add(f'auto &{val_var} = {gscanner.it_ver}.second;')
         gscanner.add(';\n'.join([f'{out.columns[i].reference()}.emplace_back({ce(x=val_var, y=key_var)})' for i, ce in enumerate(cexprs)])+';')
         
-        gscanner.finalize()
+        gscanner.finalize()
+        
+        self.datasource.groupinfo = None

engine/orderby.py

@@ -16,43 +16,50 @@ class order_item:
         return ('' if self.order else '-') + f'({self.name})'
         
     def __str__(self):
-        return self.materialize()
+        return self.name
     def __repr__(self):
         return self.__str__()
 
-class orders:
-    def __init__(self, node, datasource):
-        self.order_items = []
-        self.materialized = False
-        self.view = None
-        self.node = node 
-        self.datasource = datasource
-        self.n_attrs = -1
-
-    def materialize(self):
-        if not self.materialized:
-            self.view = View(self.node.context, self.datasource, False)
-            keys = ';'.join([f'{o}' for o in self.order_items])
-            self.n_attrs = len(self.order_items)
-            self.node.emit(f"{self.view.name}: > +`j (({',' if self.n_attrs == 1 else ''}{keys}))")
-            self.materialized = True
-
-    def append(self, o):
-        self.order_items.append(o)
-
 class orderby(ast_node):
     name = '_orderby'
-    
+    def __init__(self, parent: "ast_node", node, context: Context = None):
+        self.col_list = []
+        super().__init__(parent, node, context)
     def init(self, _):
         self.datasource = self.parent.datasource
-        self.order = orders(self, self.datasource)
+        self.order = []
         self.view = ''
     def produce(self, node):
         if type(node) is not list:
             node = [node]
         for n in node:
             order =  not ('sort' in n and n['sort'] == 'desc')
-            self.order.append(order_item(n['value'], self, order))
+            col_id = self.datasource.columns_byname[n['value']].id
+            col_id = col_id if order else -col_id-1
+            if col_id not in self.col_list:
+                self.col_list.append(col_id)
+                self.order.append(order_item(n['value'], self, order))
+            
+    def merge(self, node):
+        self.produce(node)
+        
+    def finialize(self, references):
+        self.order = [ o for o in self.order if o.name in references ]
+    
+    def result(self, sep:str = ','):
+        return sep.join([f"{c}" for c in self.col_list])
 
-    def consume(self, _):
-        self.datasource.order.append(self.order)
+class assumption(orderby):
+    name = '_assumption'
+    def __init__(self, parent: "ast_node", node, context: Context = None, exclude = []):
+        self.exclude = exclude
+        super().__init__(parent, node, context)
+    
+    def produce(self, node):
+        if type(node) is not list:
+            node = [node]
+        [n for n in node if n not in self.exclude]
+        return super().produce(node)
+    
+    def empty(self):
+        return len(self.col_list) == 0

engine/projection.py

@@ -2,9 +2,9 @@ from engine.ast import ColRef, TableInfo, ast_node, Context, include
 from engine.groupby import groupby
 from engine.join import join
 from engine.expr import expr
-from engine.orderby import orderby
+from engine.orderby import assumption, orderby
 from engine.scan import filter
-from engine.utils import base62uuid, enlist, base62alp
+from engine.utils import base62uuid, enlist, base62alp, has_other
 from engine.ddl import create_table, outfile
 import copy
 
@@ -14,6 +14,8 @@ class projection(ast_node):
         self.disp = disp
         self.outname = outname
         self.group_node = None
+        self.assumptions = None
+        self.where = None
         ast_node.__init__(self, parent, node, context)
     def init(self, _):
         if self.outname is None:
@@ -44,9 +46,8 @@ class projection(ast_node):
                     elif type(value) is str:
                         self.datasource = self.context.tables_byname[value]
                 if 'assumptions' in from_clause:
-                    for assumption in enlist(from_clause['assumptions']):
-                        orderby(self, assumption)
-
+                    self.assumptions = enlist(from_clause['assumptions'])
+                    
             elif type(from_clause) is str:
                 self.datasource = self.context.tables_byname[from_clause]
             
@@ -58,8 +59,9 @@ class projection(ast_node):
             self.prev_datasource = self.context.datasource
             self.context.datasource = self.datasource            
         if 'where' in node:
-            self.datasource = filter(self, node['where'], True).output
-            self.context.datasource = self.datasource            
+            self.where = filter(self, node['where'], True)
+            # self.datasource = filter(self, node['where'], True).output
+            # self.context.datasource = self.datasource            
 
         if 'groupby' in node:
             self.group_node = groupby(self, node['groupby'])
@@ -71,62 +73,101 @@ class projection(ast_node):
     def consume(self, node):
         self.inv = True
         disp_varname = 'd'+base62uuid(7)
-        has_groupby = False
-        if self.group_node is not None:
-            # There is group by;
-            has_groupby = True
+        has_groupby = self.group_node is not None
         cexprs = []
         flatten = False
         cols = []
         self.out_table = TableInfo('out_'+base62uuid(4), [], self.context)
         if 'outfile' in node:
             flatten = True
-            
+        
+        new_names = []
+        proj_raw_cols = []
         for i, proj in enumerate(self.projections):
             cname = ''
             compound = False
-            self.datasource.rec = []
+            self.datasource.rec = set()
             if type(proj) is dict:
                 if 'value' in proj:
                     e = proj['value']
-                    sname = expr(self, e)._expr
-                    fname = expr.toCExpr(sname)
-                    absname = expr(self, e, abs_col=True)._expr
-                    compound = True
+                    sname = expr(self, e)
+                    if type(sname.raw_col) is ColRef:
+                        proj_raw_cols.append(sname.raw_col)
+                    sname = sname._expr
+                    fname = expr.toCExpr(sname) # fastest access method at innermost context
+                    absname = expr(self, e, abs_col=True)._expr # absolute name at function scope
+                    # TODO: Make it single pass here.
+                    compound = True # compound column
                     cexprs.append(fname)
-                    cname = ''.join([a if a in base62alp else '' for a in fname()])
-
-            compound = compound and has_groupby and self.datasource.rec not in self.group_node.referenced
+                    cname = e if type(e) is str else ''.join([a if a in base62alp else '' for a in expr.toCExpr(absname)()])
+                if 'name' in proj: # renaming column by AS keyword
+                    cname = proj['name']
+                    new_names.append(cname)
+            elif type(proj) is str:
+                col = self.datasource.get_col_d(proj)
+                if type(col) is ColRef:
+                    col.reference()
+                    
+            compound = compound and has_groupby and has_other(self.datasource.rec, self.group_node.referenced)
+            self.datasource.rec = None
+            
+            typename = f'decays<decltype({absname})>'
+            if not compound:
+                typename = f'value_type<{typename}>'
+                
+            cols.append(ColRef(cname, expr.toCExpr(typename)(), self.out_table, 0, None, cname, i, compound=compound))
             
-            cols.append(ColRef(cname, expr.toCExpr(f'decays<decltype({absname})>')(0), self.out_table, 0, None, cname, i, compound=compound))
         self.out_table.add_cols(cols, False)
         
+        lineage = None
+        
         if has_groupby:
-            create_table(self, self.out_table)
+            create_table(self, self.out_table) # creates empty out_table.
+            if self.assumptions is not None:
+                self.assumptions = assumption(self, self.assumptions, exclude=self.group_node.raw_groups)
+                if not self.assumptions.empty():
+                    self.group_node.deal_with_assumptions(self.assumptions, self.out_table)
+                self.assumptions = None
             self.group_node.finalize(cexprs, self.out_table)
         else:
-            create_table(self, self.out_table, cexpr = cexprs)
-        self.datasource.group_node = None
-
-        has_orderby = 'orderby' in node
-
-        if has_orderby:
+            # if all assumptions in projections, treat as orderby
+            lineage = self.assumptions is not None and has_other(self.assumptions, proj_raw_cols) 
+            spawn = create_table(self, self.out_table, cexprs = cexprs, lineage = lineage) # create and populate out_table.
+            if lineage and type(spawn.lineage) is str:
+                lineage = spawn.lineage
+                self.assumptions = orderby(self, self.assumptions) # do not exclude proj_raw_cols
+            else:
+                lineage = None
+        if self.where is not None:
+            self.where.finalize()
+        
+        if type(lineage) is str:
+            order = 'order_' + base62uuid(6)
+            self.emit(f'auto {order} = {self.datasource.cxt_name}->order_by<{self.assumptions.result()}>({lineage});')
+            self.emit(f'{self.out_table.cxt_name}->materialize(*{order});')
+            self.assumptions = None
+        
+        if self.assumptions is not None:
+            orderby_node = orderby(self, self.assumptions)
+        else:
+            orderby_node = None
+            
+        if 'orderby' in node:
             self.datasource = self.out_table
             self.context.datasource = self.out_table # discard current ds
-            orderby_node = orderby(self, node['orderby'])
-            self.context.datasource.materialize_orderbys()
-            self.emit_no_ln(f"{f'{disp_varname}:+' if flatten else ''}(")
-            
-        if self.disp or has_orderby:
-            self.emit(f'print(*{self.out_table.cxt_name});')
-            
-        if has_orderby:
-            self.emit(f')[{orderby_node.view}]')
+            orderbys = node['orderby']
+            orderby_node = orderby(self, orderbys) if orderby_node is None else orderby_node.merge(orderbys)
+        
+        if orderby_node is not None:
+            self.emit(f'auto {disp_varname} = {self.out_table.reference()}->order_by_view<{orderby_node.result()}>();')
         else:
-            self.context.emit_flush()
+            disp_varname = f'*{self.out_table.cxt_name}'
+            
+        if self.disp:
+            self.emit(f'print({disp_varname});')
+            
+
         if flatten:
-            if len(self.projections) > 1 and not self.inv:
-                self.emit(f"{disp_varname}:+{disp_varname}")
             outfile(self, node['outfile'])
 
         if self.datasource_changed:

engine/scan.py

@@ -5,16 +5,20 @@ from engine.expr import expr
 
 class scan(ast_node):
     name = 'scan'
-    def __init__(self, parent: "ast_node", node, size = None, context: Context = None):
+    def __init__(self, parent: "ast_node", node, size = None, context: Context = None, const = False):
         self.type = type
         self.size = size
+        self.const = "const " if const else ""
         super().__init__(parent, node, context)
     def init(self, _):
         self.datasource = self.context.datasource
+        self.initializers = ''
         self.start = ''
+        self.front = ''
         self.body = ''
         self.end = '}'
-        self.filter = None
+        self.mode = None
+        self.filters = []
         scan_vars = set(s.it_var for s in self.context.scans)
         self.it_ver = 'i' + base62uuid(2)
         while(self.it_ver in scan_vars):
@@ -22,19 +26,33 @@ class scan(ast_node):
         self.parent.context.scans.append(self)
     def produce(self, node):
         if type(node) is ColRef:
+            self.colref = node
             if self.size is None:
-                self.start += f'for (auto& {self.it_ver} : {node.reference()}) {{\n'
+                self.mode = ["col", node.table]
+                self.start += f'for ({self.const}auto& {self.it_ver} : {node.reference()}) {{\n'
             else:
+                self.mode = ["idx", node.table]
                 self.start += f"for (uint32_t {self.it_ver} = 0; {self.it_ver} < {node.reference()}.size; ++{self.it_ver}){{\\n"
         elif type(node) is str:
-            self.start+= f'for(auto& {self.it_ver} : {node}) {{\n'
+            self.mode = ["idx", None]
+            self.start+= f'for({self.const}auto& {self.it_ver} : {node}) {{\n'
         else:
+            self.mode = ["idx", node] # Node is the TableInfo
             self.start += f"for (uint32_t {self.it_ver} = 0; {self.it_ver} < {self.size}; ++{self.it_ver}){{\n"
             
-    def add(self, stmt):
-        self.body+=stmt + '\n'
+    def add(self, stmt, position = "body"):
+        if position == "body":
+            self.body += stmt + '\n'
+        elif position == "init":
+            self.initializers += stmt + '\n'
+        else:
+            self.front += stmt + '\n'
+            
     def finalize(self):
-        self.context.remove_scan(self, self.start + self.body + self.end)
+        for f in self.filters:
+            self.start += f
+            self.end += '}'
+        self.context.remove_scan(self, self.initializers + self.start + self.front + self.body + self.end)
         
 class filter(ast_node):
     name = 'filter'
@@ -45,7 +63,7 @@ class filter(ast_node):
         self.datasource = self.context.datasource
         self.view = View(self.context, self.datasource)
         self.value = None
-        
+
     def spawn(self, node):
         # TODO: deal with subqueries
         self.modified_node = node
@@ -64,9 +82,18 @@ class filter(ast_node):
                 self.emit(f'{tmpVar}:{self.value}')
                 for o, c in zip(self.output.columns, self.datasource.columns):
                     self.emit(f'{o.cname}:$[{tmpVar};{c.cname};()]')
-                
-    def consume(self, node):
+    
+    def finalize(self):
+        self.scanner.finalize()
+    def consume(self, _):
         # TODO: optimizations after converting expr to cnf
+        self.scanner = None
+        for s in self.context.scans:
+            if self.datasource == s.mode[1]:
+                self.scanner = s
+                break
+        if self.scanner is None:
+            self.scanner = scan(self, self.datasource, self.datasource.get_size())
         self.expr = expr(self, self.modified_node)
-        print(node)
+        self.scanner.filters.append(f'if ({self.expr.cexpr(self.scanner.it_ver)}) {{\n')
     

engine/utils.py

@@ -16,4 +16,10 @@ def enlist(l):
     return l if type(l) is list else [l]
 
 def seps(s, i, l):
-    return s if i < len(l) - 1 else ''
+    return s if i < len(l) - 1 else ''
+
+def has_other(a, b):
+    for ai in a:
+        if ai not in b:
+            return True
+    return False