fix gitw

engine/__init__.py

@@ -0,0 +1,12 @@
+from engine.ast import Context, ast_node
+import engine.ddl, engine.projection
+
+def initialize():
+    return Context()    
+
+def generate(ast, cxt):
+    for k in ast.keys():
+        if k in ast_node.types.keys():
+            root = ast_node.types[k](None, ast, cxt)
+            
+__all__ = ["initialize", "generate"]

engine/ast.py

@@ -0,0 +1,369 @@
+from engine.utils import base62uuid
+from copy import copy
+# replace column info with this later.
+class ColRef:
+    def __init__(self, cname, _ty, cobj, cnt, table:'TableInfo', name, id, compound = False):
+        self.cname = cname # column object location
+        self.cxt_name = None # column object in context
+        self.type = _ty
+        self.cobj = cobj
+        self.cnt = cnt
+        self.table = table
+        self.name = name
+        self.id = id # position in table
+        self.order_pending = None # order_pending
+        self.compound = compound # compound field (list as a field) 
+        self.views = []
+        self.aux_columns = [] # columns for temperary calculations 
+        # e.g. order by, group by, filter by expressions
+        
+        self.__arr__ = (cname, _ty, cobj, cnt, table, name, id)
+    
+    def reference(self):
+        cxt = self.table.cxt
+        self.table.reference()
+        if self not in cxt.columns_in_context:
+            counter = 0
+            base_name = self.table.table_name + '_' + self.name
+            if base_name in cxt.columns_in_context.values():
+                while (f'{base_name}_{counter}') in cxt.columns_in_context.values():
+                    counter += 1
+                base_name = f'{base_name}_{counter}'
+            self.cxt_name = base_name
+            cxt.columns_in_context[self] = base_name
+            # TODO: change this to cname;
+            cxt.emit(f'auto& {base_name} = *(ColRef<{self.type}> *)(&{self.table.cxt_name}->colrefs[{self.id}]);')
+        elif self.cxt_name is None:
+            self.cxt_name = cxt.columns_in_context[self]
+            
+        return self.cxt_name
+             
+    def __getitem__(self, key):
+        if type(key) is str:
+            return getattr(self, key)
+        else:
+            return self.__arr__[key]
+
+    def __setitem__(self, key, value):
+        self.__arr__[key] = value
+
+    def __str__(self):
+        return self.reference()
+    def __repr__(self):
+        return self.reference()
+
+class TableInfo:
+    
+    def __init__(self, table_name, cols, cxt:'Context'):
+        # statics
+        self.table_name = table_name
+        self.alias = set([table_name])
+        self.columns_byname = dict() # column_name, type
+        self.columns = []
+        self.cxt = cxt
+        self.cxt_name = None
+        self.views = set()
+        #keep track of temp vars
+        self.local_vars = dict()
+        self.rec = None 
+        self.groupinfo = None
+        self.add_cols(cols)
+        # runtime
+        self.n_rows = 0 # number of cols
+        self.order = [] # assumptions
+
+        cxt.tables_byname[self.table_name] = self # construct reverse map
+    def reference(self):
+        if self not in self.cxt.tables_in_context:
+            counter = 0
+            base_name = self.table_name
+            if base_name in self.cxt.tables_in_context.values():
+                while (f'{base_name}_{counter}') in self.cxt.tables_in_context.values():
+                    counter += 1
+                base_name = f'{base_name}_{counter}'
+            self.cxt_name = base_name
+            self.cxt.tables_in_context[self] = base_name
+            
+            type_tags = '<'
+            for c in self.columns:
+                type_tags += c.type + ','
+            if type_tags.endswith(','):
+                type_tags = type_tags[:-1]
+            type_tags += '>'
+            
+            self.cxt.emit(f'auto& {base_name} = *(TableInfo{type_tags} *)(cxt->tables["{self.table_name}"]);')
+        return self.cxt_name
+    def refer_all(self):
+        self.reference()
+        for c in self.columns:
+            c.reference()
+    def add_cols(self, cols, new = True):
+        for i, c in enumerate(cols):
+            self.add_col(c, new, i)
+    def add_col(self, c, new = True, i = 0):
+        _ty = c['type']
+        if new:
+            cname =f'get<{i}>({self.table_name})'
+            _ty = _ty if type(c) is ColRef else list(_ty.keys())[0]
+            col_object =  ColRef(cname, _ty, c, 1, self,c['name'], len(self.columns))
+        else:
+            col_object = c
+            cname = c.cname
+            c.table = self
+        self.cxt.ccols_byname[cname] = col_object
+        self.columns_byname[c['name']] = col_object
+        self.columns.append(col_object)
+    def get_size(self):
+        size_tmp = 'tmp_sz_'+base62uuid(6)
+        self.cxt.emit(f'const auto& {size_tmp} = {self.columns[0].reference()}.size;')
+        return size_tmp
+    @property
+    def n_cols(self):
+        return len(self.columns)
+
+    def materialize_orderbys(self):
+        view_stack = ''
+        stack_name = ''
+        for o in self.order:
+            o.materialize()
+            if len(view_stack) == 0:
+                view_stack = o.view.name
+                stack_name = view_stack
+            else:
+                view_stack = view_stack+'['+ o.view.name +']'
+        # TODO: Optimize by doing everything in a stmt
+        if len(view_stack) > 0:
+            if len(self.order) > 1:
+                self.cxt.emit(f'{stack_name}:{view_stack}')
+            for c in self.columns:
+                c.order_pending = stack_name
+            self.order[0].node.view = stack_name
+        self.order.clear()
+
+    def get_col_d(self, col_name):
+        col = self.columns_byname[col_name]
+        if type(self.rec) is set:
+            self.rec.add(col)
+        return col
+
+    def get_ccolname_d(self, col_name):
+        return self.get_col_d(col_name).cname
+        
+    def get_col(self, col_name):
+        self.materialize_orderbys()
+        col = self.get_col_d(col_name)
+        if type(col.order_pending) is str:
+            self.cxt.emit_no_flush(f'{col.cname}:{col.cname}[{col.order_pending}]')
+            col.order_pending = None
+        return col
+    def get_ccolname(self, col_name):
+        return self.get_col(col_name).cname
+
+    def add_alias(self, alias):
+        # TODO: Scoping of alias should be constrainted in the query.
+        if alias in self.cxt.tables_byname.keys():
+            print("Error: table alias already exists")
+            return
+        self.cxt.tables_byname[alias] = self
+        self.alias.add(alias)
+        
+    def parse_col_names(self, colExpr, materialize = True, raw = False):
+        # get_col = self.get_col if materialize else self.get_col_d
+
+        parsedColExpr = colExpr.split('.')
+        ret = None
+        if len(parsedColExpr) <= 1:
+            ret = self.get_col_d(colExpr)
+        else: 
+            datasource = self.cxt.tables_byname[parsedColExpr[0]]
+            if datasource is None:
+                raise ValueError(f'Table name/alias not defined{parsedColExpr[0]}')
+            else:
+                ret = datasource.parse_col_names(parsedColExpr[1], raw)
+        from engine.expr import index_expr
+        string = ret.reference() + index_expr
+        if self.groupinfo is not None and ret and ret in self.groupinfo.raw_groups:
+            string = f'get<{self.groupinfo.raw_groups.index(ret)}>({{y}})'
+        return string, ret if raw else string
+
+class View:
+    def __init__(self, context, table = None, tmp = True):
+        self.table: TableInfo = table
+        self.name = 'v'+base62uuid(7)
+        if type(table) is TableInfo:
+            table.views.add(self)
+        self.context = context
+         
+    def construct(self):
+        self.context.emit(f'{self.name}:()')
+            
+class Context:
+    function_head = '''
+    extern "C" int __DLLEXPORT__ dllmain(Context* cxt) { 
+        using namespace std;
+        using namespace types;
+        
+    '''
+    LOG_INFO = 'INFO'
+    LOG_ERROR = 'ERROR'
+    LOG_SILENT = 'SILENT'
+    def new(self):
+        self.tmp_names = set()
+        self.udf_map = dict()
+        self.headers = set(['\"./server/libaquery.h\"'])
+        self.finalized = False
+        # read header
+        self.ccode = ''
+        self.ccodelet = ''
+        with open('header.cxx', 'r') as outfile:
+            self.ccode = outfile.read()         
+        # datasource will be availible after `from' clause is parsed
+        # and will be deactivated when the `from' is out of scope
+        self.datasource = None
+        self.ds_stack = []
+        self.scans = []
+        self.removing_scan = False
+
+    def __init__(self): 
+        self.tables:list[TableInfo] = []
+        self.tables_byname = dict()
+        self.ccols_byname = dict()
+        self.gc_name = 'gc_' + base62uuid(4)
+        self.tmp_names = set()
+        self.udf_map = dict()
+        self.headers = set(['\"./server/libaquery.h\"'])
+        self.finalized = False
+        self.log_level = Context.LOG_SILENT
+        self.print = print
+        # read header
+        self.ccode = ''
+        self.ccodelet = ''
+        self.columns_in_context = dict()
+        self.tables_in_context = dict()
+        with open('header.cxx', 'r') as outfile:
+            self.ccode = outfile.read()         
+        # datasource will be availible after `from' clause is parsed
+        # and will be deactivated when the `from' is out of scope
+        self.datasource = None
+        self.ds_stack = []
+        self.scans = []
+        self.removing_scan = False
+    def add_table(self, table_name, cols):
+        tbl = TableInfo(table_name, cols, self)
+        self.tables.append(tbl)
+        return tbl
+
+    def gen_tmptable(self):
+        from engine.utils import base62uuid
+        return f't{base62uuid(7)}'
+    def reg_tmp(self, name, f):
+        self.tmp_names.add(name)
+        self.emit(f"{self.gc_name}.reg({{{name}, 0,0{'' if f is None else ',{f}'}}});")
+
+    def define_tmp(self, typename, isPtr = True, f = None):
+        name = 'tmp_' + base62uuid()
+        if isPtr:
+            self.emit(f'auto* {name} = new {typename};')
+            self.reg_tmp(name, f)
+        else:
+            self.emit(f'auto {name} = {typename};')
+        return name
+    def emit(self, codelet):
+        self.ccode += self.ccodelet + codelet + '\n'
+        self.ccodelet = ''
+    def emit_no_flush(self, codelet):
+        self.ccode += codelet + '\n'
+    def emit_flush(self):
+        self.ccode += self.ccodelet + '\n'
+        self.ccodelet = ''
+    def emit_nonewline(self, codelet):
+        self.ccodelet += codelet
+
+    def datsource_top(self):
+        if len(self.ds_stack) > 0:
+            return self.ds_stack[-1]
+        else:
+            return None
+    def datasource_pop(self):
+        if len(self.ds_stack) > 0:
+            self.ds_stack.pop()
+            return self.ds_stack[-1]
+        else:
+            return None
+    def datasource_push(self, ds):
+        if type(ds) is TableInfo:
+            self.ds_stack.append(ds)
+            return ds
+        else:
+            return None
+    def remove_scan(self, scan, str_scan):
+        self.emit(str_scan)
+        self.scans.remove(scan)
+        
+    def Info(self, msg):
+        if self.log_level.upper() == Context.LOG_INFO:
+            self.print(msg)
+    def Error(self, msg):
+        if self.log_level.upper() == Context.LOG_ERROR:
+            self.print(msg)
+        else:
+            self.Info(self, msg)
+    
+    
+    def finalize(self):
+        if not self.finalized:
+            headers = ''
+            for h in self.headers:
+                if h[0] != '"':
+                    headers += '#include <' + h + '>\n'
+                else:
+                    headers += '#include ' + h + '\n'
+            self.ccode = headers + self.function_head + self.ccode + 'return 0;\n}'
+            self.headers = set()
+        return self.ccode
+    def __str__(self):
+        self.finalize()
+        return self.ccode
+    def __repr__(self) -> str:
+        return self.__str__()
+
+
+class ast_node:
+    types = dict()
+    header = []
+    def __init__(self, parent:"ast_node", node, context:Context = None):
+        self.context = parent.context if context is None else context
+        self.parent = parent
+        self.datasource = None
+        self.init(node)
+        self.produce(node)
+        self.spawn(node)
+        self.consume(node)
+    
+    def emit(self, code):
+        self.context.emit(code)
+    def emit_no_ln(self, code):
+        self.context.emit_nonewline(code)
+
+    name = 'null'
+
+    # each ast node has 3 stages. 
+    # `produce' generates info for child nodes
+    # `spawn' populates child nodes
+    # `consume' consumes info from child nodes and finalizes codegen
+    # For simple operators, there may not be need for some of these stages
+    def init(self, _):
+        pass
+    def produce(self, _):
+        pass
+    def spawn(self, _):
+        pass
+    def consume(self, _):
+        pass
+
+# include classes in module as first order operators
+def include(objs):
+    import inspect
+    for _, cls in inspect.getmembers(objs):
+        if inspect.isclass(cls) and issubclass(cls, ast_node) and not cls.name.startswith('_'):
+            ast_node.types[cls.name] = cls

engine/ddl.py

@@ -0,0 +1,128 @@
+# code-gen for data decl languages
+
+from engine.orderby import orderby
+from engine.ast import ColRef, TableInfo, ast_node, Context, include
+from engine.scan import scan
+from engine.utils import base62uuid
+
+class create_table(ast_node):
+    name = 'create_table'
+    def __init__(self, parent: "ast_node", node, context: Context = None, cexprs = None, lineage = False):
+        self.cexprs = cexprs
+        self.lineage = lineage
+        super().__init__(parent, node, context)
+    def produce(self, node): 
+        if type(node) is not TableInfo:
+            ct = node[self.name]
+            tbl = self.context.add_table(ct['name'], ct['columns'])
+        else:
+            tbl = node
+            
+        col_type_str = ','.join([c.type for c in tbl.columns])
+        # create tables in c
+        self.emit(f"auto {tbl.table_name} = new TableInfo<{col_type_str}>(\"{tbl.table_name}\", {tbl.n_cols});")
+        self.emit("cxt->tables.insert({\"" + tbl.table_name + f"\", {tbl.table_name}"+"});")
+        self.context.tables_in_context[tbl] = tbl.table_name
+        tbl.cxt_name = tbl.table_name
+        tbl.refer_all()
+        # create an empty new table
+        if self.cexprs is None:
+            for c in tbl.columns:
+                self.emit(f'{c.cxt_name}.init("{c.name}");')
+        # create an output table
+        else:
+            # 1 to 1 lineage.
+            if len(self.context.scans) == 0:
+                if self.lineage:
+                    order = 'order_' + base62uuid(6)
+                    self.emit(f'auto {order} = {self.parent.datasource.cxt_name}->order_by<{orderby(self.parent, self.parent.assumptions).result()}>();')
+                    self.lineage = '*' + order
+                else:
+                    self.lineage = None
+                for i, c in enumerate(tbl.columns):
+                    self.emit(f'{c.cxt_name}.init("{c.name}");')
+                    self.emit(f"{c.cxt_name} = {self.cexprs[i](self.lineage)};")
+                self.lineage = None
+                self.parent.assumptions = None
+            else:
+                scanner:scan = self.context.scans[-1]
+                if self.lineage:
+                    lineage_var = 'lineage_' + base62uuid(6)
+                    counter_var = 'counter_' + base62uuid(6)
+                    scanner.add(f'auto {lineage_var} = {self.datasource.cxt_name}->bind({tbl.cxt_name});', "init")
+                    scanner.add(f'auto {counter_var} = 0;', "init")
+                    scanner.add(f"{lineage_var}.emplace_back({counter_var}++);", "front")
+                    self.lineage = f"{lineage_var}.rid"
+                for i, c in enumerate(tbl.columns):
+                    scanner.add(f'{c.cxt_name}.init("{c.name}");', "init")
+                    scanner.add(f"{c.cxt_name} = {self.cexprs[i](scanner.it_ver)};")
+                    
+class insert(ast_node):
+    name = 'insert'
+    def produce(self, node):
+        ct = node[self.name]
+        table:TableInfo = self.context.tables_byname[ct]
+
+        values = node['query']['select']
+        if len(values) != table.n_cols:
+            raise ValueError("Column Mismatch")
+        table.refer_all()
+        for i, s in enumerate(values):
+            if 'value' in s:
+                cname = table.columns[i].cxt_name
+                self.emit(f"{cname}.emplace_back({s['value']});")
+            else:
+                # subquery, dispatch to select astnode
+                pass
+            
+class c(ast_node):
+    name='c'
+    def produce(self, node):
+        self.emit(node[self.name])
+        
+class load(ast_node):
+    name="load"
+    def produce(self, node):
+        self.context.headers.add('"csv.h"')
+        node = node[self.name]
+        table:TableInfo = self.context.tables_byname[node['table']]
+        table.refer_all()
+        csv_reader_name = 'csv_reader_' + base62uuid(6)
+        col_types = [c.type for c in table.columns]
+        col_tmp_names = ['tmp_'+base62uuid(8) for _ in range(len(table.columns))]
+        # col_type_str = ",".join(col_types)
+        col_names = ','.join([f'"{c.name}"' for c in table.columns])
+        
+        self.emit(f'io::CSVReader<{len(col_types)}> {csv_reader_name}("{node["file"]["literal"]}");')
+        self.emit(f'{csv_reader_name}.read_header(io::ignore_extra_column, {col_names});')
+        for t, n in zip(col_types, col_tmp_names):
+            self.emit(f'{t} {n};')
+        self.emit(f'while({csv_reader_name}.read_row({",".join(col_tmp_names)})) {{ \n')
+        for i, c in enumerate(table.columns):
+            self.emit(f'{c.cxt_name}.emplace_back({col_tmp_names[i]});')
+        self.emit('}')
+        
+            
+class outfile(ast_node):
+    name="_outfile"
+    def produce(self, node):
+        out_table:TableInfo = self.parent.out_table
+        filename = node['loc']['literal'] if 'loc' in node else node['literal']
+        sep = ',' if 'term' not in node else node['term']['literal']
+        file_pointer = 'fp_' + base62uuid(6)
+        self.emit(f'FILE* {file_pointer} = fopen("{filename}", "w");')
+        self.emit(f'{out_table.cxt_name}->printall("{sep}", "\\n", nullptr, {file_pointer});')
+        self.emit(f'fclose({file_pointer});')
+        # self.context.headers.add('fstream')
+        # cout_backup_buffer = 'stdout_' + base62uuid(4)
+        # ofstream = 'ofstream_' + base62uuid(6)
+        # self.emit(f'auto {cout_backup_buffer} = cout.rdbuf();')
+        # self.emit(f'auto {ofstream} = ofstream("{filename}");')
+        # self.emit(f'cout.rdbuf({ofstream}.rdbuf());')
+        # TODO: ADD STMTS.
+        # self.emit(f'cout.rdbuf({cout_backup_buffer});')
+        # self.emit(f'{ofstream}.close();')
+        
+        
+import sys
+include(sys.modules[__name__])

engine/expr.py

@@ -0,0 +1,131 @@
+from engine.ast import ast_node, ColRef
+start_expr = 'f"'
+index_expr = '{\'\' if x is None and y is None else f\'[{x}]\'}'
+end_expr = '"'
+
+class expr(ast_node):
+    name='expr'
+    builtin_func_maps = {
+        'max': 'max',
+        'min': 'min', 
+        'avg': 'avg',
+        'sum': 'sum',
+        'count' : 'count',
+        'mins': ['mins', 'minw'],
+        'maxs': ['maxs', 'maxw'],
+        'avgs': ['avgs', 'avgw'],
+        'sums': ['sums', 'sumw'],
+    }
+    
+    binary_ops = {
+        'sub':'-',  
+        'add':'+', 
+        'mul':'*', 
+        'div':'/',
+        'mod':'%',
+        'and':'&&',
+        'or':'||',
+        'xor' : '^',
+        'gt':'>',
+        'lt':'<',
+        'le':'<=',
+        'gt':'>='
+    }
+
+    compound_ops = {
+    }
+
+    unary_ops = {
+        'neg' : '-',
+        'not' : '!'
+    }
+    
+    coumpound_generating_ops = ['avgs', 'mins', 'maxs', 'sums'] + \
+       list( binary_ops.keys()) + list(compound_ops.keys()) + list(unary_ops.keys() )
+
+    def __init__(self, parent, node, materialize_cols = True, abs_col = False):
+        self.materialize_cols = materialize_cols
+        self.raw_col = None
+        self.__abs = abs_col
+        self.inside_agg = False
+        if(type(parent) is expr):
+            self.inside_agg = parent.inside_agg
+            self.__abs = parent.__abs
+        ast_node.__init__(self, parent, node, None)
+
+    def init(self, _):
+        from engine.projection import projection
+        parent = self.parent
+        self.isvector = parent.isvector if type(parent) is expr else False
+        self.is_compound = parent.is_compound if type(parent) is expr else False
+        if type(parent) in [projection, expr]:
+            self.datasource = parent.datasource
+        else:
+            self.datasource = self.context.datasource
+        self.udf_map = parent.context.udf_map
+        self._expr = ''
+        self.cexpr = None
+        self.func_maps = {**self.udf_map, **self.builtin_func_maps}
+
+    def produce(self, node):
+        if type(node) is dict:
+            for key, val in node.items():
+                if key in self.func_maps:
+                    # TODO: distinguish between UDF agg functions and other UDF functions.
+                    self.inside_agg = True
+                    self.context.headers.add('"./server/aggregations.h"')
+                    if type(val) is list and len(val) > 1:
+                        cfunc = self.func_maps[key]
+                        cfunc = cfunc[len(val) - 1] if type(cfunc) is list else cfunc
+                        self._expr += f"{cfunc}(" 
+                        for i, p in enumerate(val):
+                            self._expr += expr(self, p)._expr + (','if i<len(val)-1 else '')
+                    else:
+                        funcname = self.func_maps[key]
+                        funcname = funcname[0] if type(funcname) is list else funcname
+                        self._expr += f"{funcname}(" 
+                        self._expr += expr(self, val)._expr
+                    self._expr += ')'
+                    self.inside_agg = False
+                elif key in self.binary_ops:
+                    l = expr(self, val[0])._expr
+                    r = expr(self, val[1])._expr
+                    self._expr += f'({l}{self.binary_ops[key]}{r})'
+                elif key in self.compound_ops:
+                    x = []
+                    if type(val) is list:
+                        for v in val:
+                            x.append(expr(self, v)._expr)
+                    self._expr = self.compound_ops[key][1](x)
+                elif key in self.unary_ops:
+                    self._expr += f'{self.unary_ops[key]}({expr(self, val)._expr})'
+                else:
+                    self.context.Error(f'Undefined expr: {key}{val}')
+
+                if key in self.coumpound_generating_ops and not self.is_compound:
+                    self.is_compound = True
+                    p = self.parent
+                    while type(p) is expr and not p.is_compound:
+                        p.is_compound = True
+                        p = p.parent
+
+        elif type(node) is str:
+            p = self.parent
+            while type(p) is expr and not p.isvector:
+                p.isvector = True
+                p = p.parent
+                
+            self._expr, self.raw_col = self.datasource.parse_col_names(node, self.materialize_cols, True)
+            self.raw_col = self.raw_col if type(self.raw_col) is ColRef else None
+            if self.__abs and self.raw_col:
+                self._expr = self.raw_col.reference() + ("" if self.inside_agg else index_expr)
+        elif type(node) is bool:
+            self._expr = '1' if node else '0'
+        else:
+            self._expr = f'{node}'
+    def toCExpr(_expr):
+        return lambda x = None, y = None : eval(start_expr + _expr + end_expr)
+    def consume(self, _):
+        self.cexpr = expr.toCExpr(self._expr)
+    def __str__(self):
+        return self.cexpr

engine/groupby.py

@@ -0,0 +1,72 @@
+from engine.ast import ColRef, TableInfo, ast_node
+from engine.orderby import assumption
+from engine.scan import scan
+from engine.utils import base62uuid
+from engine.expr import expr
+
+class groupby(ast_node):
+    name = '_groupby'
+    def init(self, _):
+        self.context.headers.add('"./server/hasher.h"')
+        self.context.headers.add('unordered_map')
+        self.group = 'g' + base62uuid(7)
+        self.group_type = 'record_type' + base62uuid(7)
+        self.datasource = self.parent.datasource
+        self.scanner = None
+        self.datasource.rec = set()
+        self.raw_groups = []
+    def produce(self, node):
+        
+        if type(node) is not list:
+            node = [node]
+        g_contents = ''
+        g_contents_list = []
+        first_col = ''
+        for i, g in enumerate(node):
+            v = g['value']
+            e = expr(self, v)
+            if type(e.raw_col) is ColRef:
+                self.raw_groups.append(e.raw_col)
+            e = e._expr
+            # if v is compound expr, create tmp cols
+            if type(v) is not str:
+                tmpcol = 't' + base62uuid(7)
+                self.emit(f'auto {tmpcol} = {e};')
+                e = tmpcol
+            if i == 0:
+                first_col = e
+            g_contents_list.append(e)
+        g_contents_decltype = [f'decltype({c})' for c in g_contents_list]
+        g_contents = expr.toCExpr(','.join(g_contents_list))
+        self.emit(f'typedef record<{expr.toCExpr(",".join(g_contents_decltype))(0)}> {self.group_type};')
+        self.emit(f'unordered_map<{self.group_type}, vector_type<uint32_t>, '
+            f'transTypes<{self.group_type}, hasher>> {self.group};')
+        self.n_grps = len(node)
+        self.scanner = scan(self, self.datasource, expr.toCExpr(first_col)()+'.size')
+        self.scanner.add(f'{self.group}[forward_as_tuple({g_contents(self.scanner.it_ver)})].emplace_back({self.scanner.it_ver});')
+
+        
+    def consume(self, _):
+        self.referenced = self.datasource.rec
+        self.datasource.rec = None
+        self.scanner.finalize()
+        
+    def deal_with_assumptions(self, assumption:assumption, out:TableInfo):
+        gscanner = scan(self, self.group)
+        val_var = 'val_'+base62uuid(7)
+        gscanner.add(f'auto &{val_var} = {gscanner.it_ver}.second;')
+        gscanner.add(f'{self.datasource.cxt_name}->order_by<{assumption.result()}>(&{val_var});')
+        gscanner.finalize()
+        
+    def finalize(self, cexprs, out:TableInfo):
+        gscanner = scan(self, self.group)
+        key_var = 'key_'+base62uuid(7)
+        val_var = 'val_'+base62uuid(7)
+        
+        gscanner.add(f'auto &{key_var} = {gscanner.it_ver}.first;')
+        gscanner.add(f'auto &{val_var} = {gscanner.it_ver}.second;')
+        gscanner.add(';\n'.join([f'{out.columns[i].reference()}.emplace_back({ce(x=val_var, y=key_var)})' for i, ce in enumerate(cexprs)])+';')
+        
+        gscanner.finalize()
+        
+        self.datasource.groupinfo = None

engine/join.py

@@ -0,0 +1,6 @@
+from engine.ast import ast_node
+
+
+class join(ast_node):
+    name='join'
+    

engine/orderby.py

@@ -0,0 +1,65 @@
+from engine.ast import ColRef, TableInfo, View, ast_node, Context
+from engine.utils import base62uuid, seps
+from engine.expr import expr
+
+class order_item:
+    def __init__(self, name, node, order = True):
+        self.name = name
+        self.order = order
+        self.node = node
+        self.materialized = False
+        
+    def materialize(self):
+        if not self.materialized:
+            self.name = expr(self.node, self.name, False).cexpr
+            self.materialized = True
+        return ('' if self.order else '-') + f'({self.name})'
+        
+    def __str__(self):
+        return self.name
+    def __repr__(self):
+        return self.__str__()
+
+class orderby(ast_node):
+    name = '_orderby'
+    def __init__(self, parent: "ast_node", node, context: Context = None):
+        self.col_list = []
+        super().__init__(parent, node, context)
+    def init(self, _):
+        self.datasource = self.parent.datasource
+        self.order = []
+        self.view = ''
+    def produce(self, node):
+        if type(node) is not list:
+            node = [node]
+        for n in node:
+            order =  not ('sort' in n and n['sort'] == 'desc')
+            col_id = self.datasource.columns_byname[n['value']].id
+            col_id = col_id if order else -col_id-1
+            if col_id not in self.col_list:
+                self.col_list.append(col_id)
+                self.order.append(order_item(n['value'], self, order))
+            
+    def merge(self, node):
+        self.produce(node)
+        
+    def finialize(self, references):
+        self.order = [ o for o in self.order if o.name in references ]
+    
+    def result(self, sep:str = ','):
+        return sep.join([f"{c}" for c in self.col_list])
+
+class assumption(orderby):
+    name = '_assumption'
+    def __init__(self, parent: "ast_node", node, context: Context = None, exclude = []):
+        self.exclude = exclude
+        super().__init__(parent, node, context)
+    
+    def produce(self, node):
+        if type(node) is not list:
+            node = [node]
+        [n for n in node if n not in self.exclude]
+        return super().produce(node)
+    
+    def empty(self):
+        return len(self.col_list) == 0

engine/projection.py

@@ -0,0 +1,180 @@
+from engine.ast import ColRef, TableInfo, ast_node, Context, include
+from engine.groupby import groupby
+from engine.join import join
+from engine.expr import expr
+from engine.orderby import assumption, orderby
+from engine.scan import filter
+from engine.utils import base62uuid, enlist, base62alp, has_other
+from engine.ddl import create_table, outfile
+import copy
+
+class projection(ast_node):
+    name='select'
+    def __init__(self, parent:ast_node, node, context:Context = None, outname = None, disp = True):
+        self.disp = disp
+        self.outname = outname
+        self.group_node = None
+        self.assumptions = None
+        self.where = None
+        ast_node.__init__(self, parent, node, context)
+    def init(self, _):
+        if self.outname is None:
+            self.outname = self.context.gen_tmptable()
+
+    def produce(self, node):
+        p = node['select']
+        self.projections = p if type(p) is list else [p]
+        self.context.Info(node)
+
+    def spawn(self, node):
+        self.datasource = None
+        if 'from' in node:
+            from_clause = node['from']
+            if type(from_clause) is list:
+                # from joins
+                join(self, from_clause)
+            elif type(from_clause) is dict:
+                if 'value' in from_clause:
+                    value = from_clause['value']
+                    if type(value) is dict:
+                        if 'select' in value:
+                            # from subquery
+                            projection(self, from_clause, disp = False)
+                        else:
+                            # TODO: from func over table
+                            print(f'from func over table{node}')
+                    elif type(value) is str:
+                        self.datasource = self.context.tables_byname[value]
+                    if 'name' in value:
+                        self.datasource.add_alias(value['name'])
+                if 'assumptions' in from_clause:
+                    self.assumptions = enlist(from_clause['assumptions'])
+                    
+            elif type(from_clause) is str:
+                self.datasource = self.context.tables_byname[from_clause]
+                
+            if self.datasource is None:
+                raise ValueError('spawn error: from clause')
+           
+        if self.datasource is not None:
+            self.datasource_changed = True
+            self.prev_datasource = self.context.datasource
+            self.context.datasource = self.datasource            
+        if 'where' in node:
+            self.where = filter(self, node['where'], True)
+            # self.datasource = filter(self, node['where'], True).output
+            # self.context.datasource = self.datasource            
+
+        if 'groupby' in node:
+            self.group_node = groupby(self, node['groupby'])
+            self.datasource = copy.copy(self.datasource) # shallow copy
+            self.datasource.groupinfo = self.group_node
+        else:
+            self.group_node = None
+            
+    def consume(self, node):
+        self.inv = True
+        disp_varname = 'd'+base62uuid(7)
+        has_groupby = self.group_node is not None
+        cexprs = []
+        flatten = False
+        cols = []
+        self.out_table = TableInfo('out_'+base62uuid(4), [], self.context)
+        if 'outfile' in node:
+            flatten = True
+        
+        new_names = []
+        proj_raw_cols = []
+        for i, proj in enumerate(self.projections):
+            cname = ''
+            compound = False
+            self.datasource.rec = set()
+            if type(proj) is dict:
+                if 'value' in proj:
+                    e = proj['value']
+                    sname = expr(self, e)
+                    if type(sname.raw_col) is ColRef:
+                        proj_raw_cols.append(sname.raw_col)
+                    sname = sname._expr
+                    fname = expr.toCExpr(sname) # fastest access method at innermost context
+                    absname = expr(self, e, abs_col=True)._expr # absolute name at function scope
+                    # TODO: Make it single pass here.
+                    compound = True # compound column
+                    cexprs.append(fname)
+                    cname = e if type(e) is str else ''.join([a if a in base62alp else '' for a in expr.toCExpr(absname)()])
+                if 'name' in proj: # renaming column by AS keyword
+                    cname = proj['name']
+                    new_names.append(cname)
+            elif type(proj) is str:
+                col = self.datasource.get_col_d(proj)
+                if type(col) is ColRef:
+                    col.reference()
+                    
+            compound = compound and has_groupby and has_other(self.datasource.rec, self.group_node.referenced)
+            self.datasource.rec = None
+            
+            typename = f'decays<decltype({absname})>'
+            if not compound:
+                typename = f'value_type<{typename}>'
+                
+            cols.append(ColRef(cname, expr.toCExpr(typename)(), self.out_table, 0, None, cname, i, compound=compound))
+            
+        self.out_table.add_cols(cols, False)
+        
+        lineage = None
+        
+        if has_groupby:
+            create_table(self, self.out_table) # creates empty out_table.
+            if self.assumptions is not None:
+                self.assumptions = assumption(self, self.assumptions, exclude=self.group_node.raw_groups)
+                if not self.assumptions.empty():
+                    self.group_node.deal_with_assumptions(self.assumptions, self.out_table)
+                self.assumptions = None
+            self.group_node.finalize(cexprs, self.out_table)
+        else:
+            # if all assumptions in projections, treat as orderby
+            lineage = self.assumptions is not None and has_other(self.assumptions, proj_raw_cols) 
+            spawn = create_table(self, self.out_table, cexprs = cexprs, lineage = lineage) # create and populate out_table.
+            if lineage and type(spawn.lineage) is str:
+                lineage = spawn.lineage
+                self.assumptions = orderby(self, self.assumptions) # do not exclude proj_raw_cols
+            else:
+                lineage = None
+        if self.where is not None:
+            self.where.finalize()
+        
+        if type(lineage) is str:
+            order = 'order_' + base62uuid(6)
+            self.emit(f'auto {order} = {self.datasource.cxt_name}->order_by<{self.assumptions.result()}>({lineage});')
+            self.emit(f'{self.out_table.cxt_name}->materialize(*{order});')
+            self.assumptions = None
+        
+        if self.assumptions is not None:
+            orderby_node = orderby(self, self.assumptions)
+        else:
+            orderby_node = None
+            
+        if 'orderby' in node:
+            self.datasource = self.out_table
+            self.context.datasource = self.out_table # discard current ds
+            orderbys = node['orderby']
+            orderby_node = orderby(self, orderbys) if orderby_node is None else orderby_node.merge(orderbys)
+        
+        if orderby_node is not None:
+            self.emit(f'auto {disp_varname} = {self.out_table.reference()}->order_by_view<{orderby_node.result()}>();')
+        else:
+            disp_varname = f'*{self.out_table.cxt_name}'
+            
+        if self.disp:
+            self.emit(f'print({disp_varname});')
+            
+
+        if flatten:
+            outfile(self, node['outfile'])
+
+        if self.datasource_changed:
+            self.context.datasource = self.prev_datasource
+
+
+import sys
+include(sys.modules[__name__])

engine/scan.py

@@ -0,0 +1,99 @@
+from xmlrpc.client import Boolean
+from engine.ast import ColRef, TableInfo, View, ast_node, Context
+from engine.utils import base62uuid
+from engine.expr import expr
+
+class scan(ast_node):
+    name = 'scan'
+    def __init__(self, parent: "ast_node", node, size = None, context: Context = None, const = False):
+        self.type = type
+        self.size = size
+        self.const = "const " if const else ""
+        super().__init__(parent, node, context)
+    def init(self, _):
+        self.datasource = self.context.datasource
+        self.initializers = ''
+        self.start = ''
+        self.front = ''
+        self.body = ''
+        self.end = '}'
+        self.mode = None
+        self.filters = []
+        scan_vars = set(s.it_var for s in self.context.scans)
+        self.it_ver = 'i' + base62uuid(2)
+        while(self.it_ver in scan_vars):
+            self.it_ver = 'i' + base62uuid(6)
+        self.parent.context.scans.append(self)
+    def produce(self, node):
+        if type(node) is ColRef:
+            self.colref = node
+            if self.size is None:
+                self.mode = ["col", node.table]
+                self.start += f'for ({self.const}auto& {self.it_ver} : {node.reference()}) {{\n'
+            else:
+                self.mode = ["idx", node.table]
+                self.start += f"for (uint32_t {self.it_ver} = 0; {self.it_ver} < {node.reference()}.size; ++{self.it_ver}){{\\n"
+        elif type(node) is str:
+            self.mode = ["idx", None]
+            self.start+= f'for({self.const}auto& {self.it_ver} : {node}) {{\n'
+        else:
+            self.mode = ["idx", node] # Node is the TableInfo
+            self.start += f"for (uint32_t {self.it_ver} = 0; {self.it_ver} < {self.size}; ++{self.it_ver}){{\n"
+            
+    def add(self, stmt, position = "body"):
+        if position == "body":
+            self.body += stmt + '\n'
+        elif position == "init":
+            self.initializers += stmt + '\n'
+        else:
+            self.front += stmt + '\n'
+            
+    def finalize(self):
+        for f in self.filters:
+            self.start += f
+            self.end += '}'
+        self.context.remove_scan(self, self.initializers + self.start + self.front + self.body + self.end)
+        
+class filter(ast_node):
+    name = 'filter'
+    def __init__(self, parent: "ast_node", node, materialize = False, context = None):
+        self.materialize = materialize
+        super().__init__(parent, node, context)
+    def init(self, _):
+        self.datasource = self.context.datasource
+        self.view = View(self.context, self.datasource)
+        self.value = None
+
+    def spawn(self, node):
+        # TODO: deal with subqueries
+        self.modified_node = node
+        return super().spawn(node)
+    def __materialize__(self):
+        if self.materialize:
+            cols = [] if self.datasource is None else self.datasource.columns
+            self.output = TableInfo('tn'+base62uuid(6), cols, self.context)
+            self.output.construct()
+            if type(self.value) is View: # cond filtered on tables.
+                self.emit(f'{self.value.name}:&{self.value.name}')
+                for o, c in zip(self.output.columns,self.value.table.columns):
+                    self.emit(f'{o.cname}:{c.cname}[{self.value.name}]')
+            elif self.value is not None: # cond is scalar
+                tmpVar = 't'+base62uuid(7)
+                self.emit(f'{tmpVar}:{self.value}')
+                for o, c in zip(self.output.columns, self.datasource.columns):
+                    self.emit(f'{o.cname}:$[{tmpVar};{c.cname};()]')
+    
+    def finalize(self):
+        self.scanner.finalize()
+    def consume(self, _):
+        # TODO: optimizations after converting expr to cnf
+        self.scanner = None
+        for s in self.context.scans:
+            if self.datasource == s.mode[1]:
+                self.scanner = s
+                break
+        if self.scanner is None:
+            self.scanner = scan(self, self.datasource, self.datasource.get_size())
+        self.expr = expr(self, self.modified_node)
+        self.scanner.filters.append(f'if ({self.expr.cexpr(self.scanner.it_ver)}) {{\n')
+    

engine/types.py

@@ -0,0 +1,34 @@
+from engine.ast import Context
+
+
+class Types:
+    name = 'Any'
+    cname = 'void*'
+    ctype_name = "types::NONE"
+    def __init__(self, context:Context):
+        self.cxt = context
+    def cast_to(self, *_):
+        return self
+    def cast_from(self, *_):
+        return self
+    def __repr__(self) -> str:
+        return self.cname
+
+class String(Types):
+    name = 'String'
+    cname = 'const char*'
+    ctype_name = "types::ASTR"
+    def cast_from(self, ty, val, container = None):
+        if type(ty) is Int:
+            self.cxt.emit()
+
+class Int(Types):
+    name = "Int"
+    cname = "int"
+    ctype_name = "types::AINT"
+    
+class Float(Types):
+    name = "Float"
+    cname = "float"
+    ctype_name = "types::AFLOAT"
+    

engine/utils.py

@@ -0,0 +1,25 @@
+import uuid
+
+base62alp = '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
+
+def base62uuid(crop=8):
+    id = uuid.uuid4().int
+    ret = ''
+    
+    while id:
+        ret = base62alp[id % 62] + ret
+        id //= 62
+    
+    return ret[:crop] if len(ret) else '0'
+
+def enlist(l): 
+    return l if type(l) is list else [l]
+
+def seps(s, i, l):
+    return s if i < len(l) - 1 else ''
+
+def has_other(a, b):
+    for ai in a:
+        if ai not in b:
+            return True
+    return False