q

engine/ast.py

@@ -1,24 +1,39 @@
-from typing import List
-
-from pyparsing import col
-
 from engine.utils import base62uuid
 
 # replace column info with this later.
 class ColRef:
-    def __init__(self, k9name, _ty, cobj, cnt, table, name, id, compound = False):
-        self.k9name = k9name
+    def __init__(self, cname, _ty, cobj, cnt, table:'TableInfo', name, id, compound = False):
+        self.cname = cname
+        self.cxt_name = None
         self.type = _ty
         self.cobj = cobj
         self.cnt = cnt
         self.table = table
         self.name = name
-        self.id = id
+        self.id = id # position in table
         self.order_pending = None # order_pending
         self.compound = compound # compound field (list as a field) 
         self.views = []
-        self.__arr__ = (k9name, _ty, cobj, cnt, table, name, id)
-        
+        self.__arr__ = (cname, _ty, cobj, cnt, table, name, id)
+    
+    def reference(self):
+        cxt = self.table.cxt
+        self.table.reference()
+        if self not in cxt.columns_in_context:
+            counter = 0
+            base_name = self.table.table_name + '_' + self.name
+            if base_name in cxt.columns_in_context.values():
+                while (f'{base_name}_{counter}') in cxt.columns_in_context.values():
+                    counter += 1
+                base_name = f'{base_name}_{counter}'
+            self.cxt_name = base_name
+            cxt.columns_in_context[self] = base_name
+            cxt.emit(f'auto& {base_name} = *(ColRef<{self.type}> *)(&{self.table.cxt_name}->colrefs[{self.id}]);')
+        elif self.cxt_name is None:
+            self.cxt_name = cxt.columns_in_context[self]
+            
+        return self.cxt_name
+             
     def __getitem__(self, key):
         if type(key) is str:
             return getattr(self, key)
@@ -29,7 +44,7 @@ class ColRef:
         self.__arr__[key] = value
 
     def __str__(self):
-        return self.k9name
+        return self.cname
 
 class TableInfo:
     
@@ -40,7 +55,10 @@ class TableInfo:
         self.columns_byname = dict() # column_name, type
         self.columns = []
         self.cxt = cxt
+        self.cxt_name = None
         self.views = set()
+        #keep track of temp vars
+        self.local_vars = dict()
         self.rec = None 
         self.groupinfo = None
         self.add_cols(cols)
@@ -49,25 +67,47 @@ class TableInfo:
         self.order = [] # assumptions
 
         cxt.tables_byname[self.table_name] = self # construct reverse map
+    def reference(self):
+        if self not in self.cxt.tables_in_context:
+            counter = 0
+            base_name = self.table_name
+            if base_name in self.cxt.tables_in_context.values():
+                while (f'{base_name}_{counter}') in self.cxt.tables_in_context.values():
+                    counter += 1
+                base_name = f'{base_name}_{counter}'
+            self.cxt_name = base_name
+            self.cxt.tables_in_context[self] = base_name
+            
+            type_tags = '<'
+            for c in self.columns:
+                type_tags += c.type + ','
+            if type_tags.endswith(','):
+                type_tags = type_tags[:-1]
+            type_tags += '>'
+            
+            self.cxt.emit(f'auto& {base_name} = *(TableInfo{type_tags} *)(cxt->tables[{self.table_name}]);')
+    def refer_all(self):
+        for c in self.columns:
+            c.reference()
     def add_cols(self, cols, new = True):
-        for c in cols:
-            self.add_col(c, new)
-    def add_col(self, c, new = True):
+        for i, c in enumerate(cols):
+            self.add_col(c, new, i)
+    def add_col(self, c, new = True, i = 0):
         _ty = c['type']
         if new:
-            k9name = 'c' + base62uuid(7)
+            cname =f'{self.table_name}->colrefs[{i}].scast<int>()'
             _ty = _ty if type(c) is ColRef else list(_ty.keys())[0]
-            col_object =  ColRef(k9name, _ty, c, 1, self,c['name'], len(self.columns))
+            col_object =  ColRef(cname, _ty, c, 1, self,c['name'], len(self.columns))
         else:
             col_object = c
-            k9name = c.k9name
-        self.cxt.k9cols_byname[k9name] = col_object
+            cname = c.cname
+        self.cxt.ccols_byname[cname] = col_object
         self.columns_byname[c['name']] = col_object
         self.columns.append(col_object)
         
     def construct(self):
         for c in self.columns:
-            self.cxt.emit(f'{c.k9name}:()')
+            self.cxt.emit(f'{c.cname}:()')
     @property
     def n_cols(self):
         return len(self.columns)
@@ -97,18 +137,18 @@ class TableInfo:
             self.rec.append(col)
         return col
 
-    def get_k9colname_d(self, col_name):
-        return self.get_col_d(col_name).k9name
+    def get_ccolname_d(self, col_name):
+        return self.get_col_d(col_name).cname
         
     def get_col(self, col_name):
         self.materialize_orderbys()
         col = self.get_col_d(col_name)
         if type(col.order_pending) is str:
-            self.cxt.emit_no_flush(f'{col.k9name}:{col.k9name}[{col.order_pending}]')
+            self.cxt.emit_no_flush(f'{col.cname}:{col.cname}[{col.order_pending}]')
             col.order_pending = None
         return col
-    def get_k9colname(self, col_name):
-        return self.get_col(col_name).k9name
+    def get_ccolname(self, col_name):
+        return self.get_col(col_name).cname
 
     def add_alias(self, alias):
         # TODO: Exception when alias already defined.
@@ -130,9 +170,9 @@ class TableInfo:
             else:
                 ret = datasource.get_col(parsedColExpr[1])
         if self.groupinfo is not None and ret:
-            ret = f"{ret.k9name}[{'start' if ret in self.groupinfo.referenced else 'range'}]"
+            ret = f"{ret.reference()}[{'start' if ret in self.groupinfo.referenced else 'range'}]"
         else:
-            ret = ret.k9name
+            ret = ret.reference()
         return ret
 
 class View:
@@ -147,17 +187,23 @@ class View:
         self.context.emit(f'{self.name}:()')
             
 class Context:
+    function_head = 'extern \"C\" int dllmain(Context* cxt){ \n'
     def __init__(self): 
         self.tables:List[TableInfo] = []
         self.tables_byname = dict()
-        self.k9cols_byname = dict()
-        
+        self.ccols_byname = dict()
+        self.gc_name = 'gc_' + base62uuid(4)
+        self.tmp_names = set()
         self.udf_map = dict()
+        self.headers = set(['\"./server/libaquery.h\"'])
+        self.finalized = False
         # read header
-        self.k9code = ''
-        self.k9codelet = ''
-        with open('header.k', 'r') as outfile:
-            self.k9code = outfile.read()         
+        self.ccode = ''
+        self.ccodelet = ''
+        self.columns_in_context = dict()
+        self.tables_in_context = dict()
+        with open('header.cxx', 'r') as outfile:
+            self.ccode = outfile.read()         
         # datasource will be availible after `from' clause is parsed
         # and will be deactivated when the `from' is out of scope
         self.datasource = None
@@ -171,17 +217,28 @@ class Context:
     def gen_tmptable(self):
         from engine.utils import base62uuid
         return f't{base62uuid(7)}'
+    def reg_tmp(self, name, f):
+        self.tmp_names.add(name)
+        self.emit(f"{self.gc_name}.reg({{{name}, 0,0{'' if f is None else ',{f}'}}});")
 
+    def define_tmp(self, typename, isPtr = True, f = None):
+        name = 'tmp_' + base62uuid()
+        if isPtr:
+            self.emit(f'auto* {name} = new {typename};')
+            self.reg_tmp(name, f)
+        else:
+            self.emit(f'auto {name} = {typename};')
+        return name
     def emit(self, codelet):
-        self.k9code += self.k9codelet + codelet + '\n'
-        self.k9codelet = ''
+        self.ccode += self.ccodelet + codelet + '\n'
+        self.ccodelet = ''
     def emit_no_flush(self, codelet):
-        self.k9code += codelet + '\n'
+        self.ccode += codelet + '\n'
     def emit_flush(self):
-        self.k9code += self.k9codelet + '\n'
-        self.k9codelet = ''
+        self.ccode += self.ccodelet + '\n'
+        self.ccodelet = ''
     def emit_nonewline(self, codelet):
-        self.k9codelet += codelet
+        self.ccodelet += codelet
 
     def datsource_top(self):
         if len(self.ds_stack) > 0:
@@ -200,19 +257,33 @@ class Context:
             return ds
         else:
             return None
-
+    def finalize(self):
+        if not self.finalized:
+            headers = ''
+            for h in self.headers:
+                if h[0] != '"':
+                    headers += '#include <' + h + '>\n'
+                else:
+                    headers += '#include ' + h + '\n'
+            self.ccode = headers + self.function_head + self.ccode + 'return 0;\n}'
+            self.headers = set()
+        return self.ccode
     def __str__(self):
-        return self.k9code
+        self.finalize()
+        return self.ccode
     def __repr__(self) -> str:
         return self.__str__()
 
 
 class ast_node:
     types = dict()
+    header = []
     def __init__(self, parent:"ast_node", node, context:Context = None):
         self.context = parent.context if context is None else context
         self.parent = parent
         self.datasource = None
+        for h in self.header:
+            self.context.headers.add(h)
         self.init(node)
         self.produce(node)
         self.spawn(node)

engine/ddl.py

@@ -7,10 +7,15 @@ class create_table(ast_node):
     def produce(self, node):
         ct = node[self.name]
         tbl = self.context.add_table(ct['name'], ct['columns'])
-        # create tables in k9
-        for c in ct['columns']:
-            self.emit(f"{tbl.get_k9colname(c['name'])}:()")
-
+        # create tables in c
+        self.emit(f"auto {tbl.table_name} = new TableInfo(\"{tbl.table_name}\", {tbl.n_cols});")
+        self.emit("cxt->tables.insert({\"" + tbl.table_name + f"\", {tbl.table_name}"+"});")
+        self.context.tables_in_context[tbl] = tbl.table_name
+        tbl.cxt_name = tbl.table_name
+        for i, c in enumerate(ct['columns']):
+            # TODO: more self awareness
+            self.emit(f"{tbl.table_name}->colrefs[{i}].ty = types::AINT;")
+            
 class insert(ast_node):
     name = 'insert'
     def produce(self, node):
@@ -20,16 +25,17 @@ class insert(ast_node):
         values = node['query']['select']
         if len(values) != table.n_cols:
             raise ValueError("Column Mismatch")
+        table.refer_all()
         for i, s in enumerate(values):
             if 'value' in s:
-                k9name = table.columns[i][0]
-                self.emit(f"{k9name}:{k9name},{s['value']}")
+                cname = table.columns[i].cxt_name
+                self.emit(f"{cname}.emplace_back({s['value']});")
             else:
                 # subquery, dispatch to select astnode
                 pass
             
-class k9(ast_node):
-    name='k9'
+class c(ast_node):
+    name='c'
     def produce(self, node):
         self.emit(node[self.name])
         
@@ -47,7 +53,7 @@ class load(ast_node):
         self.emit(f"{tablename}:({keys}!(+(`csv ? 1:\"{node['file']['literal']}\")))[{keys}]")
 
         for i, c in enumerate(table.columns):
-            self.emit(f'{c.k9name}:{tablename}[{i}]')
+            self.emit(f'{c.cname}:{tablename}[{i}]')
             
 class outfile(ast_node):
     name="_outfile"
@@ -64,17 +70,17 @@ class outfile(ast_node):
             l_keys += '`' + c.name
             if c.compound:
                 if l_compound:
-                    l_cols=f'flatBOTH\'+(({ending(l_cols)});{c.k9name})'
+                    l_cols=f'flatBOTH\'+(({ending(l_cols)});{c.cname})'
                 else:
                     l_compound = True
                     if i >= 1:
-                        l_cols = f'flatRO\'+(({ending(l_cols)});{c.k9name})'
+                        l_cols = f'flatRO\'+(({ending(l_cols)});{c.cname})'
                     else:
-                        l_cols = c.k9name + ';'
+                        l_cols = c.cname + ';'
             elif l_compound:
-                l_cols = f'flatLO\'+(({ending(l_cols)});{c.k9name})'
+                l_cols = f'flatLO\'+(({ending(l_cols)});{c.cname})'
             else:
-                l_cols += f"{c.k9name};"
+                l_cols += f"{c.cname};"
         if not l_compound:
             self.emit_no_ln(l_keys + '!(' + ending(l_cols) + ')')
         else:

engine/expr.py

@@ -54,7 +54,7 @@ class expr(ast_node):
         else:
             self.datasource = self.context.datasource
         self.udf_map = parent.context.udf_map
-        self.k9expr = ''
+        self.cexpr = ''
         self.func_maps = {**self.udf_map, **self.builtin_func_maps}
 
     def produce(self, node):
@@ -63,29 +63,29 @@ class expr(ast_node):
                 if key in self.func_maps:
                     # if type(val) in [dict, str]:
                     if type(val) is list and len(val) > 1:
-                        k9func = self.func_maps[key]
-                        k9func = k9func[len(val) - 1] if type(k9func) is list else k9func
-                        self.k9expr += f"{k9func}[" 
+                        cfunc = self.func_maps[key]
+                        cfunc = cfunc[len(val) - 1] if type(cfunc) is list else cfunc
+                        self.cexpr += f"{cfunc}(" 
                         for i, p in enumerate(val):
-                            self.k9expr += expr(self, p).k9expr + (';'if i<len(val)-1 else '')
+                            self.cexpr += expr(self, p).cexpr + (';'if i<len(val)-1 else '')
                     else:
                         funcname = self.func_maps[key]
                         funcname = funcname[0] if type(funcname) is list else funcname
-                        self.k9expr += f"{funcname}[" 
-                        self.k9expr += expr(self, val).k9expr
-                    self.k9expr += ']'
+                        self.cexpr += f"{funcname}(" 
+                        self.cexpr += expr(self, val).cexpr
+                    self.cexpr += ')'
                 elif key in self.binary_ops:
-                    l = expr(self, val[0]).k9expr
-                    r = expr(self, val[1]).k9expr
-                    self.k9expr += f'({l}{self.binary_ops[key]}{r})'
+                    l = expr(self, val[0]).cexpr
+                    r = expr(self, val[1]).cexpr
+                    self.cexpr += f'({l}{self.binary_ops[key]}{r})'
                 elif key in self.compound_ops:
                     x = []
                     if type(val) is list:
                         for v in val:
-                            x.append(expr(self, v).k9expr)
-                    self.k9expr = self.compound_ops[key][1](x)
+                            x.append(expr(self, v).cexpr)
+                    self.cexpr = self.compound_ops[key][1](x)
                 elif key in self.unary_ops:
-                    self.k9expr += f'({expr(self, val).k9expr}{self.unary_ops[key]})'
+                    self.cexpr += f'({expr(self, val).cexpr}{self.unary_ops[key]})'
                 else:
                     print(f'Undefined expr: {key}{val}')
 
@@ -101,10 +101,10 @@ class expr(ast_node):
             while type(p) is expr and not p.isvector:
                 p.isvector = True
                 p = p.parent
-            self.k9expr = self.datasource.parse_tablenames(node, self.materialize_cols)
+            self.cexpr = self.datasource.parse_tablenames(node, self.materialize_cols)
         elif type(node) is bool:
-            self.k9expr = '1' if node else '0'
+            self.cexpr = '1' if node else '0'
         else:
-            self.k9expr = f'{node}'
+            self.cexpr = f'{node}'
     def __str__(self):
-        return self.k9expr
+        return self.cexpr

engine/groupby.py

@@ -15,7 +15,7 @@ class groupby(ast_node):
         first_col = ''
         for i, g in enumerate(node):
             v = g['value']
-            e = expr(self, v).k9expr
+            e = expr(self, v).cexpr
             # if v is compound expr, create tmp cols
             if type(v) is not str:
                 tmpcol = 't' + base62uuid(7)
@@ -41,16 +41,16 @@ class groupby(ast_node):
         self.groupby_function = 'fgrp'+base62uuid(4)
         grp = self.group
         if self.n_grps <= 1:
-            k9fn = "{[range] start:*range;"+ ret + "}"
-            self.emit(f'{out}:(({k9fn}\'{grp})[!{grp}])')
+            cfn = "{[range] start:*range;"+ ret + "}"
+            self.emit(f'{out}:(({cfn}\'{grp})[!{grp}])')
             self.parent.inv = False
         else:
-            k9fn = "{[ids;grps;ll;dim;x] " + \
+            cfn = "{[ids;grps;ll;dim;x] " + \
                     "start:grps[x][dim];" + \
                     "end:$[x=0;ll;grps[x-1][dim]];" + \
                     "range:(end-start)#((start-ll)#ids);" + \
                     "start:ids[start];" + \
                     ret + '}'
-            self.emit(f'{self.groupby_function}:{k9fn}')
+            self.emit(f'{self.groupby_function}:{cfn}')
             self.emit(f'{out}:+({self.groupby_function}' + \
                 f'[{grp}[1];{grp}[0];(#{grp}[0])+1;(#({grp}[0][0]))-1]\'!(#({grp}[0])))')

engine/orderby.py

@@ -1,7 +1,6 @@
 from engine.ast import ColRef, TableInfo, View, ast_node, Context
 from engine.utils import base62uuid, seps
 from engine.expr import expr
-import k
 
 class order_item:
     def __init__(self, name, node, order = True):
@@ -12,7 +11,7 @@ class order_item:
         
     def materialize(self):
         if not self.materialized:
-            self.name = expr(self.node, self.name, False).k9expr
+            self.name = expr(self.node, self.name, False).cexpr
             self.materialized = True
         return ('' if self.order else '-') + f'({self.name})'
         

engine/projection.py

@@ -1,4 +1,3 @@
-from attr import has
 from engine.ast import ColRef, TableInfo, ast_node, Context, include
 from engine.groupby import groupby
 from engine.join import join
@@ -76,7 +75,7 @@ class projection(ast_node):
         if self.group_node is not None:
             # There is group by;
             has_groupby = True
-        k9expr = f'(' 
+        cexpr = f'(' 
         flatten = False
         cols = []
         self.out_table = TableInfo('out_'+base62uuid(4), [], self.context)
@@ -92,25 +91,25 @@ class projection(ast_node):
                     e = proj['value']
                     if type(e) is str:
                         cname = e # TODO: deal w/ alias
-                        k9expr += (f"{self.datasource.parse_tablenames(proj['value'])}")
+                        cexpr += (f"{self.datasource.parse_tablenames(proj['value'])}")
                     elif type(e) is dict:
                         p_expr = expr(self, e)
-                        cname = p_expr.k9expr
+                        cname = p_expr.cexpr
                         compound = True
-                        k9expr += f"{cname}"
+                        cexpr += f"{cname}"
                     cname = ''.join([a if a in base62alp else '' for a in cname])
-                    k9expr += ';'if i < len(self.projections)-1 else ''
+                    cexpr += ';'if i < len(self.projections)-1 else ''
 
             compound = compound and has_groupby and self.datasource.rec not in self.group_node.referenced
             
             cols.append(ColRef(f'{disp_varname}[{i}]', 'generic', self.out_table, 0, None, cname, i, compound=compound))
         self.out_table.add_cols(cols, False)
         
-        k9expr += ')'
+        cexpr += ')'
         if has_groupby:
-            self.group_node.finalize(k9expr, disp_varname)
+            self.group_node.finalize(cexpr, disp_varname)
         else:
-            self.emit(f'{disp_varname}:{k9expr}')
+            self.emit(f'auto {disp_varname} = {cexpr};')
         self.datasource.group_node = None
 
         has_orderby = 'orderby' in node
@@ -126,7 +125,7 @@ class projection(ast_node):
             if len(self.projections) > 1:
                 self.emit_no_ln(f"{'+' if self.inv else ''}{disp_varname}")
             else:
-                self.emit_no_ln(f'$[(#{disp_varname})>1;+,({disp_varname});+,(,{disp_varname})]')
+                self.emit_no_ln(f'print({disp_varname});')
             if flatten:
                 self.emit_no_ln(f'{disp_varname}')
         if has_orderby:

engine/scan.py

@@ -27,12 +27,12 @@ class filter(ast_node):
             if type(self.value) is View: # cond filtered on tables.
                 self.emit(f'{self.value.name}:&{self.value.name}')
                 for o, c in zip(self.output.columns,self.value.table.columns):
-                    self.emit(f'{o.k9name}:{c.k9name}[{self.value.name}]')
+                    self.emit(f'{o.cname}:{c.cname}[{self.value.name}]')
             elif self.value is not None: # cond is scalar
                 tmpVar = 't'+base62uuid(7)
                 self.emit(f'{tmpVar}:{self.value}')
                 for o, c in zip(self.output.columns, self.datasource.columns):
-                    self.emit(f'{o.k9name}:$[{tmpVar};{c.k9name};()]')
+                    self.emit(f'{o.cname}:$[{tmpVar};{c.cname};()]')
                 
     def consume(self, node):
         # TODO: optimizations after converting expr to cnf
@@ -86,7 +86,7 @@ class filter(ast_node):
                     elif type(v) is View:
                         if len(v.table.columns) > 0:
                             all_rows = View(self.context, v.table)
-                            self.emit(f'{all_rows.name}:(#{v.table.columns[0].k9name})#1')
+                            self.emit(f'{all_rows.name}:(#{v.table.columns[0].cname})#1')
                             self.emit(f'{v.name}:{all_rows.name}-{v.name}')
                             self.value = v
                     else:
@@ -97,10 +97,10 @@ class filter(ast_node):
                     if e.isvector:
                         v = View(self.context, self.datasource)
                         v.construct()
-                        self.emit(f'{v.name}:{e.k9expr}')
+                        self.emit(f'{v.name}:{e.cexpr}')
                         self.value = v
                     else:
-                        self.value = e.k9expr
+                        self.value = e.cexpr
             self.__materialize__()        
 
         print(node)

engine/types.py

@@ -0,0 +1,33 @@
+from engine.ast import Context
+
+
+class Types:
+    name = 'Any'
+    cname = 'void*'
+    ctype_name = "types::NONE"
+    def __init__(self, context:Context):
+        self.cxt = context
+    def cast_to(self, *_):
+        return self
+    def cast_from(self, *_):
+        return self
+    def __repr__(self) -> str:
+        return self.cname
+
+class String(Types):
+    name = 'String'
+    cname = 'const char*'
+    ctype_name = "types::ASTR"
+    def cast_from(self, ty, val, container = None):
+        if type(ty) is Int:
+            self.cxt.emit()
+
+class Int(Types):
+    name = "Int"
+    cname = "int"
+    ctype_name = "types::AINT"
+class Float(Types):
+    name = "Float"
+    cname = "float"
+    ctype_name = "types::AFLOAT"
+