Group by

Flatten
Rewritten Agg funcs and csv I/O
bug fixes

.gitignore

@@ -20,6 +20,9 @@ k
 *.pdf
 test*.c*
 *.csv
+!test.csv
+!moving_avg.csv
+!nyctx100.csv
 *.out
 *.asm
 !mmw.so

engine/ast.py

@@ -1,5 +1,7 @@
 from typing import List
 
+from pyparsing import col
+
 from engine.utils import base62uuid
 
 # replace column info with this later.
@@ -23,6 +25,9 @@ class ColRef:
     def __setitem__(self, key, value):
         self.__arr__[key] = value
 
+    def __str__(self):
+        return self.k9name
+
 class TableInfo:
     
     def __init__(self, table_name, cols, cxt:'Context'):
@@ -61,11 +66,13 @@ class TableInfo:
     def n_cols(self):
         return len(self.columns)
 
-    def get_k9colname(self, col_name):
+    def get_col(self, col_name):
         col = self.columns_byname[col_name]
         if type(self.rec) is list:
             self.rec.append(col)
-        return col.k9name
+        return col
+    def get_k9colname(self, col_name):
+        return self.get_col(col_name).k9name
         
     def add_alias(self, alias):
         # TODO: Exception when alias already defined.
@@ -75,14 +82,20 @@ class TableInfo:
         
     def parse_tablenames(self, colExpr):
         parsedColExpr = colExpr.split('.')
+        ret = None
         if len(parsedColExpr) <= 1:
-            return self.get_k9colname(colExpr)
+            ret = self.get_col(colExpr)
         else: 
             datasource = self.cxt.tables_byname[parsedColExpr[0]]
             if datasource is None:
                 raise ValueError(f'Table name/alias not defined{parsedColExpr[0]}')
             else:
-                return datasource.get_k9colname(parsedColExpr[1])
+                ret = datasource.get_col(parsedColExpr[1])
+        if self.groupinfo is not None and ret:
+            ret = f"{ret.k9name}[{'start' if ret in self.groupinfo.referenced else 'range'}]"
+        else:
+            ret = ret.k9name
+        return ret
 
 class View:
     def __init__(self, context, table = None, tmp = True):

engine/ddl.py

@@ -40,24 +40,46 @@ class load(ast_node):
         table:TableInfo = self.context.tables_byname[node['table']]
         n_keys = len(table.columns)
         keys = ''
-        for _ in n_keys:
+        for _ in range(n_keys):
             keys+='`tk'+base62uuid(6)
         tablename = 'l'+base62uuid(7)        
 
-        self.emit(f"{tablename}:[{keys}!+(`csv ? 1:\"{node['file']['literal']}\")][{keys}]")
+        self.emit(f"{tablename}:({keys}!(+(`csv ? 1:\"{node['file']['literal']}\")))[{keys}]")
 
         for i, c in enumerate(table.columns):
-            c:ColRef
             self.emit(f'{c.k9name}:{tablename}[{i}]')
             
 class outfile(ast_node):
     name="_outfile"
     def produce(self, node):
         out_table:TableInfo = self.parent.out_table
-        self.emit_no_ln(f"\"{node['loc']['literal']}\"1:`csv@[[]")
+        filename = node['loc']['literal'] if 'loc' in node else node['literal']
+        self.emit_no_ln(f"\"{filename}\"1:`csv@(+(")
+        l_compound = False
+        l_cols = ''
+        l_keys = ''
+        ending = lambda x: x[:-1] if len(x) > 0 and x[-1]==';' else x
         for i, c in enumerate(out_table.columns):
-            self.emit_no_ln(f"{c.name}:{c.k9name}{';' if i < len(out_table.columns) - 1 else ''}")
-        self.emit(']')
+            c:ColRef
+            l_keys += '`' + c.name
+            if c.compound:
+                if l_compound:
+                    l_cols=f'flatBOTH\'+(({ending(l_cols)});{c.k9name})'
+                else:
+                    l_compound = True
+                    if i >= 1:
+                        l_cols = f'flatRO\'+(({ending(l_cols)});{c.k9name})'
+                    else:
+                        l_cols = c.k9name + ';'
+            elif l_compound:
+                l_cols = f'flatLO\'+(({ending(l_cols)});{c.k9name})'
+            else:
+                l_cols += f"{c.k9name};"
+        if not l_compound:
+            self.emit_no_ln(l_keys + '!(' + ending(l_cols) + ')')
+        else:
+            self.emit_no_ln(f'{l_keys}!+,/({ending(l_cols)})')
+        self.emit('))')
             
 import sys
 include(sys.modules[__name__])

engine/expr.py

@@ -32,6 +32,9 @@ class expr(ast_node):
         'neg' : '-',
         'not' : '~'
     }
+    coumpound_generating_ops = ['mod', 'mins', 'maxs', 'sums'] + \
+       list( binary_ops.keys()) + list(compound_ops.keys()) + list(unary_ops.keys() )
+
     def __init__(self, parent, node):
         ast_node.__init__(self, parent, node, None)
 
@@ -39,6 +42,7 @@ class expr(ast_node):
         from engine.projection import projection
         parent = self.parent
         self.isvector = parent.isvector if type(parent) is expr else False
+        self.is_compound = parent.is_compound if type(parent) is expr else False
         if type(parent) in [projection, expr]:
             self.datasource = parent.datasource
         else:
@@ -59,7 +63,9 @@ class expr(ast_node):
                         for i, p in enumerate(val):
                             self.k9expr += expr(self, p).k9expr + (';'if i<len(val)-1 else '')
                     else:
-                        self.k9expr += f"{self.func_maps[key]}[" 
+                        funcname = self.func_maps[key]
+                        funcname = funcname[0] if type(funcname) is list else funcname
+                        self.k9expr += f"{funcname}[" 
                         self.k9expr += expr(self, val).k9expr
                     self.k9expr += ']'
                 elif key in self.binary_ops:
@@ -76,7 +82,14 @@ class expr(ast_node):
                     self.k9expr += f'({expr(self, val).k9expr}{self.unary_ops[key]})'
                 else:
                     print(f'Undefined expr: {key}{val}')
-                    
+
+                if key in self.coumpound_generating_ops and not self.is_compound:
+                    self.is_compound = True
+                    p = self.parent
+                    while type(p) is expr and not p.is_compound:
+                        p.is_compound = True
+                        p = p.parent
+
         elif type(node) is str:
             p = self.parent
             while type(p) is expr and not p.isvector:

engine/groupby.py

@@ -26,13 +26,30 @@ class groupby(ast_node):
             g_contents += e + (';'if i < len(node)-1 else '')
             
         self.emit(f'{self.group}:'+g_contents+')')
-        
-        if len(node) <= 1:
+        self.n_grps = len(node)
+        if self.n_grps <= 1:
             self.emit(f'{self.group}:={self.group}')
         else:
-            self.emit(f'{self.group}:groupby[+({self.group},(,!(#({first_col}))))]')
+            self.emit(f'{self.group}:groupby[({self.group},(,!(#({first_col}))))]')
         
     def consume(self, _):
         self.referenced = self.datasource.rec
         self.datasource.rec = None
-        return super().consume(_)
+        return super().consume(_)
+
+    def finalize(self, ret, out):
+        self.groupby_function = 'fgrp'+base62uuid(4)
+        grp = self.group
+        if self.n_grps <= 1:
+            k9fn = "{[range] start:*range;"+ ret + "}"
+            self.emit(f'{out}:(({k9fn}\'{grp})[!{grp}])')
+            self.parent.inv = False
+        else:
+            k9fn = "{[ids;grps;ll;dim;x] " + \
+                    "start:$[x=ll;ll;grps[x+1][dim-1]];" + \
+                    "end: grps[x][dim-1];" + \
+                    "range:(end-start)#(((start-ll))#ids);" + \
+                    ret + '}'
+            self.emit(f'{self.groupby_function}:{k9fn}')
+            self.emit(f'{out}:+({self.groupby_function}' + \
+                f'[{grp}[1];{grp}[0];(#{grp}[0])-1;#({grp}[0][0])]\'!((#({grp}[0]))-1))')

engine/orderby.py

@@ -0,0 +1,38 @@
+from engine.ast import ColRef, TableInfo, ast_node
+from engine.utils import base62uuid
+from engine.expr import expr
+
+class orderby(ast_node):
+    name = '_orderby'
+    def init(self, _):
+        self.group = 'g' + base62uuid(7)
+        self.datasource = self.parent.datasource
+        self.datasource.rec = []
+    def produce(self, node):
+        if type(node) is not list:
+            node = [node]
+        g_contents = '('
+        first_col = ''
+        for i, g in enumerate(node):
+            v = g['value']
+            e = expr(self, v).k9expr
+            # if v is compound expr, create tmp cols
+            if type(v) is not str:
+                tmpcol = 't' + base62uuid(7)
+                self.emit(f'{tmpcol}:{e}')
+                e = tmpcol
+            if i == 0:
+                first_col = e
+            g_contents += e + (';'if i < len(node)-1 else '')
+            
+        self.emit(f'{self.group}:'+g_contents+')')
+        self.n_grps = len(node)
+        if self.n_grps <= 1:
+            self.emit(f'{self.group}:={self.group}')
+        else:
+            self.emit(f'{self.group}:groupby[+({self.group},(,!(#({first_col}))))]')
+        
+    def consume(self, _):
+        self.referenced = self.datasource.rec
+        self.datasource.rec = None
+        return super().consume(_)

engine/projection.py

@@ -12,6 +12,7 @@ class projection(ast_node):
     def __init__(self, parent:ast_node, node, context:Context = None, outname = None, disp = True):
         self.disp = disp
         self.outname = outname
+        self.group_node = None
         ast_node.__init__(self, parent, node, context)
     def init(self, _):
         if self.outname is None:
@@ -64,36 +65,19 @@ class projection(ast_node):
 
         if 'groupby' in node:
             self.group_node = groupby(self, node['groupby'])
-            self.datasource = copy(self.datasource) # shallow copy
+            self.datasource = copy.copy(self.datasource) # shallow copy
             self.datasource.groupinfo = self.group_node
         else:
             self.group_node = None
             
     def consume(self, node):
+        self.inv = True
         disp_varname = 'd'+base62uuid(7)
-        pcolrefs = []
-        if type(self.group_node) is groupby:
-            grp_table = self.group_node.group
-            grp_refs = self.group_node.referenced
-            for i, proj in enumerate(self.projections):
-                self.datasource.rec = []
-                cname = ''
-                if type(proj) is dict:
-                    if 'value' in proj:
-                        e = proj['value']
-                        if type(e) is str:
-                            cname = self.datasource.parse_tablenames(proj['value'])
-                        elif type(e) is dict:
-                            cname = expr(self, e).k9expr
-                            cname = ''.join([a if a in base62alp else '' for a in cname])
-                pcolrefs.append(self.datasource.rec)
-                self.datasource.rec = None
-            keys = 'k'+base62uuid(7)
-            self.emit(f'{keys}:!{grp_table}')
-            fn = 'fn' + base62uuid(6)
-            # self.emit
-        
-        self.emit_no_ln(f'{disp_varname}:(')
+        has_groupby = False
+        if self.group_node is not None:
+            # There is group by;
+            has_groupby = True
+        k9expr = f'(' 
         flatten = False
         cols = []
         self.out_table = TableInfo('out_'+base62uuid(4), [], self.context)
@@ -102,27 +86,40 @@ class projection(ast_node):
             
         for i, proj in enumerate(self.projections):
             cname = ''
+            compound = False
+            self.datasource.rec = []
             if type(proj) is dict:
                 if 'value' in proj:
                     e = proj['value']
                     if type(e) is str:
                         cname = self.datasource.parse_tablenames(proj['value'])
-                        self.emit_no_ln(f"{cname}")
+                        k9expr += (f"{cname}")
                     elif type(e) is dict:
-                        cname = expr(self, e).k9expr
-                        self.emit_no_ln(f"{cname}")
-                        cname = ''.join([a if a in base62alp else '' for a in cname])
-                    self.emit_no_ln(';'if i < len(self.projections)-1 else '')
-            cols.append(ColRef(f'(+{disp_varname})[{i}]', 'generic', self.out_table, 0, None, cname, i))
-        self.emit(')')
+                        p_expr = expr(self, e)
+                        cname = p_expr.k9expr
+                        compound = True
+                        k9expr += f"{cname}"
+                    cname = ''.join([a if a in base62alp else '' for a in cname])
+                    k9expr += ';'if i < len(self.projections)-1 else ''
+
+            compound = compound and has_groupby and self.datasource.rec not in self.group_node.referenced
+            
+            cols.append(ColRef(f'(+{disp_varname})[{i}]', 'generic', self.out_table, 0, None, cname, i, compound=compound))
+        k9expr += ')'
+        if has_groupby:
+            self.group_node.finalize(k9expr, disp_varname)
+        else:
+            self.emit(f'{disp_varname}:{k9expr}')
+        
+        self.datasource.group_node = None
         if flatten:
             self.emit_no_ln(f'{disp_varname}:' if flatten else '')
             
         if flatten or self.disp:
             if len(self.projections) > 1:
-                self.emit(f"+{disp_varname}")
+                self.emit(f"{'+' if self.inv else ''}{disp_varname}")
             else:
-                self.emit(f'+,(,{disp_varname})')
+                self.emit(f'$[(#{disp_varname})>1;+,({disp_varname});+,(,{disp_varname})]')
             if flatten:
                 self.emit(f'{disp_varname}')
         if flatten:

header.k

@@ -50,11 +50,15 @@ groupby:{[L]
 lststr:{[L](+({[x;y] ($x,$y)}/L))[0]}
 delist:{[L] $[(@L)in(`LL`LC`LG`L);delist[(,/L)];L]}
 cntlist:{[L;i] $[(@L)in(`LL`LC`LG`L);cntlist[(,/L);i+1];i+1]}
+flatRO:{[x]x[0],/:x[1]}
+flatLO:{[x]x[0],\:x[1]}
+flatBOTH:{[x],/(x[0],/:\:x[1])}
+
 
 sumswkrl:{[L;w;x;y] ((x-L[y-w])+L[y])}
-sumsw:{[L;w] $[(#L)=0;L;(sumswkrl[L;w])\@[!#L;0;L[0]]]}
+sumsw:{[w;L] $[(#L)=0;L;(sumswkrl[L;w])\@[!#L;0;L[file 0]]]}
 avgswkrl:{[L;w;x;y] (x-(L[y-w]-L[y])%w)}
-avgsw:{[L;w] $[(#L)=0;L;(avgswkrl[L;w])\@[!#L;0;L[0]]]}
+avgsw:{[w;L] $[(#L)=0;L;(avgswkrl[L;w])\@[!#L;0;L[0]]]}
 
 / minsw:{[w;L] ({[L;w;x] min(L[$[x>w;(!w) + ((x-w)+1);!(x+1)]])}[L;w])'!#L}
 import`mmw

mmw.cpp

@@ -1,17 +1,12 @@
-
-#include <cstring>
-#include <cstdlib>
 #include <cstdint>
 #include <deque>
 
-using std::size_t;
 using std::uint32_t;
 
 template<class T, bool minmax>
 void running(void *array, uint32_t len, uint32_t w){
-	using std::deque;
 	T* arr = static_cast<T*> (array);
-	deque<std::pair<T, uint32_t>> cache;
+	std::deque<std::pair<T, uint32_t>> cache;
 	for(int i = 0; i < len; ++i){
 		if(!cache.empty() && cache.front().second == i-w) cache.pop_front();
 		if constexpr(minmax)

moving_avg.a

@@ -9,3 +9,6 @@ FROM sale
      ASSUMING ASC Month
 INTO OUTFILE "moving_avg_output.csv"
 FIELDS TERMINATED BY ","
+
+select sales, mins(2,Month) from sale group by sales
+into outfile "flatten.csv" 

nyctx100.csv

@@ -0,0 +1,99 @@
+VendorID,tpep_pickup_datetime,tpep_dropoff_datetime,passenger_count,trip_distance,RatecodeID,store_and_fwd_flag,PULocationID,DOLocationID,payment_type,fare_amount,extra,mta_tax,tip_amount,tolls_amount,improvement_surcharge,total_amount
+1,1/9/2017 11:13,1/9/2017 11:25,1,3.3,1,N,263,161,1,12.5,0,0.5,2,0,0.3,15.3
+1,1/9/2017 11:32,1/9/2017 11:36,1,0.9,1,N,186,234,1,5,0,0.5,1.45,0,0.3,7.25
+1,1/9/2017 11:38,1/9/2017 11:42,1,1.1,1,N,164,161,1,5.5,0,0.5,1,0,0.3,7.3
+1,1/9/2017 11:52,1/9/2017 11:57,1,1.1,1,N,236,75,1,6,0,0.5,1.7,0,0.3,8.5
+2,1/1/2017 0:00,1/1/2017 0:00,1,0.02,2,N,249,234,2,52,0,0.5,0,0,0.3,52.8
+1,1/1/2017 0:00,1/1/2017 0:03,1,0.5,1,N,48,48,2,4,0.5,0.5,0,0,0.3,5.3
+2,1/1/2017 0:00,1/1/2017 0:39,4,7.75,1,N,186,36,1,22,0.5,0.5,4.66,0,0.3,27.96
+1,1/1/2017 0:00,1/1/2017 0:06,1,0.8,1,N,162,161,1,6,0.5,0.5,1.45,0,0.3,8.75
+1,1/1/2017 0:00,1/1/2017 0:08,2,0.9,1,N,48,50,1,7,0.5,0.5,0,0,0.3,8.3
+2,1/1/2017 0:00,1/1/2017 0:05,5,1.76,1,N,140,74,2,7,0.5,0.5,0,0,0.3,8.3
+2,1/1/2017 0:00,1/1/2017 0:15,1,8.47,1,N,138,262,1,24,0.5,0.5,7.71,5.54,0.3,38.55
+1,1/1/2017 0:00,1/1/2017 0:11,2,2.4,1,N,142,236,2,10.5,0.5,0.5,0,0,0.3,11.8
+1,1/1/2017 0:00,1/1/2017 0:23,2,12.6,5,N,161,265,1,60,0,0,10,0,0.3,70.3
+1,1/1/2017 0:00,1/1/2017 0:08,1,0.9,1,N,234,186,1,7,0.5,0.5,2.05,0,0.3,10.35
+2,1/1/2017 0:00,1/1/2017 0:09,4,2.43,1,N,141,107,1,9.5,0.5,0.5,2.7,0,0.3,13.5
+2,1/1/2017 0:00,1/1/2017 0:16,2,2.6,1,N,79,163,1,12.5,0.5,0.5,2.76,0,0.3,16.56
+2,1/1/2017 0:00,1/1/2017 0:18,5,4.25,1,N,148,36,2,16.5,0.5,0.5,0,0,0.3,17.8
+2,1/1/2017 0:00,1/1/2017 0:07,1,0.65,1,N,48,68,1,6.5,0.5,0.5,1.7,0,0.3,9.5
+2,1/1/2017 0:00,1/1/2017 0:34,1,3.42,1,N,230,148,1,22.5,0.5,0.5,0,0,0.3,23.8
+1,1/1/2017 0:00,1/1/2017 0:24,1,6.6,1,N,186,232,2,23,0.5,0.5,0,0,0.3,24.3
+1,1/1/2017 0:00,1/1/2017 0:02,1,0.5,1,N,36,37,2,4,0.5,0.5,0,0,0.3,5.3
+1,1/1/2017 0:00,1/1/2017 0:08,1,1.2,1,N,41,74,1,7.5,0.5,0.5,1.75,0,0.3,10.55
+1,1/1/2017 0:00,1/1/2017 0:12,1,1.7,1,N,125,45,2,9.5,0.5,0.5,0,0,0.3,10.8
+1,1/1/2017 0:00,1/1/2017 0:09,1,5.3,1,N,138,192,2,16,0.5,0.5,0,0,0.3,17.3
+1,1/1/2017 0:00,1/1/2017 0:01,2,0.2,1,N,143,143,2,3,0.5,0.5,0,0,0.3,4.3
+2,1/1/2017 0:00,1/1/2017 0:15,1,4.68,1,N,140,223,1,16.5,0.5,0.5,4.45,0,0.3,22.25
+2,1/1/2017 0:00,1/1/2017 0:12,1,1.9,1,N,68,48,1,10,0.5,0.5,0,0,0.3,11.3
+2,1/1/2017 0:00,1/1/2017 0:24,2,2.1,1,N,239,48,2,16,0.5,0.5,0,0,0.3,17.3
+2,1/1/2017 0:00,1/1/2017 0:13,1,2.75,1,N,233,114,1,11.5,0.5,0.5,2,0,0.3,14.8
+2,1/1/2017 0:00,1/1/2017 0:00,1,0,5,N,14,14,1,32.8,0,0.5,0,0,0.3,33.6
+2,1/1/2017 0:00,1/1/2017 0:10,5,1.53,1,N,239,48,2,6,0.5,0.5,0,0,0.3,7.3
+2,1/1/2017 0:00,1/1/2017 0:16,1,1.26,1,N,164,161,1,10.5,0.5,0.5,2.36,0,0.3,14.16
+1,1/1/2017 0:00,1/1/2017 0:07,1,1,1,N,256,255,2,6.5,0.5,0.5,0,0,0.3,7.8
+2,1/1/2017 0:00,1/1/2017 0:19,2,13.97,1,N,138,181,1,37.5,0.5,0.5,11.64,0,0.3,50.44
+1,1/1/2017 0:00,1/1/2017 0:04,1,0.7,1,N,237,43,2,5.5,0.5,0.5,0,0,0.3,6.8
+1,1/1/2017 0:00,1/1/2017 0:04,1,0.4,1,N,158,68,2,4.5,0.5,0.5,0,0,0.3,5.8
+2,1/1/2017 0:00,1/1/2017 0:20,4,1.85,1,N,229,68,2,13,0.5,0.5,0,0,0.3,14.3
+2,1/1/2017 0:00,1/1/2017 0:08,1,4.29,1,N,132,10,2,13.5,0.5,0.5,0,0,0.3,14.8
+1,1/1/2017 0:00,1/1/2017 0:09,1,4.4,1,N,170,87,1,14.5,0.5,0.5,3.15,0,0.3,18.95
+1,1/1/2017 0:00,1/1/2017 0:01,1,0.1,1,N,243,243,2,3,0.5,0.5,0,0,0.3,4.3
+2,1/1/2017 0:00,1/1/2017 0:03,1,0.24,1,N,161,161,1,4,0.5,0.5,1.06,0,0.3,6.36
+2,1/1/2017 0:00,1/1/2017 0:03,1,0.92,1,N,50,246,1,4.5,0.5,0.5,1.45,0,0.3,7.25
+1,1/1/2017 0:00,1/1/2017 0:18,1,1.1,1,N,237,161,2,12,0.5,0.5,0,0,0.3,13.3
+1,1/1/2017 0:00,1/1/2017 0:12,1,6.9,1,N,138,80,2,20,0.5,0.5,0,0,0.3,21.3
+2,1/1/2017 0:00,1/1/2017 0:04,5,0.6,1,N,238,238,2,5,0.5,0.5,0,0,0.3,6.3
+2,1/1/2017 0:00,1/1/2017 0:03,3,0.56,1,N,74,75,2,4.5,0.5,0.5,0,0,0.3,5.8
+1,1/1/2017 0:00,1/1/2017 0:06,1,0.9,1,N,113,249,1,6,0.5,0.5,2.15,0,0.3,9.45
+2,1/1/2017 0:00,1/1/2017 0:35,1,2.88,1,N,163,113,1,21.5,0.5,0.5,4.56,0,0.3,27.36
+1,1/1/2017 0:00,1/1/2017 0:24,2,3.9,1,N,129,179,2,19.5,0.5,0.5,0,0,0.3,20.8
+2,1/1/2017 0:00,1/1/2017 0:09,1,1.7,1,N,41,75,1,8.5,0.5,0.5,2.45,0,0.3,12.25
+1,1/1/2017 0:00,1/1/2017 0:06,3,0.7,1,N,229,161,1,6,0.5,0.5,1.45,0,0.3,8.75
+1,1/1/2017 0:00,1/1/2017 0:03,4,0.9,1,N,236,236,2,5,0.5,0.5,0,0,0.3,6.3
+2,1/1/2017 0:00,1/1/2017 0:06,1,0.79,1,N,163,237,1,6,0.5,0.5,1.82,0,0.3,9.12
+2,1/1/2017 0:00,1/1/2017 0:08,1,1.25,1,N,234,162,2,7.5,0.5,0.5,0,0,0.3,8.8
+2,1/1/2017 0:00,1/1/2017 0:08,5,2.05,1,N,113,233,1,8.5,0.5,0.5,1,0,0.3,10.8
+2,1/1/2017 0:00,1/1/2017 0:12,1,2.56,1,N,113,229,1,10.5,0.5,0.5,8.2,0,0.3,20
+2,1/1/2017 0:00,1/1/2017 0:11,1,0.1,1,N,264,264,2,8,0.5,0.5,0,0,0.3,9.3
+2,1/1/2017 0:00,1/1/2017 0:08,1,1.78,1,N,113,231,1,8,0.5,0.5,1.86,0,0.3,11.16
+1,1/1/2017 0:00,1/1/2017 0:31,2,10.5,1,N,138,230,2,35.5,0.5,0.5,0,5.54,0.3,42.34
+1,1/1/2017 0:00,1/1/2017 0:05,1,0.9,1,N,148,4,1,6,0.5,0.5,1.45,0,0.3,8.75
+2,1/1/2017 0:00,1/1/2017 0:09,1,1.97,1,N,74,168,2,9,0.5,0.5,0,0,0.3,10.3
+2,1/1/2017 0:00,1/1/2017 0:42,6,20.99,2,N,132,249,2,52,0,0.5,0,0,0.3,52.8
+1,1/1/2017 0:00,1/1/2017 0:02,1,0,1,N,249,249,3,3,0.5,0.5,0,0,0.3,4.3
+2,1/1/2017 0:00,1/1/2017 0:06,2,0.66,1,N,79,79,1,5.5,0.5,0.5,1.36,0,0.3,8.16
+1,1/1/2017 0:00,1/1/2017 0:12,1,3.6,1,N,263,7,2,13,0.5,0.5,0,0,0.3,14.3
+2,1/1/2017 0:00,1/1/2017 0:12,6,4.02,1,N,41,235,1,14,0.5,0.5,3.06,0,0.3,18.36
+2,1/1/2017 0:00,1/1/2017 0:07,2,2.66,1,N,148,233,1,10,0.5,0.5,2.26,0,0.3,13.56
+1,1/1/2017 0:00,1/1/2017 0:19,1,11.4,1,N,138,181,2,31.5,0.5,0.5,0,0,0.3,32.8
+2,1/1/2017 0:00,1/1/2017 0:15,1,6.59,1,N,143,42,2,20,0.5,0.5,0,0,0.3,21.3
+2,1/1/2017 0:00,1/1/2017 0:20,1,2.82,1,N,144,170,2,14,0.5,0.5,0,0,0.3,15.3
+2,1/1/2017 0:00,1/1/2017 0:07,1,1.09,1,N,129,129,2,7,0.5,0.5,0,0,0.3,8.3
+2,1/1/2017 0:00,1/1/2017 0:01,2,0.41,1,N,239,238,1,3.5,0.5,0.5,0.72,0,0.3,5.52
+1,1/1/2017 0:00,1/1/2017 0:13,2,2.1,1,Y,45,137,3,10.5,0.5,0.5,0,0,0.3,11.8
+1,1/1/2017 0:00,1/1/2017 0:03,1,2.2,1,N,137,137,2,3.5,0.5,0.5,0,0,0.3,4.8
+1,1/1/2017 0:00,1/1/2017 0:43,1,20.3,1,N,132,61,1,58,0.5,0.5,5,0,0.3,64.3
+1,1/1/2017 0:00,1/1/2017 0:12,1,2.8,1,N,239,229,1,11.5,0.5,0.5,1,0,0.3,13.8
+2,1/1/2017 0:00,1/1/2017 0:08,3,1.96,1,N,249,88,2,8.5,0.5,0.5,0,0,0.3,9.8
+1,1/1/2017 0:00,1/1/2017 0:01,1,0,1,N,138,138,3,2.5,0.5,0.5,0,0,0.3,3.8
+1,1/1/2017 0:00,1/1/2017 0:11,1,4.8,1,N,162,45,1,15.5,0.5,0.5,3.35,0,0.3,20.15
+2,1/1/2017 0:00,1/1/2017 0:13,2,4.69,1,N,262,90,2,16,0.5,0.5,0,0,0.3,17.3
+2,1/1/2017 0:00,1/1/2017 0:05,5,0.6,1,N,170,233,1,4.5,0.5,0.5,1.16,0,0.3,6.96
+2,1/1/2017 0:00,1/1/2017 0:04,5,0.93,1,N,239,142,1,4.5,0.5,0.5,15,0,0.3,20.8
+2,1/1/2017 0:00,1/1/2017 0:14,2,8.6,1,N,148,7,2,25,0.5,0.5,0,0,0.3,26.3
+2,1/1/2017 0:00,1/1/2017 0:07,5,1.17,1,N,158,90,1,6.5,0.5,0.5,2.34,0,0.3,10.14
+2,1/1/2017 0:00,1/1/2017 0:14,2,4.41,1,N,233,74,1,14.5,0.5,0.5,2,0,0.3,17.8
+1,1/1/2017 0:00,1/1/2017 0:05,1,0.9,1,N,106,106,2,6,0.5,0.5,0,0,0.3,7.3
+1,1/1/2017 0:00,1/1/2017 0:17,2,3,1,N,231,107,1,14,0.5,0.5,2,0,0.3,17.3
+1,1/1/2017 0:00,1/1/2017 0:10,1,1.2,1,N,113,137,2,8,0.5,0.5,0,0,0.3,9.3
+1,1/1/2017 0:00,1/1/2017 0:14,1,1.2,1,N,246,186,1,9.5,0.5,0.5,3.2,0,0.3,14
+1,1/1/2017 0:00,1/1/2017 0:21,1,5.9,1,N,162,129,1,20.5,0.5,0.5,4.35,0,0.3,26.15
+1,1/1/2017 0:00,1/1/2017 0:05,2,0.5,1,N,264,264,1,5,0.5,0.5,1.25,0,0.3,7.55
+2,1/1/2017 0:00,1/1/2017 0:23,1,6.58,1,N,186,152,1,23.5,0.5,0.5,7.44,0,0.3,32.24
+1,1/1/2017 0:01,1/1/2017 0:17,1,3.7,1,N,50,79,1,14,0.5,0.5,3.05,0,0.3,18.35
+2,1/1/2017 0:01,1/1/2017 0:23,1,5.91,1,N,125,37,1,21,0.5,0.5,4.46,0,0.3,26.76
+1,1/1/2017 0:01,1/1/2017 0:09,1,1.2,1,N,152,42,2,7.5,0.5,0.5,0,0,0.3,8.8
+1,1/1/2017 0:01,1/1/2017 0:05,2,1.2,1,N,137,162,1,6,0.5,0.5,1,0,0.3,8.3
+1,1/1/2017 0:01,1/1/2017 0:16,2,7.4,1,N,132,130,2,22.5,0.5,0.5,0,0,0.3,23.8
+2,1/1/2017 0:01,1/1/2017 0:08,6,0.87,1,N,107,107,1,6,0.5,0.5,5,0,0.3,12.3

prompt.py

@@ -6,7 +6,10 @@ import subprocess
 import sys
 if sys.platform != 'win32':
     import readline
-    
+    basecmd = ['bash', '-c', 'k']
+else:
+    basecmd = ['bash.exe', '-c', './k']
+
 test_parser = True
 
 # code to test parser
@@ -18,11 +21,14 @@ res = parser.parse(q)
 
 print(res)
 
+keep = False
+cxt = None
 while test_parser:
     try:
         q = input()
         if q == 'exec':
-            cxt = engine.initialize()
+            if not keep or cxt is None:
+                cxt = engine.initialize()
             stmts_stmts = stmts['stmts']
             if type(stmts_stmts) is list:
                 for s in stmts_stmts:
@@ -32,14 +38,17 @@ while test_parser:
             print(cxt.k9code)
             with open('out.k', 'wb') as outfile:
                 outfile.write((cxt.k9code+'\n\\\\').encode('utf-8'))
-            subprocess.call(['bash.exe', '-c',"./k out.k"])
+            subprocess.call(basecmd[:-1] + [basecmd[-1] + ' out.k'])
             continue
         elif q == 'k':
-            subprocess.call(['bash.exe', '-c',"./k"])
+            subprocess.call(basecmd)
             continue
         elif q == 'print':
             print(stmts)
             continue
+        elif q == 'keep':
+            keep = not keep
+            continue
         trimed = ws.sub(' ', q.lower()).split(' ') 
         if trimed[0].startswith('f'):
             fn = 'stock.a' if len(trimed) <= 1 or len(trimed[1]) == 0 \

test.csv

@@ -0,0 +1,11 @@
+a, b, c, d
+1,1,2,2
+1,2,2,2
+1,2,3,4
+4,2,1,4
+2,1,3,4
+1,2,3,4
+1,2,3,3
+3,2,1,2
+2,1,2,2
+1,2,3,1