bill
/
AQuery
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Group by

Browse Source 
Flatten
Rewritten Agg funcs and csv I/O
bug fixes
dev
BillSun 3 years ago
parent 7eac7837a3
commit b9a8ad3ac7
16 changed files with 285 additions and 61 deletions
Show Stats Download Patch File Download Diff File

3
.gitignore vendored
Unescape View File

@ -20,6 +20,9 @@ k
*.pdf
test*.c*
*.csv
!test.csv
!moving_avg.csv
!nyctx100.csv
*.out
*.asm
!mmw.so

BIN
csv.dylib
View File

Binary file not shown.

21
engine/ast.py
Unescape View File

@ -1,5 +1,7 @@
from typing import List
from pyparsing import col
from engine.utils import base62uuid
# replace column info with this later.
@ -23,6 +25,9 @@ class ColRef:
def __setitem__(self, key, value):
self.__arr__[key] = value
def __str__(self):
return self.k9name
class TableInfo:
def __init__(self, table_name, cols, cxt:'Context'):
@ -61,11 +66,13 @@ class TableInfo:
def n_cols(self):
return len(self.columns)
def get_k9colname(self, col_name):
def get_col(self, col_name):
col = self.columns_byname[col_name]
if type(self.rec) is list:
self.rec.append(col)
return col.k9name
return col
def get_k9colname(self, col_name):
return self.get_col(col_name).k9name
def add_alias(self, alias):
# TODO: Exception when alias already defined.
@ -75,14 +82,20 @@ class TableInfo:
def parse_tablenames(self, colExpr):
parsedColExpr = colExpr.split('.')
ret = None
if len(parsedColExpr) <= 1:
return self.get_k9colname(colExpr)
ret = self.get_col(colExpr)
else:
datasource = self.cxt.tables_byname[parsedColExpr[0]]
if datasource is None:
raise ValueError(f'Table name/alias not defined{parsedColExpr[0]}')
else:
return datasource.get_k9colname(parsedColExpr[1])
ret = datasource.get_col(parsedColExpr[1])
if self.groupinfo is not None and ret:
ret = f"{ret.k9name}[{'start' if ret in self.groupinfo.referenced else 'range'}]"
else:
ret = ret.k9name
return ret
class View:
def __init__(self, context, table = None, tmp = True):

34
engine/ddl.py
Unescape View File

@ -40,24 +40,46 @@ class load(ast_node):
table:TableInfo = self.context.tables_byname[node['table']]
n_keys = len(table.columns)
keys = ''
for _ in n_keys:
for _ in range(n_keys):
keys+='`tk'+base62uuid(6)
tablename = 'l'+base62uuid(7)
self.emit(f"{tablename}:[{keys}!+(`csv ? 1:\"{node['file']['literal']}\")][{keys}]")
self.emit(f"{tablename}:({keys}!(+(`csv ? 1:\"{node['file']['literal']}\")))[{keys}]")
for i, c in enumerate(table.columns):
c:ColRef
self.emit(f'{c.k9name}:{tablename}[{i}]')
class outfile(ast_node):
name="_outfile"
def produce(self, node):
out_table:TableInfo = self.parent.out_table
self.emit_no_ln(f"\"{node['loc']['literal']}\"1:`csv@[[]")
filename = node['loc']['literal'] if 'loc' in node else node['literal']
self.emit_no_ln(f"\"{filename}\"1:`csv@(+(")
l_compound = False
l_cols = ''
l_keys = ''
ending = lambda x: x[:-1] if len(x) > 0 and x[-1]==';' else x
for i, c in enumerate(out_table.columns):
self.emit_no_ln(f"{c.name}:{c.k9name}{';' if i < len(out_table.columns) - 1 else ''}")
self.emit(']')
c:ColRef
l_keys += '`' + c.name
if c.compound:
if l_compound:
l_cols=f'flatBOTH\'+(({ending(l_cols)});{c.k9name})'
else:
l_compound = True
if i >= 1:
l_cols = f'flatRO\'+(({ending(l_cols)});{c.k9name})'
else:
l_cols = c.k9name + ';'
elif l_compound:
l_cols = f'flatLO\'+(({ending(l_cols)});{c.k9name})'
else:
l_cols += f"{c.k9name};"
if not l_compound:
self.emit_no_ln(l_keys + '!(' + ending(l_cols) + ')')
else:
self.emit_no_ln(f'{l_keys}!+,/({ending(l_cols)})')
self.emit('))')
import sys
include(sys.modules[__name__])

17
engine/expr.py
Unescape View File

@ -32,6 +32,9 @@ class expr(ast_node):
'neg' : '-',
'not' : '~'
}
coumpound_generating_ops = ['mod', 'mins', 'maxs', 'sums'] + \
list( binary_ops.keys()) + list(compound_ops.keys()) + list(unary_ops.keys() )
def __init__(self, parent, node):
ast_node.__init__(self, parent, node, None)
@ -39,6 +42,7 @@ class expr(ast_node):
from engine.projection import projection
parent = self.parent
self.isvector = parent.isvector if type(parent) is expr else False
self.is_compound = parent.is_compound if type(parent) is expr else False
if type(parent) in [projection, expr]:
self.datasource = parent.datasource
else:
@ -59,7 +63,9 @@ class expr(ast_node):
for i, p in enumerate(val):
self.k9expr += expr(self, p).k9expr + (';'if i<len(val)-1 else '')
else:
self.k9expr += f"{self.func_maps[key]}["
funcname = self.func_maps[key]
funcname = funcname[0] if type(funcname) is list else funcname
self.k9expr += f"{funcname}["
self.k9expr += expr(self, val).k9expr
self.k9expr += ']'
elif key in self.binary_ops:
@ -76,7 +82,14 @@ class expr(ast_node):
self.k9expr += f'({expr(self, val).k9expr}{self.unary_ops[key]})'
else:
print(f'Undefined expr: {key}{val}')
if key in self.coumpound_generating_ops and not self.is_compound:
self.is_compound = True
p = self.parent
while type(p) is expr and not p.is_compound:
p.is_compound = True
p = p.parent
elif type(node) is str:
p = self.parent
while type(p) is expr and not p.isvector:

25
engine/groupby.py
Unescape View File

@ -26,13 +26,30 @@ class groupby(ast_node):
g_contents += e + (';'if i < len(node)-1 else '')
self.emit(f'{self.group}:'+g_contents+')')
if len(node) <= 1:
self.n_grps = len(node)
if self.n_grps <= 1:
self.emit(f'{self.group}:={self.group}')
else:
self.emit(f'{self.group}:groupby[+({self.group},(,!(#({first_col}))))]')
self.emit(f'{self.group}:groupby[({self.group},(,!(#({first_col}))))]')
def consume(self, _):
self.referenced = self.datasource.rec
self.datasource.rec = None
return super().consume(_)
return super().consume(_)
def finalize(self, ret, out):
self.groupby_function = 'fgrp'+base62uuid(4)
grp = self.group
if self.n_grps <= 1:
k9fn = "{[range] start:*range;"+ ret + "}"
self.emit(f'{out}:(({k9fn}\'{grp})[!{grp}])')
self.parent.inv = False
else:
k9fn = "{[ids;grps;ll;dim;x] " + \
"start:$[x=ll;ll;grps[x+1][dim-1]];" + \
"end: grps[x][dim-1];" + \
"range:(end-start)#(((start-ll))#ids);" + \
ret + '}'
self.emit(f'{self.groupby_function}:{k9fn}')
self.emit(f'{out}:+({self.groupby_function}' + \
f'[{grp}[1];{grp}[0];(#{grp}[0])-1;#({grp}[0][0])]\'!((#({grp}[0]))-1))')

38
engine/orderby.py
Unescape View File

@ -0,0 +1,38 @@
from engine.ast import ColRef, TableInfo, ast_node
from engine.utils import base62uuid
from engine.expr import expr
class orderby(ast_node):
name = '_orderby'
def init(self, _):
self.group = 'g' + base62uuid(7)
self.datasource = self.parent.datasource
self.datasource.rec = []
def produce(self, node):
if type(node) is not list:
node = [node]
g_contents = '('
first_col = ''
for i, g in enumerate(node):
v = g['value']
e = expr(self, v).k9expr
# if v is compound expr, create tmp cols
if type(v) is not str:
tmpcol = 't' + base62uuid(7)
self.emit(f'{tmpcol}:{e}')
e = tmpcol
if i == 0:
first_col = e
g_contents += e + (';'if i < len(node)-1 else '')
self.emit(f'{self.group}:'+g_contents+')')
self.n_grps = len(node)
if self.n_grps <= 1:
self.emit(f'{self.group}:={self.group}')
else:
self.emit(f'{self.group}:groupby[+({self.group},(,!(#({first_col}))))]')
def consume(self, _):
self.referenced = self.datasource.rec
self.datasource.rec = None
return super().consume(_)

63
engine/projection.py
Unescape View File

@ -12,6 +12,7 @@ class projection(ast_node):
def __init__(self, parent:ast_node, node, context:Context = None, outname = None, disp = True):
self.disp = disp
self.outname = outname
self.group_node = None
ast_node.__init__(self, parent, node, context)
def init(self, _):
if self.outname is None:
@ -64,36 +65,19 @@ class projection(ast_node):
if 'groupby' in node:
self.group_node = groupby(self, node['groupby'])
self.datasource = copy(self.datasource) # shallow copy
self.datasource = copy.copy(self.datasource) # shallow copy
self.datasource.groupinfo = self.group_node
else:
self.group_node = None
def consume(self, node):
self.inv = True
disp_varname = 'd'+base62uuid(7)
pcolrefs = []
if type(self.group_node) is groupby:
grp_table = self.group_node.group
grp_refs = self.group_node.referenced
for i, proj in enumerate(self.projections):
self.datasource.rec = []
cname = ''
if type(proj) is dict:
if 'value' in proj:
e = proj['value']
if type(e) is str:
cname = self.datasource.parse_tablenames(proj['value'])
elif type(e) is dict:
cname = expr(self, e).k9expr
cname = ''.join([a if a in base62alp else '' for a in cname])
pcolrefs.append(self.datasource.rec)
self.datasource.rec = None
keys = 'k'+base62uuid(7)
self.emit(f'{keys}:!{grp_table}')
fn = 'fn' + base62uuid(6)
# self.emit
self.emit_no_ln(f'{disp_varname}:(')
has_groupby = False
if self.group_node is not None:
# There is group by;
has_groupby = True
k9expr = f'('
flatten = False
cols = []
self.out_table = TableInfo('out_'+base62uuid(4), [], self.context)
@ -102,27 +86,40 @@ class projection(ast_node):
for i, proj in enumerate(self.projections):
cname = ''
compound = False
self.datasource.rec = []
if type(proj) is dict:
if 'value' in proj:
e = proj['value']
if type(e) is str:
cname = self.datasource.parse_tablenames(proj['value'])
self.emit_no_ln(f"{cname}")
k9expr += (f"{cname}")
elif type(e) is dict:
cname = expr(self, e).k9expr
self.emit_no_ln(f"{cname}")
cname = ''.join([a if a in base62alp else '' for a in cname])
self.emit_no_ln(';'if i < len(self.projections)-1 else '')
cols.append(ColRef(f'(+{disp_varname})[{i}]', 'generic', self.out_table, 0, None, cname, i))
self.emit(')')
p_expr = expr(self, e)
cname = p_expr.k9expr
compound = True
k9expr += f"{cname}"
cname = ''.join([a if a in base62alp else '' for a in cname])
k9expr += ';'if i < len(self.projections)-1 else ''
compound = compound and has_groupby and self.datasource.rec not in self.group_node.referenced
cols.append(ColRef(f'(+{disp_varname})[{i}]', 'generic', self.out_table, 0, None, cname, i, compound=compound))
k9expr += ')'
if has_groupby:
self.group_node.finalize(k9expr, disp_varname)
else:
self.emit(f'{disp_varname}:{k9expr}')
self.datasource.group_node = None
if flatten:
self.emit_no_ln(f'{disp_varname}:' if flatten else '')
if flatten or self.disp:
if len(self.projections) > 1:
self.emit(f"+{disp_varname}")
self.emit(f"{'+' if self.inv else ''}{disp_varname}")
else:
self.emit(f'+,(,{disp_varname})')
self.emit(f'$[(#{disp_varname})>1;+,({disp_varname});+,(,{disp_varname})]')
if flatten:
self.emit(f'{disp_varname}')
if flatten:

8
header.k
Unescape View File

@ -50,11 +50,15 @@ groupby:{[L]
lststr:{[L](+({[x;y] ($x,$y)}/L))[0]}
delist:{[L] $[(@L)in(`LL`LC`LG`L);delist[(,/L)];L]}
cntlist:{[L;i] $[(@L)in(`LL`LC`LG`L);cntlist[(,/L);i+1];i+1]}
flatRO:{[x]x[0],/:x[1]}
flatLO:{[x]x[0],\:x[1]}
flatBOTH:{[x],/(x[0],/:\:x[1])}
sumswkrl:{[L;w;x;y] ((x-L[y-w])+L[y])}
sumsw:{[L;w] $[(#L)=0;L;(sumswkrl[L;w])\@[!#L;0;L[0]]]}
sumsw:{[w;L] $[(#L)=0;L;(sumswkrl[L;w])\@[!#L;0;L[file 0]]]}
avgswkrl:{[L;w;x;y] (x-(L[y-w]-L[y])%w)}
avgsw:{[L;w] $[(#L)=0;L;(avgswkrl[L;w])\@[!#L;0;L[0]]]}
avgsw:{[w;L] $[(#L)=0;L;(avgswkrl[L;w])\@[!#L;0;L[0]]]}
/ minsw:{[w;L] ({[L;w;x] min(L[$[x>w;(!w) + ((x-w)+1);!(x+1)]])}[L;w])'!#L}
import`mmw

7
mmw.cpp
Unescape View File

@ -1,17 +1,12 @@
#include <cstring>
#include <cstdlib>
#include <cstdint>
#include <deque>
using std::size_t;
using std::uint32_t;
template<class T, bool minmax>
void running(void *array, uint32_t len, uint32_t w){
using std::deque;
T* arr = static_cast<T*> (array);
deque<std::pair<T, uint32_t>> cache;
std::deque<std::pair<T, uint32_t>> cache;
for(int i = 0; i < len; ++i){
if(!cache.empty() && cache.front().second == i-w) cache.pop_front();
if constexpr(minmax)

BIN
mmw.dylib
View File

Binary file not shown.

0
mmw.so
Unescape View File

3
moving_avg.a
Unescape View File

@ -9,3 +9,6 @@ FROM sale
ASSUMING ASC Month
INTO OUTFILE "moving_avg_output.csv"
FIELDS TERMINATED BY ","
select sales, mins(2,Month) from sale group by sales
into outfile "flatten.csv"

99
nyctx100.csv
Unescape View File

@ -0,0 +1,99 @@
VendorID,tpep_pickup_datetime,tpep_dropoff_datetime,passenger_count,trip_distance,RatecodeID,store_and_fwd_flag,PULocationID,DOLocationID,payment_type,fare_amount,extra,mta_tax,tip_amount,tolls_amount,improvement_surcharge,total_amount
1,1/9/2017 11:13,1/9/2017 11:25,1,3.3,1,N,263,161,1,12.5,0,0.5,2,0,0.3,15.3
1,1/9/2017 11:32,1/9/2017 11:36,1,0.9,1,N,186,234,1,5,0,0.5,1.45,0,0.3,7.25
1,1/9/2017 11:38,1/9/2017 11:42,1,1.1,1,N,164,161,1,5.5,0,0.5,1,0,0.3,7.3
1,1/9/2017 11:52,1/9/2017 11:57,1,1.1,1,N,236,75,1,6,0,0.5,1.7,0,0.3,8.5
2,1/1/2017 0:00,1/1/2017 0:00,1,0.02,2,N,249,234,2,52,0,0.5,0,0,0.3,52.8
1,1/1/2017 0:00,1/1/2017 0:03,1,0.5,1,N,48,48,2,4,0.5,0.5,0,0,0.3,5.3
2,1/1/2017 0:00,1/1/2017 0:39,4,7.75,1,N,186,36,1,22,0.5,0.5,4.66,0,0.3,27.96
1,1/1/2017 0:00,1/1/2017 0:06,1,0.8,1,N,162,161,1,6,0.5,0.5,1.45,0,0.3,8.75
1,1/1/2017 0:00,1/1/2017 0:08,2,0.9,1,N,48,50,1,7,0.5,0.5,0,0,0.3,8.3
2,1/1/2017 0:00,1/1/2017 0:05,5,1.76,1,N,140,74,2,7,0.5,0.5,0,0,0.3,8.3
2,1/1/2017 0:00,1/1/2017 0:15,1,8.47,1,N,138,262,1,24,0.5,0.5,7.71,5.54,0.3,38.55
1,1/1/2017 0:00,1/1/2017 0:11,2,2.4,1,N,142,236,2,10.5,0.5,0.5,0,0,0.3,11.8
1,1/1/2017 0:00,1/1/2017 0:23,2,12.6,5,N,161,265,1,60,0,0,10,0,0.3,70.3
1,1/1/2017 0:00,1/1/2017 0:08,1,0.9,1,N,234,186,1,7,0.5,0.5,2.05,0,0.3,10.35
2,1/1/2017 0:00,1/1/2017 0:09,4,2.43,1,N,141,107,1,9.5,0.5,0.5,2.7,0,0.3,13.5
2,1/1/2017 0:00,1/1/2017 0:16,2,2.6,1,N,79,163,1,12.5,0.5,0.5,2.76,0,0.3,16.56
2,1/1/2017 0:00,1/1/2017 0:18,5,4.25,1,N,148,36,2,16.5,0.5,0.5,0,0,0.3,17.8
2,1/1/2017 0:00,1/1/2017 0:07,1,0.65,1,N,48,68,1,6.5,0.5,0.5,1.7,0,0.3,9.5
2,1/1/2017 0:00,1/1/2017 0:34,1,3.42,1,N,230,148,1,22.5,0.5,0.5,0,0,0.3,23.8
1,1/1/2017 0:00,1/1/2017 0:24,1,6.6,1,N,186,232,2,23,0.5,0.5,0,0,0.3,24.3
1,1/1/2017 0:00,1/1/2017 0:02,1,0.5,1,N,36,37,2,4,0.5,0.5,0,0,0.3,5.3
1,1/1/2017 0:00,1/1/2017 0:08,1,1.2,1,N,41,74,1,7.5,0.5,0.5,1.75,0,0.3,10.55
1,1/1/2017 0:00,1/1/2017 0:12,1,1.7,1,N,125,45,2,9.5,0.5,0.5,0,0,0.3,10.8
1,1/1/2017 0:00,1/1/2017 0:09,1,5.3,1,N,138,192,2,16,0.5,0.5,0,0,0.3,17.3
1,1/1/2017 0:00,1/1/2017 0:01,2,0.2,1,N,143,143,2,3,0.5,0.5,0,0,0.3,4.3
2,1/1/2017 0:00,1/1/2017 0:15,1,4.68,1,N,140,223,1,16.5,0.5,0.5,4.45,0,0.3,22.25
2,1/1/2017 0:00,1/1/2017 0:12,1,1.9,1,N,68,48,1,10,0.5,0.5,0,0,0.3,11.3
2,1/1/2017 0:00,1/1/2017 0:24,2,2.1,1,N,239,48,2,16,0.5,0.5,0,0,0.3,17.3
2,1/1/2017 0:00,1/1/2017 0:13,1,2.75,1,N,233,114,1,11.5,0.5,0.5,2,0,0.3,14.8
2,1/1/2017 0:00,1/1/2017 0:00,1,0,5,N,14,14,1,32.8,0,0.5,0,0,0.3,33.6
2,1/1/2017 0:00,1/1/2017 0:10,5,1.53,1,N,239,48,2,6,0.5,0.5,0,0,0.3,7.3
2,1/1/2017 0:00,1/1/2017 0:16,1,1.26,1,N,164,161,1,10.5,0.5,0.5,2.36,0,0.3,14.16
1,1/1/2017 0:00,1/1/2017 0:07,1,1,1,N,256,255,2,6.5,0.5,0.5,0,0,0.3,7.8
2,1/1/2017 0:00,1/1/2017 0:19,2,13.97,1,N,138,181,1,37.5,0.5,0.5,11.64,0,0.3,50.44
1,1/1/2017 0:00,1/1/2017 0:04,1,0.7,1,N,237,43,2,5.5,0.5,0.5,0,0,0.3,6.8
1,1/1/2017 0:00,1/1/2017 0:04,1,0.4,1,N,158,68,2,4.5,0.5,0.5,0,0,0.3,5.8
2,1/1/2017 0:00,1/1/2017 0:20,4,1.85,1,N,229,68,2,13,0.5,0.5,0,0,0.3,14.3
2,1/1/2017 0:00,1/1/2017 0:08,1,4.29,1,N,132,10,2,13.5,0.5,0.5,0,0,0.3,14.8
1,1/1/2017 0:00,1/1/2017 0:09,1,4.4,1,N,170,87,1,14.5,0.5,0.5,3.15,0,0.3,18.95
1,1/1/2017 0:00,1/1/2017 0:01,1,0.1,1,N,243,243,2,3,0.5,0.5,0,0,0.3,4.3
2,1/1/2017 0:00,1/1/2017 0:03,1,0.24,1,N,161,161,1,4,0.5,0.5,1.06,0,0.3,6.36
2,1/1/2017 0:00,1/1/2017 0:03,1,0.92,1,N,50,246,1,4.5,0.5,0.5,1.45,0,0.3,7.25
1,1/1/2017 0:00,1/1/2017 0:18,1,1.1,1,N,237,161,2,12,0.5,0.5,0,0,0.3,13.3
1,1/1/2017 0:00,1/1/2017 0:12,1,6.9,1,N,138,80,2,20,0.5,0.5,0,0,0.3,21.3
2,1/1/2017 0:00,1/1/2017 0:04,5,0.6,1,N,238,238,2,5,0.5,0.5,0,0,0.3,6.3
2,1/1/2017 0:00,1/1/2017 0:03,3,0.56,1,N,74,75,2,4.5,0.5,0.5,0,0,0.3,5.8
1,1/1/2017 0:00,1/1/2017 0:06,1,0.9,1,N,113,249,1,6,0.5,0.5,2.15,0,0.3,9.45
2,1/1/2017 0:00,1/1/2017 0:35,1,2.88,1,N,163,113,1,21.5,0.5,0.5,4.56,0,0.3,27.36
1,1/1/2017 0:00,1/1/2017 0:24,2,3.9,1,N,129,179,2,19.5,0.5,0.5,0,0,0.3,20.8
2,1/1/2017 0:00,1/1/2017 0:09,1,1.7,1,N,41,75,1,8.5,0.5,0.5,2.45,0,0.3,12.25
1,1/1/2017 0:00,1/1/2017 0:06,3,0.7,1,N,229,161,1,6,0.5,0.5,1.45,0,0.3,8.75
1,1/1/2017 0:00,1/1/2017 0:03,4,0.9,1,N,236,236,2,5,0.5,0.5,0,0,0.3,6.3
2,1/1/2017 0:00,1/1/2017 0:06,1,0.79,1,N,163,237,1,6,0.5,0.5,1.82,0,0.3,9.12
2,1/1/2017 0:00,1/1/2017 0:08,1,1.25,1,N,234,162,2,7.5,0.5,0.5,0,0,0.3,8.8
2,1/1/2017 0:00,1/1/2017 0:08,5,2.05,1,N,113,233,1,8.5,0.5,0.5,1,0,0.3,10.8
2,1/1/2017 0:00,1/1/2017 0:12,1,2.56,1,N,113,229,1,10.5,0.5,0.5,8.2,0,0.3,20
2,1/1/2017 0:00,1/1/2017 0:11,1,0.1,1,N,264,264,2,8,0.5,0.5,0,0,0.3,9.3
2,1/1/2017 0:00,1/1/2017 0:08,1,1.78,1,N,113,231,1,8,0.5,0.5,1.86,0,0.3,11.16
1,1/1/2017 0:00,1/1/2017 0:31,2,10.5,1,N,138,230,2,35.5,0.5,0.5,0,5.54,0.3,42.34
1,1/1/2017 0:00,1/1/2017 0:05,1,0.9,1,N,148,4,1,6,0.5,0.5,1.45,0,0.3,8.75
2,1/1/2017 0:00,1/1/2017 0:09,1,1.97,1,N,74,168,2,9,0.5,0.5,0,0,0.3,10.3
2,1/1/2017 0:00,1/1/2017 0:42,6,20.99,2,N,132,249,2,52,0,0.5,0,0,0.3,52.8
1,1/1/2017 0:00,1/1/2017 0:02,1,0,1,N,249,249,3,3,0.5,0.5,0,0,0.3,4.3
2,1/1/2017 0:00,1/1/2017 0:06,2,0.66,1,N,79,79,1,5.5,0.5,0.5,1.36,0,0.3,8.16
1,1/1/2017 0:00,1/1/2017 0:12,1,3.6,1,N,263,7,2,13,0.5,0.5,0,0,0.3,14.3
2,1/1/2017 0:00,1/1/2017 0:12,6,4.02,1,N,41,235,1,14,0.5,0.5,3.06,0,0.3,18.36
2,1/1/2017 0:00,1/1/2017 0:07,2,2.66,1,N,148,233,1,10,0.5,0.5,2.26,0,0.3,13.56
1,1/1/2017 0:00,1/1/2017 0:19,1,11.4,1,N,138,181,2,31.5,0.5,0.5,0,0,0.3,32.8
2,1/1/2017 0:00,1/1/2017 0:15,1,6.59,1,N,143,42,2,20,0.5,0.5,0,0,0.3,21.3
2,1/1/2017 0:00,1/1/2017 0:20,1,2.82,1,N,144,170,2,14,0.5,0.5,0,0,0.3,15.3
2,1/1/2017 0:00,1/1/2017 0:07,1,1.09,1,N,129,129,2,7,0.5,0.5,0,0,0.3,8.3
2,1/1/2017 0:00,1/1/2017 0:01,2,0.41,1,N,239,238,1,3.5,0.5,0.5,0.72,0,0.3,5.52
1,1/1/2017 0:00,1/1/2017 0:13,2,2.1,1,Y,45,137,3,10.5,0.5,0.5,0,0,0.3,11.8
1,1/1/2017 0:00,1/1/2017 0:03,1,2.2,1,N,137,137,2,3.5,0.5,0.5,0,0,0.3,4.8
1,1/1/2017 0:00,1/1/2017 0:43,1,20.3,1,N,132,61,1,58,0.5,0.5,5,0,0.3,64.3
1,1/1/2017 0:00,1/1/2017 0:12,1,2.8,1,N,239,229,1,11.5,0.5,0.5,1,0,0.3,13.8
2,1/1/2017 0:00,1/1/2017 0:08,3,1.96,1,N,249,88,2,8.5,0.5,0.5,0,0,0.3,9.8
1,1/1/2017 0:00,1/1/2017 0:01,1,0,1,N,138,138,3,2.5,0.5,0.5,0,0,0.3,3.8
1,1/1/2017 0:00,1/1/2017 0:11,1,4.8,1,N,162,45,1,15.5,0.5,0.5,3.35,0,0.3,20.15
2,1/1/2017 0:00,1/1/2017 0:13,2,4.69,1,N,262,90,2,16,0.5,0.5,0,0,0.3,17.3
2,1/1/2017 0:00,1/1/2017 0:05,5,0.6,1,N,170,233,1,4.5,0.5,0.5,1.16,0,0.3,6.96
2,1/1/2017 0:00,1/1/2017 0:04,5,0.93,1,N,239,142,1,4.5,0.5,0.5,15,0,0.3,20.8
2,1/1/2017 0:00,1/1/2017 0:14,2,8.6,1,N,148,7,2,25,0.5,0.5,0,0,0.3,26.3
2,1/1/2017 0:00,1/1/2017 0:07,5,1.17,1,N,158,90,1,6.5,0.5,0.5,2.34,0,0.3,10.14
2,1/1/2017 0:00,1/1/2017 0:14,2,4.41,1,N,233,74,1,14.5,0.5,0.5,2,0,0.3,17.8
1,1/1/2017 0:00,1/1/2017 0:05,1,0.9,1,N,106,106,2,6,0.5,0.5,0,0,0.3,7.3
1,1/1/2017 0:00,1/1/2017 0:17,2,3,1,N,231,107,1,14,0.5,0.5,2,0,0.3,17.3
1,1/1/2017 0:00,1/1/2017 0:10,1,1.2,1,N,113,137,2,8,0.5,0.5,0,0,0.3,9.3
1,1/1/2017 0:00,1/1/2017 0:14,1,1.2,1,N,246,186,1,9.5,0.5,0.5,3.2,0,0.3,14
1,1/1/2017 0:00,1/1/2017 0:21,1,5.9,1,N,162,129,1,20.5,0.5,0.5,4.35,0,0.3,26.15
1,1/1/2017 0:00,1/1/2017 0:05,2,0.5,1,N,264,264,1,5,0.5,0.5,1.25,0,0.3,7.55
2,1/1/2017 0:00,1/1/2017 0:23,1,6.58,1,N,186,152,1,23.5,0.5,0.5,7.44,0,0.3,32.24
1,1/1/2017 0:01,1/1/2017 0:17,1,3.7,1,N,50,79,1,14,0.5,0.5,3.05,0,0.3,18.35
2,1/1/2017 0:01,1/1/2017 0:23,1,5.91,1,N,125,37,1,21,0.5,0.5,4.46,0,0.3,26.76
1,1/1/2017 0:01,1/1/2017 0:09,1,1.2,1,N,152,42,2,7.5,0.5,0.5,0,0,0.3,8.8
1,1/1/2017 0:01,1/1/2017 0:05,2,1.2,1,N,137,162,1,6,0.5,0.5,1,0,0.3,8.3
1,1/1/2017 0:01,1/1/2017 0:16,2,7.4,1,N,132,130,2,22.5,0.5,0.5,0,0,0.3,23.8
2,1/1/2017 0:01,1/1/2017 0:08,6,0.87,1,N,107,107,1,6,0.5,0.5,5,0,0.3,12.3
1 VendorID tpep_pickup_datetime tpep_dropoff_datetime passenger_count trip_distance RatecodeID store_and_fwd_flag PULocationID DOLocationID payment_type fare_amount extra mta_tax tip_amount tolls_amount improvement_surcharge total_amount
2 1 1/9/2017 11:13 1/9/2017 11:25 1 3.3 1 N 263 161 1 12.5 0 0.5 2 0 0.3 15.3
3 1 1/9/2017 11:32 1/9/2017 11:36 1 0.9 1 N 186 234 1 5 0 0.5 1.45 0 0.3 7.25
4 1 1/9/2017 11:38 1/9/2017 11:42 1 1.1 1 N 164 161 1 5.5 0 0.5 1 0 0.3 7.3
5 1 1/9/2017 11:52 1/9/2017 11:57 1 1.1 1 N 236 75 1 6 0 0.5 1.7 0 0.3 8.5
6 2 1/1/2017 0:00 1/1/2017 0:00 1 0.02 2 N 249 234 2 52 0 0.5 0 0 0.3 52.8
7 1 1/1/2017 0:00 1/1/2017 0:03 1 0.5 1 N 48 48 2 4 0.5 0.5 0 0 0.3 5.3
8 2 1/1/2017 0:00 1/1/2017 0:39 4 7.75 1 N 186 36 1 22 0.5 0.5 4.66 0 0.3 27.96
9 1 1/1/2017 0:00 1/1/2017 0:06 1 0.8 1 N 162 161 1 6 0.5 0.5 1.45 0 0.3 8.75
10 1 1/1/2017 0:00 1/1/2017 0:08 2 0.9 1 N 48 50 1 7 0.5 0.5 0 0 0.3 8.3
11 2 1/1/2017 0:00 1/1/2017 0:05 5 1.76 1 N 140 74 2 7 0.5 0.5 0 0 0.3 8.3
12 2 1/1/2017 0:00 1/1/2017 0:15 1 8.47 1 N 138 262 1 24 0.5 0.5 7.71 5.54 0.3 38.55
13 1 1/1/2017 0:00 1/1/2017 0:11 2 2.4 1 N 142 236 2 10.5 0.5 0.5 0 0 0.3 11.8
14 1 1/1/2017 0:00 1/1/2017 0:23 2 12.6 5 N 161 265 1 60 0 0 10 0 0.3 70.3
15 1 1/1/2017 0:00 1/1/2017 0:08 1 0.9 1 N 234 186 1 7 0.5 0.5 2.05 0 0.3 10.35
16 2 1/1/2017 0:00 1/1/2017 0:09 4 2.43 1 N 141 107 1 9.5 0.5 0.5 2.7 0 0.3 13.5
17 2 1/1/2017 0:00 1/1/2017 0:16 2 2.6 1 N 79 163 1 12.5 0.5 0.5 2.76 0 0.3 16.56
18 2 1/1/2017 0:00 1/1/2017 0:18 5 4.25 1 N 148 36 2 16.5 0.5 0.5 0 0 0.3 17.8
19 2 1/1/2017 0:00 1/1/2017 0:07 1 0.65 1 N 48 68 1 6.5 0.5 0.5 1.7 0 0.3 9.5
20 2 1/1/2017 0:00 1/1/2017 0:34 1 3.42 1 N 230 148 1 22.5 0.5 0.5 0 0 0.3 23.8
21 1 1/1/2017 0:00 1/1/2017 0:24 1 6.6 1 N 186 232 2 23 0.5 0.5 0 0 0.3 24.3
22 1 1/1/2017 0:00 1/1/2017 0:02 1 0.5 1 N 36 37 2 4 0.5 0.5 0 0 0.3 5.3
23 1 1/1/2017 0:00 1/1/2017 0:08 1 1.2 1 N 41 74 1 7.5 0.5 0.5 1.75 0 0.3 10.55
24 1 1/1/2017 0:00 1/1/2017 0:12 1 1.7 1 N 125 45 2 9.5 0.5 0.5 0 0 0.3 10.8
25 1 1/1/2017 0:00 1/1/2017 0:09 1 5.3 1 N 138 192 2 16 0.5 0.5 0 0 0.3 17.3
26 1 1/1/2017 0:00 1/1/2017 0:01 2 0.2 1 N 143 143 2 3 0.5 0.5 0 0 0.3 4.3
27 2 1/1/2017 0:00 1/1/2017 0:15 1 4.68 1 N 140 223 1 16.5 0.5 0.5 4.45 0 0.3 22.25
28 2 1/1/2017 0:00 1/1/2017 0:12 1 1.9 1 N 68 48 1 10 0.5 0.5 0 0 0.3 11.3
29 2 1/1/2017 0:00 1/1/2017 0:24 2 2.1 1 N 239 48 2 16 0.5 0.5 0 0 0.3 17.3
30 2 1/1/2017 0:00 1/1/2017 0:13 1 2.75 1 N 233 114 1 11.5 0.5 0.5 2 0 0.3 14.8
31 2 1/1/2017 0:00 1/1/2017 0:00 1 0 5 N 14 14 1 32.8 0 0.5 0 0 0.3 33.6
32 2 1/1/2017 0:00 1/1/2017 0:10 5 1.53 1 N 239 48 2 6 0.5 0.5 0 0 0.3 7.3
33 2 1/1/2017 0:00 1/1/2017 0:16 1 1.26 1 N 164 161 1 10.5 0.5 0.5 2.36 0 0.3 14.16
34 1 1/1/2017 0:00 1/1/2017 0:07 1 1 1 N 256 255 2 6.5 0.5 0.5 0 0 0.3 7.8
35 2 1/1/2017 0:00 1/1/2017 0:19 2 13.97 1 N 138 181 1 37.5 0.5 0.5 11.64 0 0.3 50.44
36 1 1/1/2017 0:00 1/1/2017 0:04 1 0.7 1 N 237 43 2 5.5 0.5 0.5 0 0 0.3 6.8
37 1 1/1/2017 0:00 1/1/2017 0:04 1 0.4 1 N 158 68 2 4.5 0.5 0.5 0 0 0.3 5.8
38 2 1/1/2017 0:00 1/1/2017 0:20 4 1.85 1 N 229 68 2 13 0.5 0.5 0 0 0.3 14.3
39 2 1/1/2017 0:00 1/1/2017 0:08 1 4.29 1 N 132 10 2 13.5 0.5 0.5 0 0 0.3 14.8
40 1 1/1/2017 0:00 1/1/2017 0:09 1 4.4 1 N 170 87 1 14.5 0.5 0.5 3.15 0 0.3 18.95
41 1 1/1/2017 0:00 1/1/2017 0:01 1 0.1 1 N 243 243 2 3 0.5 0.5 0 0 0.3 4.3
42 2 1/1/2017 0:00 1/1/2017 0:03 1 0.24 1 N 161 161 1 4 0.5 0.5 1.06 0 0.3 6.36
43 2 1/1/2017 0:00 1/1/2017 0:03 1 0.92 1 N 50 246 1 4.5 0.5 0.5 1.45 0 0.3 7.25
44 1 1/1/2017 0:00 1/1/2017 0:18 1 1.1 1 N 237 161 2 12 0.5 0.5 0 0 0.3 13.3
45 1 1/1/2017 0:00 1/1/2017 0:12 1 6.9 1 N 138 80 2 20 0.5 0.5 0 0 0.3 21.3
46 2 1/1/2017 0:00 1/1/2017 0:04 5 0.6 1 N 238 238 2 5 0.5 0.5 0 0 0.3 6.3
47 2 1/1/2017 0:00 1/1/2017 0:03 3 0.56 1 N 74 75 2 4.5 0.5 0.5 0 0 0.3 5.8
48 1 1/1/2017 0:00 1/1/2017 0:06 1 0.9 1 N 113 249 1 6 0.5 0.5 2.15 0 0.3 9.45
49 2 1/1/2017 0:00 1/1/2017 0:35 1 2.88 1 N 163 113 1 21.5 0.5 0.5 4.56 0 0.3 27.36
50 1 1/1/2017 0:00 1/1/2017 0:24 2 3.9 1 N 129 179 2 19.5 0.5 0.5 0 0 0.3 20.8
51 2 1/1/2017 0:00 1/1/2017 0:09 1 1.7 1 N 41 75 1 8.5 0.5 0.5 2.45 0 0.3 12.25
52 1 1/1/2017 0:00 1/1/2017 0:06 3 0.7 1 N 229 161 1 6 0.5 0.5 1.45 0 0.3 8.75
53 1 1/1/2017 0:00 1/1/2017 0:03 4 0.9 1 N 236 236 2 5 0.5 0.5 0 0 0.3 6.3
54 2 1/1/2017 0:00 1/1/2017 0:06 1 0.79 1 N 163 237 1 6 0.5 0.5 1.82 0 0.3 9.12
55 2 1/1/2017 0:00 1/1/2017 0:08 1 1.25 1 N 234 162 2 7.5 0.5 0.5 0 0 0.3 8.8
56 2 1/1/2017 0:00 1/1/2017 0:08 5 2.05 1 N 113 233 1 8.5 0.5 0.5 1 0 0.3 10.8
57 2 1/1/2017 0:00 1/1/2017 0:12 1 2.56 1 N 113 229 1 10.5 0.5 0.5 8.2 0 0.3 20
58 2 1/1/2017 0:00 1/1/2017 0:11 1 0.1 1 N 264 264 2 8 0.5 0.5 0 0 0.3 9.3
59 2 1/1/2017 0:00 1/1/2017 0:08 1 1.78 1 N 113 231 1 8 0.5 0.5 1.86 0 0.3 11.16
60 1 1/1/2017 0:00 1/1/2017 0:31 2 10.5 1 N 138 230 2 35.5 0.5 0.5 0 5.54 0.3 42.34
61 1 1/1/2017 0:00 1/1/2017 0:05 1 0.9 1 N 148 4 1 6 0.5 0.5 1.45 0 0.3 8.75
62 2 1/1/2017 0:00 1/1/2017 0:09 1 1.97 1 N 74 168 2 9 0.5 0.5 0 0 0.3 10.3
63 2 1/1/2017 0:00 1/1/2017 0:42 6 20.99 2 N 132 249 2 52 0 0.5 0 0 0.3 52.8
64 1 1/1/2017 0:00 1/1/2017 0:02 1 0 1 N 249 249 3 3 0.5 0.5 0 0 0.3 4.3
65 2 1/1/2017 0:00 1/1/2017 0:06 2 0.66 1 N 79 79 1 5.5 0.5 0.5 1.36 0 0.3 8.16
66 1 1/1/2017 0:00 1/1/2017 0:12 1 3.6 1 N 263 7 2 13 0.5 0.5 0 0 0.3 14.3
67 2 1/1/2017 0:00 1/1/2017 0:12 6 4.02 1 N 41 235 1 14 0.5 0.5 3.06 0 0.3 18.36
68 2 1/1/2017 0:00 1/1/2017 0:07 2 2.66 1 N 148 233 1 10 0.5 0.5 2.26 0 0.3 13.56
69 1 1/1/2017 0:00 1/1/2017 0:19 1 11.4 1 N 138 181 2 31.5 0.5 0.5 0 0 0.3 32.8
70 2 1/1/2017 0:00 1/1/2017 0:15 1 6.59 1 N 143 42 2 20 0.5 0.5 0 0 0.3 21.3
71 2 1/1/2017 0:00 1/1/2017 0:20 1 2.82 1 N 144 170 2 14 0.5 0.5 0 0 0.3 15.3
72 2 1/1/2017 0:00 1/1/2017 0:07 1 1.09 1 N 129 129 2 7 0.5 0.5 0 0 0.3 8.3
73 2 1/1/2017 0:00 1/1/2017 0:01 2 0.41 1 N 239 238 1 3.5 0.5 0.5 0.72 0 0.3 5.52
74 1 1/1/2017 0:00 1/1/2017 0:13 2 2.1 1 Y 45 137 3 10.5 0.5 0.5 0 0 0.3 11.8
75 1 1/1/2017 0:00 1/1/2017 0:03 1 2.2 1 N 137 137 2 3.5 0.5 0.5 0 0 0.3 4.8
76 1 1/1/2017 0:00 1/1/2017 0:43 1 20.3 1 N 132 61 1 58 0.5 0.5 5 0 0.3 64.3
77 1 1/1/2017 0:00 1/1/2017 0:12 1 2.8 1 N 239 229 1 11.5 0.5 0.5 1 0 0.3 13.8
78 2 1/1/2017 0:00 1/1/2017 0:08 3 1.96 1 N 249 88 2 8.5 0.5 0.5 0 0 0.3 9.8
79 1 1/1/2017 0:00 1/1/2017 0:01 1 0 1 N 138 138 3 2.5 0.5 0.5 0 0 0.3 3.8
80 1 1/1/2017 0:00 1/1/2017 0:11 1 4.8 1 N 162 45 1 15.5 0.5 0.5 3.35 0 0.3 20.15
81 2 1/1/2017 0:00 1/1/2017 0:13 2 4.69 1 N 262 90 2 16 0.5 0.5 0 0 0.3 17.3
82 2 1/1/2017 0:00 1/1/2017 0:05 5 0.6 1 N 170 233 1 4.5 0.5 0.5 1.16 0 0.3 6.96
83 2 1/1/2017 0:00 1/1/2017 0:04 5 0.93 1 N 239 142 1 4.5 0.5 0.5 15 0 0.3 20.8
84 2 1/1/2017 0:00 1/1/2017 0:14 2 8.6 1 N 148 7 2 25 0.5 0.5 0 0 0.3 26.3
85 2 1/1/2017 0:00 1/1/2017 0:07 5 1.17 1 N 158 90 1 6.5 0.5 0.5 2.34 0 0.3 10.14
86 2 1/1/2017 0:00 1/1/2017 0:14 2 4.41 1 N 233 74 1 14.5 0.5 0.5 2 0 0.3 17.8
87 1 1/1/2017 0:00 1/1/2017 0:05 1 0.9 1 N 106 106 2 6 0.5 0.5 0 0 0.3 7.3
88 1 1/1/2017 0:00 1/1/2017 0:17 2 3 1 N 231 107 1 14 0.5 0.5 2 0 0.3 17.3
89 1 1/1/2017 0:00 1/1/2017 0:10 1 1.2 1 N 113 137 2 8 0.5 0.5 0 0 0.3 9.3
90 1 1/1/2017 0:00 1/1/2017 0:14 1 1.2 1 N 246 186 1 9.5 0.5 0.5 3.2 0 0.3 14
91 1 1/1/2017 0:00 1/1/2017 0:21 1 5.9 1 N 162 129 1 20.5 0.5 0.5 4.35 0 0.3 26.15
92 1 1/1/2017 0:00 1/1/2017 0:05 2 0.5 1 N 264 264 1 5 0.5 0.5 1.25 0 0.3 7.55
93 2 1/1/2017 0:00 1/1/2017 0:23 1 6.58 1 N 186 152 1 23.5 0.5 0.5 7.44 0 0.3 32.24
94 1 1/1/2017 0:01 1/1/2017 0:17 1 3.7 1 N 50 79 1 14 0.5 0.5 3.05 0 0.3 18.35
95 2 1/1/2017 0:01 1/1/2017 0:23 1 5.91 1 N 125 37 1 21 0.5 0.5 4.46 0 0.3 26.76
96 1 1/1/2017 0:01 1/1/2017 0:09 1 1.2 1 N 152 42 2 7.5 0.5 0.5 0 0 0.3 8.8
97 1 1/1/2017 0:01 1/1/2017 0:05 2 1.2 1 N 137 162 1 6 0.5 0.5 1 0 0.3 8.3
98 1 1/1/2017 0:01 1/1/2017 0:16 2 7.4 1 N 132 130 2 22.5 0.5 0.5 0 0 0.3 23.8
99 2 1/1/2017 0:01 1/1/2017 0:08 6 0.87 1 N 107 107 1 6 0.5 0.5 5 0 0.3 12.3

17
prompt.py
Unescape View File

@ -6,7 +6,10 @@ import subprocess
import sys
if sys.platform != 'win32':
import readline
basecmd = ['bash', '-c', 'k']
else:
basecmd = ['bash.exe', '-c', './k']
test_parser = True
# code to test parser
@ -18,11 +21,14 @@ res = parser.parse(q)
print(res)
keep = False
cxt = None
while test_parser:
try:
q = input()
if q == 'exec':
cxt = engine.initialize()
if not keep or cxt is None:
cxt = engine.initialize()
stmts_stmts = stmts['stmts']
if type(stmts_stmts) is list:
for s in stmts_stmts:
@ -32,14 +38,17 @@ while test_parser:
print(cxt.k9code)
with open('out.k', 'wb') as outfile:
outfile.write((cxt.k9code+'\n\\\\').encode('utf-8'))
subprocess.call(['bash.exe', '-c',"./k out.k"])
subprocess.call(basecmd[:-1] + [basecmd[-1] + ' out.k'])
continue
elif q == 'k':
subprocess.call(['bash.exe', '-c',"./k"])
subprocess.call(basecmd)
continue
elif q == 'print':
print(stmts)
continue
elif q == 'keep':
keep = not keep
continue
trimed = ws.sub(' ', q.lower()).split(' ')
if trimed[0].startswith('f'):
fn = 'stock.a' if len(trimed) <= 1 or len(trimed[1]) == 0 \

11
test.csv
Unescape View File

@ -0,0 +1,11 @@
a, b, c, d
1,1,2,2
1,2,2,2
1,2,3,4
4,2,1,4
2,1,3,4
1,2,3,4
1,2,3,3
3,2,1,2
2,1,2,2
1,2,3,1
1 a b c d
2 1 1 2 2
3 1 2 2 2
4 1 2 3 4
5 4 2 1 4
6 2 1 3 4
7 1 2 3 4
8 1 2 3 3
9 3 2 1 2
10 2 1 2 2
11 1 2 3 1
Write Preview
Loading…
Cancel
Save