diff --git a/engine/ast.py b/engine/ast.py index 85cdd24..7c02e2e 100644 --- a/engine/ast.py +++ b/engine/ast.py @@ -110,7 +110,10 @@ class TableInfo: self.cxt.ccols_byname[cname] = col_object self.columns_byname[c['name']] = col_object self.columns.append(col_object) - + def get_size(self): + size_tmp = 'tmp_sz_'+base62uuid(6) + self.cxt.emit(f'const auto& {size_tmp} = {self.columns[0].reference()}.size;') + return size_tmp @property def n_cols(self): return len(self.columns) diff --git a/engine/ddl.py b/engine/ddl.py index b3e9f3a..130d54f 100644 --- a/engine/ddl.py +++ b/engine/ddl.py @@ -1,6 +1,7 @@ # code-gen for data decl languages from engine.ast import ColRef, TableInfo, ast_node, Context, include +from engine.scan import scan from engine.utils import base62uuid class create_table(ast_node): @@ -26,9 +27,15 @@ class create_table(ast_node): for c in tbl.columns: self.emit(f"{c.cxt_name}.init();") else: - for i, c in enumerate(tbl.columns): - self.emit(f"{c.cxt_name}.init();") - self.emit(f"{c.cxt_name} = {self.cexpr[i]()};") + if len(self.context.scans) == 0: + for i, c in enumerate(tbl.columns): + self.emit(f"{c.cxt_name}.init();") + self.emit(f"{c.cxt_name} = {self.cexpr[i]()};") + else: + scanner:scan = self.context.scans[-1] + for i, c in enumerate(tbl.columns): + scanner.add(f"{c.cxt_name}.init();", "init") + scanner.add(f"{c.cxt_name} = {self.cexpr[i](scanner.it_ver)};") class insert(ast_node): name = 'insert' diff --git a/engine/expr.py b/engine/expr.py index ae9769f..0c8d380 100644 --- a/engine/expr.py +++ b/engine/expr.py @@ -23,8 +23,8 @@ class expr(ast_node): 'mul':'*', 'div':'/', 'mod':'%', - 'and':'&', - 'or':'|', + 'and':'&&', + 'or':'||', 'xor' : '^', 'gt':'>', 'lt':'<', @@ -92,7 +92,7 @@ class expr(ast_node): x.append(expr(self, v)._expr) self._expr = self.compound_ops[key][1](x) elif key in self.unary_ops: - self._expr += f'({expr(self, val)._expr}{self.unary_ops[key]})' + self._expr += f'{self.unary_ops[key]}({expr(self, val)._expr})' else: print(f'Undefined expr: {key}{val}') diff --git a/engine/groupby.py b/engine/groupby.py index 62a4e69..d03aef5 100644 --- a/engine/groupby.py +++ b/engine/groupby.py @@ -40,7 +40,7 @@ class groupby(ast_node): self.emit(f'unordered_map<{self.group_type}, vector_type, ' f'transTypes<{self.group_type}, hasher>> {self.group};') self.n_grps = len(node) - self.scanner = scan(self, None, expr.toCExpr(first_col)()+'.size') + self.scanner = scan(self, self.datasource, expr.toCExpr(first_col)()+'.size') self.scanner.add(f'{self.group}[forward_as_tuple({g_contents(self.scanner.it_ver)})].emplace_back({self.scanner.it_ver});') diff --git a/engine/projection.py b/engine/projection.py index 088bde0..644deb4 100644 --- a/engine/projection.py +++ b/engine/projection.py @@ -14,6 +14,7 @@ class projection(ast_node): self.disp = disp self.outname = outname self.group_node = None + self.where = None ast_node.__init__(self, parent, node, context) def init(self, _): if self.outname is None: @@ -58,8 +59,9 @@ class projection(ast_node): self.prev_datasource = self.context.datasource self.context.datasource = self.datasource if 'where' in node: - self.datasource = filter(self, node['where'], True).output - self.context.datasource = self.datasource + self.where = filter(self, node['where'], True) + # self.datasource = filter(self, node['where'], True).output + #self.context.datasource = self.datasource if 'groupby' in node: self.group_node = groupby(self, node['groupby']) @@ -107,7 +109,10 @@ class projection(ast_node): else: create_table(self, self.out_table, cexpr = cexprs) self.datasource.group_node = None - + + if self.where is not None: + self.where.finalize() + has_orderby = 'orderby' in node if has_orderby: diff --git a/engine/scan.py b/engine/scan.py index 4766d7a..fdbca2b 100644 --- a/engine/scan.py +++ b/engine/scan.py @@ -11,10 +11,12 @@ class scan(ast_node): super().__init__(parent, node, context) def init(self, _): self.datasource = self.context.datasource + self.initializers = '' self.start = '' self.body = '' self.end = '}' - self.filter = None + self.mode = None + self.filters = [] scan_vars = set(s.it_var for s in self.context.scans) self.it_ver = 'i' + base62uuid(2) while(self.it_ver in scan_vars): @@ -22,19 +24,30 @@ class scan(ast_node): self.parent.context.scans.append(self) def produce(self, node): if type(node) is ColRef: + self.colref = node if self.size is None: + self.mode = ["col", node.table] self.start += f'for (auto& {self.it_ver} : {node.reference()}) {{\n' else: + self.mode = ["idx", node.table] self.start += f"for (uint32_t {self.it_ver} = 0; {self.it_ver} < {node.reference()}.size; ++{self.it_ver}){{\\n" elif type(node) is str: + self.mode = ["idx", None] self.start+= f'for(auto& {self.it_ver} : {node}) {{\n' else: + self.mode = ["idx", node] # Node is the TableInfo self.start += f"for (uint32_t {self.it_ver} = 0; {self.it_ver} < {self.size}; ++{self.it_ver}){{\n" - def add(self, stmt): - self.body+=stmt + '\n' + def add(self, stmt, position = "body"): + if position == "body": + self.body += stmt + '\n' + else: + self.initializers += stmt + '\n' def finalize(self): - self.context.remove_scan(self, self.start + self.body + self.end) + for f in self.filters: + self.start += f + self.end += '}' + self.context.remove_scan(self, self.initializers + self.start + self.body + self.end) class filter(ast_node): name = 'filter' @@ -45,7 +58,7 @@ class filter(ast_node): self.datasource = self.context.datasource self.view = View(self.context, self.datasource) self.value = None - + def spawn(self, node): # TODO: deal with subqueries self.modified_node = node @@ -64,9 +77,18 @@ class filter(ast_node): self.emit(f'{tmpVar}:{self.value}') for o, c in zip(self.output.columns, self.datasource.columns): self.emit(f'{o.cname}:$[{tmpVar};{c.cname};()]') - - def consume(self, node): + + def finalize(self): + self.scanner.finalize() + def consume(self, _): # TODO: optimizations after converting expr to cnf + self.scanner = None + for s in self.context.scans: + if self.datasource == s.mode[1]: + self.scanner = s + break + if self.scanner is None: + self.scanner = scan(self, self.datasource, self.datasource.get_size()) self.expr = expr(self, self.modified_node) - print(node) + self.scanner.filters.append(f'if ({self.expr.cexpr(self.scanner.it_ver)}) {{\n') \ No newline at end of file diff --git a/out.cpp b/out.cpp index 8e4f407..6867019 100644 --- a/out.cpp +++ b/out.cpp @@ -1,60 +1,56 @@ -#include "./server/aggregations.h" +#include "csv.h" +#include #include "./server/libaquery.h" +#include "./server/hasher.h" +#include "./server/aggregations.h" extern "C" int __DLLEXPORT__ dllmain(Context* cxt) { using namespace std; using namespace types; - auto stocks = new TableInfo("stocks", 2); -cxt->tables.insert({"stocks", stocks}); -auto& stocks_timestamp = *(ColRef *)(&stocks->colrefs[0]); -auto& stocks_price = *(ColRef *)(&stocks->colrefs[1]); -stocks_timestamp.init(); -stocks_price.init(); -stocks_timestamp.emplace_back(1); -stocks_price.emplace_back(15); -stocks_timestamp.emplace_back(2); -stocks_price.emplace_back(19); -stocks_timestamp.emplace_back(3); -stocks_price.emplace_back(16); -stocks_timestamp.emplace_back(4); -stocks_price.emplace_back(17); -stocks_timestamp.emplace_back(5); -stocks_price.emplace_back(15); -stocks_timestamp.emplace_back(6); -stocks_price.emplace_back(13); -stocks_timestamp.emplace_back(7); -stocks_price.emplace_back(5); -stocks_timestamp.emplace_back(8); -stocks_price.emplace_back(8); -stocks_timestamp.emplace_back(9); -stocks_price.emplace_back(7); -stocks_timestamp.emplace_back(10); -stocks_price.emplace_back(13); -stocks_timestamp.emplace_back(11); -stocks_price.emplace_back(11); -stocks_timestamp.emplace_back(12); -stocks_price.emplace_back(14); -stocks_timestamp.emplace_back(13); -stocks_price.emplace_back(10); -stocks_timestamp.emplace_back(14); -stocks_price.emplace_back(5); -stocks_timestamp.emplace_back(15); -stocks_price.emplace_back(2); -stocks_timestamp.emplace_back(16); -stocks_price.emplace_back(5); -auto out_ZPYh = new TableInfo>("out_ZPYh", 1); -cxt->tables.insert({"out_ZPYh", out_ZPYh}); -auto& out_ZPYh_maxstockspriceminstockstimestamp = *(ColRef> *)(&out_ZPYh->colrefs[0]); -out_ZPYh_maxstockspriceminstockstimestamp.init(); -out_ZPYh_maxstockspriceminstockstimestamp = max((stocks_price-min(stocks_timestamp))); -print(*out_ZPYh); + auto test = new TableInfo("test", 4); +cxt->tables.insert({"test", test}); +auto& test_a = *(ColRef *)(&test->colrefs[0]); +auto& test_b = *(ColRef *)(&test->colrefs[1]); +auto& test_c = *(ColRef *)(&test->colrefs[2]); +auto& test_d = *(ColRef *)(&test->colrefs[3]); +test_a.init(); +test_b.init(); +test_c.init(); +test_d.init(); +io::CSVReader<4> csv_reader_6qlGpe("test.csv"); +csv_reader_6qlGpe.read_header(io::ignore_extra_column, "a","b","c","d"); +int tmp_39gHMkie; +int tmp_190h2sZs; +int tmp_4a8dDzSN; +int tmp_3LAKxSmM; +while(csv_reader_6qlGpe.read_row(tmp_39gHMkie,tmp_190h2sZs,tmp_4a8dDzSN,tmp_3LAKxSmM)) { -auto out_1ac3 = new TableInfo>("out_1ac3", 1); -cxt->tables.insert({"out_1ac3", out_1ac3}); -auto& out_1ac3_maxstockspriceminsstocksprice = *(ColRef> *)(&out_1ac3->colrefs[0]); -out_1ac3_maxstockspriceminsstocksprice.init(); -out_1ac3_maxstockspriceminsstocksprice = max((stocks_price-mins(stocks_price))); -print(*out_1ac3); +test_a.emplace_back(tmp_39gHMkie); +test_b.emplace_back(tmp_190h2sZs); +test_c.emplace_back(tmp_4a8dDzSN); +test_d.emplace_back(tmp_3LAKxSmM); +} +typedef record record_type2Te4GFo; +unordered_map, transTypes> g79JNXM8; +for (uint32_t i5x = 0; i5x < test_a.size; ++i5x){ +g79JNXM8[forward_as_tuple(test_a[i5x],test_b[i5x],test_d[i5x])].emplace_back(i5x); +} +auto out_5NL7 = new TableInfo,decays,decays>("out_5NL7", 3); +cxt->tables.insert({"out_5NL7", out_5NL7}); +auto& out_5NL7_sumtestc = *(ColRef> *)(&out_5NL7->colrefs[0]); +auto& out_5NL7_get1None = *(ColRef> *)(&out_5NL7->colrefs[1]); +auto& out_5NL7_get2None = *(ColRef> *)(&out_5NL7->colrefs[2]); +out_5NL7_sumtestc.init(); +out_5NL7_get1None.init(); +out_5NL7_get2None.init(); +for(auto& i4l : g79JNXM8) { +auto &key_ADPihOU = i4l.first; +auto &val_7LsrkDP = i4l.second; +out_5NL7_sumtestc.emplace_back(sum(test_c[val_7LsrkDP])); +out_5NL7_get1None.emplace_back(get<1>(key_ADPihOU)); +out_5NL7_get2None.emplace_back(get<2>(key_ADPihOU)); +} +print(*out_5NL7); return 0; } \ No newline at end of file diff --git a/prompt.py b/prompt.py index 9333997..94fbd62 100644 --- a/prompt.py +++ b/prompt.py @@ -1,6 +1,8 @@ from concurrent.futures import thread import re import time + +from mo_parsing import ParseException import aquery_parser as parser import engine import subprocess @@ -137,7 +139,7 @@ while test_parser: continue stmts = parser.parse(q) print(stmts) - except (ValueError, FileNotFoundError) as e: + except (ValueError, FileNotFoundError, ParseException) as e: rm() print(type(e), e) diff --git a/server/server.sln b/server/server.sln index e3b37e3..598688f 100644 --- a/server/server.sln +++ b/server/server.sln @@ -8,7 +8,7 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "server", "server.vcxproj", {8081FDAA-4D13-4B7A-ADB2-8224AF7F1C81} = {8081FDAA-4D13-4B7A-ADB2-8224AF7F1C81} EndProjectSection EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "Project1", "..\Project1\Project1.vcxproj", "{8081FDAA-4D13-4B7A-ADB2-8224AF7F1C81}" +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "msc-plugin", "..\msc-plugin\msc-plugin.vcxproj", "{8081FDAA-4D13-4B7A-ADB2-8224AF7F1C81}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution diff --git a/server/table.h b/server/table.h index 44a8448..abdeb06 100644 --- a/server/table.h +++ b/server/table.h @@ -103,6 +103,39 @@ struct TableInfo { template using getRecordType = typename GetTypes::type; TableInfo(const char* name, uint32_t n_cols); + //template + //struct Iterator_t { + // uint32_t val; + // const TableInfo* info; + // constexpr Iterator_t(const uint32_t* val, const TableInfo* info) noexcept : val(val), info(info) {} + // getRecordType operator*() { + // return getRecordType(info->colrefs[Idxs].operator[](*val)...); + // } + // bool operator != (const Iterator_t& rhs) { return rhs.val != val; } + // Iterator_t& operator++ () { + // ++val; + // return *this; + // } + // Iterator_t operator++ (int) { + // Iterator_t tmp = *this; + // ++val; + // return tmp; + // } + //}; + //template + //Iterator_t begin() const { + // + //} + //template + //Iterator_t end() const { + // + //} + // + //template + //order_by() { + // vector_type order(colrefs[0].size); + // std::sort(this->begin) + //} }; template diff --git a/stock.a b/stock.a index 8c194bf..f322626 100644 --- a/stock.a +++ b/stock.a @@ -22,9 +22,9 @@ SELECT max(price-min(timestamp)) FROM stocks /* "q1" */ SELECT max(price-mins(price)) FROM stocks +SELECT price, timestamp FROM stocks where price - timestamp > 1 and not (price*timestamp<100) + /* - "q2" -SELECT price, timestamp FROM stocks where price -timestamp > 1 and not (price*timestamp<100); "q3" SELECT max(price-mins(price)) FROM stocks