diff --git a/Makefile.old b/Makefile.old deleted file mode 100644 index fef8c50..0000000 --- a/Makefile.old +++ /dev/null @@ -1,8 +0,0 @@ -all: - g++ mmw.cpp --std=c++1z -shared -fPIC -Ofast -march=native -g0 -s -o mmw.so -avx512: - g++ mmw.cpp --std=c++1z -shared -fPIC -Ofast -mavx512f -g0 -s -o mmw.so -debug: - g++ mmw.cpp --std=c++1z -shared -fPIC -O0 -march=native -g3 -o mmw.so -clean: - rm mmw.so -rf diff --git a/README.md b/README.md index f1c753b..e6cec8f 100644 --- a/README.md +++ b/README.md @@ -1,20 +1,44 @@ -# AQuery++ +# AQuery++ DB +## Introduction -AQuery++ Compiler that compiles AQuery into C++11. -Frontend built on top of [mo-sql-parsing](https://github.com/klahnakoski/mo-sql-parsing). +AQuery++ Database is a cross-platform, In-Memory Column-Store Database that incorporates compiled query execution. +Compiler frontend built on top of [mo-sql-parsing](https://github.com/klahnakoski/mo-sql-parsing). ## Roadmap - [x] SQL Parser -> AQuery Parser (Front End) - [ ] AQuery-C++ Compiler (Back End) - [x] Schema and Data Model - [x] Data acquisition/output from/to csv file - - [ ] Single table queries + - [x] Single table queries - [x] Projections and Single Table Aggregations - [x] Group by Aggregations - [x] Filters - - [ ] Order by - - [ ] Assumption + - [x] Order by + - [x] Assumption + - [x] Flatten - [ ] Multi-table - [ ] Join - [ ] Subqueries - [ ] -> Optimizing Compiler + +## TODO: +- [ ] C++ Meta-Programming: Elimilate template recursions as much as possible. +- [ ] IPC: Better ways to communicate between Interpreter (Python) and Executer (C++). + - [ ] Sockets? stdin/stdout capture? + +## Requirements +Recent version of Linux, Windows or MacOS, with recent C++ compiler that has C++17 (1z) support (e.g. gcc 6.0, MSVC 2017, clang 6.0), and python 3.6 or above. + +## Usage +`python3 prompt.py` will launch the interactive command prompt. The server binary will be autometically rebuilt and started. +#### Commands: +- ``: parse sql statement +- `f `: parse all sql statements in file +- `print`: printout parsed sql statements +- `exec`: execute last parsed statement(s) +- `r`: run the last generated code snippet +- `save `: save current code snippet. will use random filename if not specified. +- `exit`: quit the prompt +#### Example: + `f moving_avg.a`
+ `exec` \ No newline at end of file diff --git a/csv.dylib b/csv.dylib deleted file mode 100644 index 3dc33e4..0000000 Binary files a/csv.dylib and /dev/null differ diff --git a/engine/ast.py b/engine/ast.py index 85cdd24..651fa26 100644 --- a/engine/ast.py +++ b/engine/ast.py @@ -15,6 +15,9 @@ class ColRef: self.order_pending = None # order_pending self.compound = compound # compound field (list as a field) self.views = [] + self.aux_columns = [] # columns for temperary calculations + # e.g. order by, group by, filter by expressions + self.__arr__ = (cname, _ty, cobj, cnt, table, name, id) def reference(self): @@ -89,7 +92,8 @@ class TableInfo: type_tags = type_tags[:-1] type_tags += '>' - self.cxt.emit(f'auto& {base_name} = *(TableInfo{type_tags} *)(cxt->tables[{self.table_name}]);') + self.cxt.emit(f'auto& {base_name} = *(TableInfo{type_tags} *)(cxt->tables["{self.table_name}"]);') + return self.cxt_name def refer_all(self): self.reference() for c in self.columns: @@ -110,7 +114,10 @@ class TableInfo: self.cxt.ccols_byname[cname] = col_object self.columns_byname[c['name']] = col_object self.columns.append(col_object) - + def get_size(self): + size_tmp = 'tmp_sz_'+base62uuid(6) + self.cxt.emit(f'const auto& {size_tmp} = {self.columns[0].reference()}.size;') + return size_tmp @property def n_cols(self): return len(self.columns) @@ -136,8 +143,8 @@ class TableInfo: def get_col_d(self, col_name): col = self.columns_byname[col_name] - if type(self.rec) is list: - self.rec.append(col) + if type(self.rec) is set: + self.rec.add(col) return col def get_ccolname_d(self, col_name): @@ -160,12 +167,12 @@ class TableInfo: self.alias.add(alias) def parse_tablenames(self, colExpr, materialize = True, raw = False): - self.get_col = self.get_col if materialize else self.get_col_d + # get_col = self.get_col if materialize else self.get_col_d parsedColExpr = colExpr.split('.') ret = None if len(parsedColExpr) <= 1: - ret = self.get_col(colExpr) + ret = self.get_col_d(colExpr) else: datasource = self.cxt.tables_byname[parsedColExpr[0]] if datasource is None: @@ -177,6 +184,7 @@ class TableInfo: if self.groupinfo is not None and ret and ret in self.groupinfo.raw_groups: string = f'get<{self.groupinfo.raw_groups.index(ret)}>({{y}})' return string, ret if raw else string + class View: def __init__(self, context, table = None, tmp = True): self.table: TableInfo = table @@ -193,6 +201,7 @@ class Context: extern "C" int __DLLEXPORT__ dllmain(Context* cxt) { using namespace std; using namespace types; + ''' def __init__(self): self.tables:List[TableInfo] = [] diff --git a/engine/ddl.py b/engine/ddl.py index b3e9f3a..60be2be 100644 --- a/engine/ddl.py +++ b/engine/ddl.py @@ -1,14 +1,17 @@ # code-gen for data decl languages +from engine.orderby import orderby from engine.ast import ColRef, TableInfo, ast_node, Context, include +from engine.scan import scan from engine.utils import base62uuid class create_table(ast_node): name = 'create_table' - def __init__(self, parent: "ast_node", node, context: Context = None, cexpr = None): - self.cexpr = cexpr + def __init__(self, parent: "ast_node", node, context: Context = None, cexprs = None, lineage = False): + self.cexprs = cexprs + self.lineage = lineage super().__init__(parent, node, context) - def produce(self, node): + def produce(self, node): if type(node) is not TableInfo: ct = node[self.name] tbl = self.context.add_table(ct['name'], ct['columns']) @@ -22,14 +25,38 @@ class create_table(ast_node): self.context.tables_in_context[tbl] = tbl.table_name tbl.cxt_name = tbl.table_name tbl.refer_all() - if self.cexpr is None: + # create an empty new table + if self.cexprs is None: for c in tbl.columns: self.emit(f"{c.cxt_name}.init();") + # create an output table else: - for i, c in enumerate(tbl.columns): - self.emit(f"{c.cxt_name}.init();") - self.emit(f"{c.cxt_name} = {self.cexpr[i]()};") - + # 1 to 1 lineage. + if len(self.context.scans) == 0: + if self.lineage: + order = 'order_' + base62uuid(6) + self.emit(f'auto {order} = {self.parent.datasource.cxt_name}->order_by<{orderby(self.parent, self.parent.assumptions).result()}>();') + self.lineage = '*' + order + else: + self.lineage = None + for i, c in enumerate(tbl.columns): + self.emit(f"{c.cxt_name}.init();") + self.emit(f"{c.cxt_name} = {self.cexprs[i](self.lineage)};") + self.lineage = None + self.parent.assumptions = None + else: + scanner:scan = self.context.scans[-1] + if self.lineage: + lineage_var = 'lineage_' + base62uuid(6) + counter_var = 'counter_' + base62uuid(6) + scanner.add(f'auto {lineage_var} = {self.datasource.cxt_name}->bind({tbl.cxt_name});', "init") + scanner.add(f'auto {counter_var} = 0;', "init") + scanner.add(f"{lineage_var}.emplace_back({counter_var}++);", "front") + self.lineage = f"{lineage_var}.rid" + for i, c in enumerate(tbl.columns): + scanner.add(f"{c.cxt_name}.init();", "init") + scanner.add(f"{c.cxt_name} = {self.cexprs[i](scanner.it_ver)};") + class insert(ast_node): name = 'insert' def produce(self, node): @@ -81,32 +108,21 @@ class outfile(ast_node): def produce(self, node): out_table:TableInfo = self.parent.out_table filename = node['loc']['literal'] if 'loc' in node else node['literal'] - self.emit_no_ln(f"\"{filename}\"1:`csv@(+(") - l_compound = False - l_cols = '' - l_keys = '' - ending = lambda x: x[:-1] if len(x) > 0 and x[-1]==';' else x - for i, c in enumerate(out_table.columns): - c:ColRef - l_keys += '`' + c.name - if c.compound: - if l_compound: - l_cols=f'flatBOTH\'+(({ending(l_cols)});{c.cname})' - else: - l_compound = True - if i >= 1: - l_cols = f'flatRO\'+(({ending(l_cols)});{c.cname})' - else: - l_cols = c.cname + ';' - elif l_compound: - l_cols = f'flatLO\'+(({ending(l_cols)});{c.cname})' - else: - l_cols += f"{c.cname};" - if not l_compound: - self.emit_no_ln(l_keys + '!(' + ending(l_cols) + ')') - else: - self.emit_no_ln(f'{l_keys}!+,/({ending(l_cols)})') - self.emit('))') - + sep = ',' if 'term' not in node else node['term']['literal'] + file_pointer = 'fp_' + base62uuid(6) + self.emit(f'FILE* {file_pointer} = fopen("{filename}", "w");') + self.emit(f'{out_table.cxt_name}->printall("{sep}", "\\n", nullptr, {file_pointer});') + self.emit(f'fclose({file_pointer});') + # self.context.headers.add('fstream') + # cout_backup_buffer = 'stdout_' + base62uuid(4) + # ofstream = 'ofstream_' + base62uuid(6) + # self.emit(f'auto {cout_backup_buffer} = cout.rdbuf();') + # self.emit(f'auto {ofstream} = ofstream("{filename}");') + # self.emit(f'cout.rdbuf({ofstream}.rdbuf());') + # TODO: ADD STMTS. + # self.emit(f'cout.rdbuf({cout_backup_buffer});') + # self.emit(f'{ofstream}.close();') + + import sys include(sys.modules[__name__]) \ No newline at end of file diff --git a/engine/expr.py b/engine/expr.py index ae9769f..b387d79 100644 --- a/engine/expr.py +++ b/engine/expr.py @@ -11,10 +11,10 @@ class expr(ast_node): 'avg': 'avg', 'sum': 'sum', 'count' : 'count', - 'mins': ['mins', 'minsw'], - 'maxs': ['maxs', 'maxsw'], - 'avgs': ['avgs', 'avgsw'], - 'sums': ['sums', 'sumsw'], + 'mins': ['mins', 'minw'], + 'maxs': ['maxs', 'maxw'], + 'avgs': ['avgs', 'avgw'], + 'sums': ['sums', 'sumw'], } binary_ops = { @@ -23,8 +23,8 @@ class expr(ast_node): 'mul':'*', 'div':'/', 'mod':'%', - 'and':'&', - 'or':'|', + 'and':'&&', + 'or':'||', 'xor' : '^', 'gt':'>', 'lt':'<', @@ -40,13 +40,16 @@ class expr(ast_node): 'not' : '!' } - coumpound_generating_ops = ['mod', 'mins', 'maxs', 'sums'] + \ + coumpound_generating_ops = ['avgs', 'mins', 'maxs', 'sums'] + \ list( binary_ops.keys()) + list(compound_ops.keys()) + list(unary_ops.keys() ) def __init__(self, parent, node, materialize_cols = True, abs_col = False): self.materialize_cols = materialize_cols self.raw_col = None self.__abs = abs_col + self.inside_agg = False + if(type(parent) is expr): + self.inside_agg = parent.inside_agg ast_node.__init__(self, parent, node, None) def init(self, _): @@ -67,7 +70,8 @@ class expr(ast_node): if type(node) is dict: for key, val in node.items(): if key in self.func_maps: - # if type(val) in [dict, str]: + # TODO: distinguish between UDF agg functions and other UDF functions. + self.inside_agg = True self.context.headers.add('"./server/aggregations.h"') if type(val) is list and len(val) > 1: cfunc = self.func_maps[key] @@ -81,6 +85,7 @@ class expr(ast_node): self._expr += f"{funcname}(" self._expr += expr(self, val)._expr self._expr += ')' + self.inside_agg = False elif key in self.binary_ops: l = expr(self, val[0])._expr r = expr(self, val[1])._expr @@ -92,7 +97,7 @@ class expr(ast_node): x.append(expr(self, v)._expr) self._expr = self.compound_ops[key][1](x) elif key in self.unary_ops: - self._expr += f'({expr(self, val)._expr}{self.unary_ops[key]})' + self._expr += f'{self.unary_ops[key]}({expr(self, val)._expr})' else: print(f'Undefined expr: {key}{val}') @@ -112,7 +117,7 @@ class expr(ast_node): self._expr, self.raw_col = self.datasource.parse_tablenames(node, self.materialize_cols, True) self.raw_col = self.raw_col if type(self.raw_col) is ColRef else None if self.__abs and self.raw_col: - self._expr = self.raw_col.reference() + index_expr + self._expr = self.raw_col.reference() + ("" if self.inside_agg else index_expr) elif type(node) is bool: self._expr = '1' if node else '0' else: diff --git a/engine/groupby.py b/engine/groupby.py index 62a4e69..3a23c34 100644 --- a/engine/groupby.py +++ b/engine/groupby.py @@ -1,8 +1,9 @@ -from engine.ast import TableInfo, ast_node +from engine.ast import ColRef, TableInfo, ast_node +from engine.orderby import assumption from engine.scan import scan from engine.utils import base62uuid from engine.expr import expr -import engine.types + class groupby(ast_node): name = '_groupby' def init(self, _): @@ -12,7 +13,7 @@ class groupby(ast_node): self.group_type = 'record_type' + base62uuid(7) self.datasource = self.parent.datasource self.scanner = None - self.datasource.rec = [] + self.datasource.rec = set() self.raw_groups = [] def produce(self, node): @@ -24,7 +25,8 @@ class groupby(ast_node): for i, g in enumerate(node): v = g['value'] e = expr(self, v) - self.raw_groups.append(e.raw_col) + if type(e.raw_col) is ColRef: + self.raw_groups.append(e.raw_col) e = e._expr # if v is compound expr, create tmp cols if type(v) is not str: @@ -40,7 +42,7 @@ class groupby(ast_node): self.emit(f'unordered_map<{self.group_type}, vector_type, ' f'transTypes<{self.group_type}, hasher>> {self.group};') self.n_grps = len(node) - self.scanner = scan(self, None, expr.toCExpr(first_col)()+'.size') + self.scanner = scan(self, self.datasource, expr.toCExpr(first_col)()+'.size') self.scanner.add(f'{self.group}[forward_as_tuple({g_contents(self.scanner.it_ver)})].emplace_back({self.scanner.it_ver});') @@ -48,7 +50,14 @@ class groupby(ast_node): self.referenced = self.datasource.rec self.datasource.rec = None self.scanner.finalize() - + + def deal_with_assumptions(self, assumption:assumption, out:TableInfo): + gscanner = scan(self, self.group) + val_var = 'val_'+base62uuid(7) + gscanner.add(f'auto &{val_var} = {gscanner.it_ver}.second;') + gscanner.add(f'{out.cxt_name}->order_by<{assumption.result()}>(&{val_var});') + gscanner.finalize() + def finalize(self, cexprs, out:TableInfo): gscanner = scan(self, self.group) key_var = 'key_'+base62uuid(7) @@ -58,4 +67,6 @@ class groupby(ast_node): gscanner.add(f'auto &{val_var} = {gscanner.it_ver}.second;') gscanner.add(';\n'.join([f'{out.columns[i].reference()}.emplace_back({ce(x=val_var, y=key_var)})' for i, ce in enumerate(cexprs)])+';') - gscanner.finalize() \ No newline at end of file + gscanner.finalize() + + self.datasource.groupinfo = None \ No newline at end of file diff --git a/engine/orderby.py b/engine/orderby.py index ae410d4..99e4ab5 100644 --- a/engine/orderby.py +++ b/engine/orderby.py @@ -16,43 +16,50 @@ class order_item: return ('' if self.order else '-') + f'({self.name})' def __str__(self): - return self.materialize() + return self.name def __repr__(self): return self.__str__() -class orders: - def __init__(self, node, datasource): - self.order_items = [] - self.materialized = False - self.view = None - self.node = node - self.datasource = datasource - self.n_attrs = -1 - - def materialize(self): - if not self.materialized: - self.view = View(self.node.context, self.datasource, False) - keys = ';'.join([f'{o}' for o in self.order_items]) - self.n_attrs = len(self.order_items) - self.node.emit(f"{self.view.name}: > +`j (({',' if self.n_attrs == 1 else ''}{keys}))") - self.materialized = True - - def append(self, o): - self.order_items.append(o) - class orderby(ast_node): name = '_orderby' - + def __init__(self, parent: "ast_node", node, context: Context = None): + self.col_list = [] + super().__init__(parent, node, context) def init(self, _): self.datasource = self.parent.datasource - self.order = orders(self, self.datasource) + self.order = [] self.view = '' def produce(self, node): if type(node) is not list: node = [node] for n in node: order = not ('sort' in n and n['sort'] == 'desc') - self.order.append(order_item(n['value'], self, order)) + col_id = self.datasource.columns_byname[n['value']].id + col_id = col_id if order else -col_id-1 + if col_id not in self.col_list: + self.col_list.append(col_id) + self.order.append(order_item(n['value'], self, order)) + + def merge(self, node): + self.produce(node) + + def finialize(self, references): + self.order = [ o for o in self.order if o.name in references ] + + def result(self, sep:str = ','): + return sep.join([f"{c}" for c in self.col_list]) - def consume(self, _): - self.datasource.order.append(self.order) \ No newline at end of file +class assumption(orderby): + name = '_assumption' + def __init__(self, parent: "ast_node", node, context: Context = None, exclude = []): + self.exclude = exclude + super().__init__(parent, node, context) + + def produce(self, node): + if type(node) is not list: + node = [node] + [n for n in node if n not in self.exclude] + return super().produce(node) + + def empty(self): + return len(self.col_list) == 0 \ No newline at end of file diff --git a/engine/projection.py b/engine/projection.py index 088bde0..11c613a 100644 --- a/engine/projection.py +++ b/engine/projection.py @@ -2,9 +2,9 @@ from engine.ast import ColRef, TableInfo, ast_node, Context, include from engine.groupby import groupby from engine.join import join from engine.expr import expr -from engine.orderby import orderby +from engine.orderby import assumption, orderby from engine.scan import filter -from engine.utils import base62uuid, enlist, base62alp +from engine.utils import base62uuid, enlist, base62alp, has_other from engine.ddl import create_table, outfile import copy @@ -14,6 +14,8 @@ class projection(ast_node): self.disp = disp self.outname = outname self.group_node = None + self.assumptions = None + self.where = None ast_node.__init__(self, parent, node, context) def init(self, _): if self.outname is None: @@ -44,9 +46,8 @@ class projection(ast_node): elif type(value) is str: self.datasource = self.context.tables_byname[value] if 'assumptions' in from_clause: - for assumption in enlist(from_clause['assumptions']): - orderby(self, assumption) - + self.assumptions = enlist(from_clause['assumptions']) + elif type(from_clause) is str: self.datasource = self.context.tables_byname[from_clause] @@ -58,8 +59,9 @@ class projection(ast_node): self.prev_datasource = self.context.datasource self.context.datasource = self.datasource if 'where' in node: - self.datasource = filter(self, node['where'], True).output - self.context.datasource = self.datasource + self.where = filter(self, node['where'], True) + # self.datasource = filter(self, node['where'], True).output + # self.context.datasource = self.datasource if 'groupby' in node: self.group_node = groupby(self, node['groupby']) @@ -71,62 +73,101 @@ class projection(ast_node): def consume(self, node): self.inv = True disp_varname = 'd'+base62uuid(7) - has_groupby = False - if self.group_node is not None: - # There is group by; - has_groupby = True + has_groupby = self.group_node is not None cexprs = [] flatten = False cols = [] self.out_table = TableInfo('out_'+base62uuid(4), [], self.context) if 'outfile' in node: flatten = True - + + new_names = [] + proj_raw_cols = [] for i, proj in enumerate(self.projections): cname = '' compound = False - self.datasource.rec = [] + self.datasource.rec = set() if type(proj) is dict: if 'value' in proj: e = proj['value'] - sname = expr(self, e)._expr - fname = expr.toCExpr(sname) - absname = expr(self, e, abs_col=True)._expr - compound = True + sname = expr(self, e) + if type(sname.raw_col) is ColRef: + proj_raw_cols.append(sname.raw_col) + sname = sname._expr + fname = expr.toCExpr(sname) # fastest access method at innermost context + absname = expr(self, e, abs_col=True)._expr # absolute name at function scope + # TODO: Make it single pass here. + compound = True # compound column cexprs.append(fname) - cname = ''.join([a if a in base62alp else '' for a in fname()]) - - compound = compound and has_groupby and self.datasource.rec not in self.group_node.referenced + cname = e if type(e) is str else ''.join([a if a in base62alp else '' for a in expr.toCExpr(absname)()]) + if 'name' in proj: # renaming column by AS keyword + cname = proj['name'] + new_names.append(cname) + elif type(proj) is str: + col = self.datasource.get_col_d(proj) + if type(col) is ColRef: + col.reference() + + compound = compound and has_groupby and has_other(self.datasource.rec, self.group_node.referenced) + self.datasource.rec = None + + typename = f'decays' + if not compound: + typename = f'value_type<{typename}>' + + cols.append(ColRef(cname, expr.toCExpr(typename)(), self.out_table, 0, None, cname, i, compound=compound)) - cols.append(ColRef(cname, expr.toCExpr(f'decays')(0), self.out_table, 0, None, cname, i, compound=compound)) self.out_table.add_cols(cols, False) + lineage = None + if has_groupby: - create_table(self, self.out_table) + create_table(self, self.out_table) # creates empty out_table. + if self.assumptions is not None: + self.assumptions = assumption(self, self.assumptions, exclude=self.group_node.raw_groups) + if not self.assumptions.empty(): + self.group_node.deal_with_assumptions(self.assumptions, self.out_table) + self.assumptions = None self.group_node.finalize(cexprs, self.out_table) else: - create_table(self, self.out_table, cexpr = cexprs) - self.datasource.group_node = None - - has_orderby = 'orderby' in node - - if has_orderby: + # if all assumptions in projections, treat as orderby + lineage = self.assumptions is not None and has_other(self.assumptions, proj_raw_cols) + spawn = create_table(self, self.out_table, cexprs = cexprs, lineage = lineage) # create and populate out_table. + if lineage and type(spawn.lineage) is str: + lineage = spawn.lineage + self.assumptions = orderby(self, self.assumptions) # do not exclude proj_raw_cols + else: + lineage = None + if self.where is not None: + self.where.finalize() + + if type(lineage) is str: + order = 'order_' + base62uuid(6) + self.emit(f'auto {order} = {self.datasource.cxt_name}->order_by<{self.assumptions.result()}>({lineage});') + self.emit(f'{self.out_table.cxt_name}->materialize(*{order});') + self.assumptions = None + + if self.assumptions is not None: + orderby_node = orderby(self, self.assumptions) + else: + orderby_node = None + + if 'orderby' in node: self.datasource = self.out_table self.context.datasource = self.out_table # discard current ds - orderby_node = orderby(self, node['orderby']) - self.context.datasource.materialize_orderbys() - self.emit_no_ln(f"{f'{disp_varname}:+' if flatten else ''}(") + orderbys = node['orderby'] + orderby_node = orderby(self, orderbys) if orderby_node is None else orderby_node.merge(orderbys) + + if orderby_node is not None: + self.emit(f'auto {disp_varname} = {self.out_table.reference()}->order_by_view<{orderby_node.result()}>();') + else: + disp_varname = f'*{self.out_table.cxt_name}' - if self.disp or has_orderby: - self.emit(f'print(*{self.out_table.cxt_name});') + if self.disp: + self.emit(f'print({disp_varname});') - if has_orderby: - self.emit(f')[{orderby_node.view}]') - else: - self.context.emit_flush() + if flatten: - if len(self.projections) > 1 and not self.inv: - self.emit(f"{disp_varname}:+{disp_varname}") outfile(self, node['outfile']) if self.datasource_changed: diff --git a/engine/scan.py b/engine/scan.py index 4766d7a..812165e 100644 --- a/engine/scan.py +++ b/engine/scan.py @@ -5,16 +5,20 @@ from engine.expr import expr class scan(ast_node): name = 'scan' - def __init__(self, parent: "ast_node", node, size = None, context: Context = None): + def __init__(self, parent: "ast_node", node, size = None, context: Context = None, const = False): self.type = type self.size = size + self.const = "const " if const else "" super().__init__(parent, node, context) def init(self, _): self.datasource = self.context.datasource + self.initializers = '' self.start = '' + self.front = '' self.body = '' self.end = '}' - self.filter = None + self.mode = None + self.filters = [] scan_vars = set(s.it_var for s in self.context.scans) self.it_ver = 'i' + base62uuid(2) while(self.it_ver in scan_vars): @@ -22,19 +26,33 @@ class scan(ast_node): self.parent.context.scans.append(self) def produce(self, node): if type(node) is ColRef: + self.colref = node if self.size is None: - self.start += f'for (auto& {self.it_ver} : {node.reference()}) {{\n' + self.mode = ["col", node.table] + self.start += f'for ({self.const}auto& {self.it_ver} : {node.reference()}) {{\n' else: + self.mode = ["idx", node.table] self.start += f"for (uint32_t {self.it_ver} = 0; {self.it_ver} < {node.reference()}.size; ++{self.it_ver}){{\\n" elif type(node) is str: - self.start+= f'for(auto& {self.it_ver} : {node}) {{\n' + self.mode = ["idx", None] + self.start+= f'for({self.const}auto& {self.it_ver} : {node}) {{\n' else: + self.mode = ["idx", node] # Node is the TableInfo self.start += f"for (uint32_t {self.it_ver} = 0; {self.it_ver} < {self.size}; ++{self.it_ver}){{\n" - def add(self, stmt): - self.body+=stmt + '\n' + def add(self, stmt, position = "body"): + if position == "body": + self.body += stmt + '\n' + elif position == "init": + self.initializers += stmt + '\n' + else: + self.front += stmt + '\n' + def finalize(self): - self.context.remove_scan(self, self.start + self.body + self.end) + for f in self.filters: + self.start += f + self.end += '}' + self.context.remove_scan(self, self.initializers + self.start + self.front + self.body + self.end) class filter(ast_node): name = 'filter' @@ -45,7 +63,7 @@ class filter(ast_node): self.datasource = self.context.datasource self.view = View(self.context, self.datasource) self.value = None - + def spawn(self, node): # TODO: deal with subqueries self.modified_node = node @@ -64,9 +82,18 @@ class filter(ast_node): self.emit(f'{tmpVar}:{self.value}') for o, c in zip(self.output.columns, self.datasource.columns): self.emit(f'{o.cname}:$[{tmpVar};{c.cname};()]') - - def consume(self, node): + + def finalize(self): + self.scanner.finalize() + def consume(self, _): # TODO: optimizations after converting expr to cnf + self.scanner = None + for s in self.context.scans: + if self.datasource == s.mode[1]: + self.scanner = s + break + if self.scanner is None: + self.scanner = scan(self, self.datasource, self.datasource.get_size()) self.expr = expr(self, self.modified_node) - print(node) + self.scanner.filters.append(f'if ({self.expr.cexpr(self.scanner.it_ver)}) {{\n') \ No newline at end of file diff --git a/engine/utils.py b/engine/utils.py index 283a80f..5a7c427 100644 --- a/engine/utils.py +++ b/engine/utils.py @@ -16,4 +16,10 @@ def enlist(l): return l if type(l) is list else [l] def seps(s, i, l): - return s if i < len(l) - 1 else '' \ No newline at end of file + return s if i < len(l) - 1 else '' + +def has_other(a, b): + for ai in a: + if ai not in b: + return True + return False \ No newline at end of file diff --git a/header.k b/header.k deleted file mode 100644 index 7be225d..0000000 --- a/header.k +++ /dev/null @@ -1,68 +0,0 @@ -import`csv - -md:{y-x*_y%x} - -maxs:{[L]{max(x, y)}\L} -mins:{[L]{min(x, y)}\L} -sums:{[L]{(x + y)}\L} - -avgsimpl:{[L;i] curr:L[i]%(i+1); $[i<(#L)-1;curr, avgsimpl[L;i+1];curr]} -avgs:{[L] avgsimpl[sums[L];0]} - -/ maxswimp:{[L;w;i] curr:max(L@(((i-w)+!w)|0)); $[i<#L;curr, maxswimp[L; w; i + 1];curr]} -/ maxsw:{[w;L]maxswimp[L; w; 1]} - -/ minswimp:{[L;w;i] curr:min(L@(((i-w)+!w)|0)); $[i<#L;curr, maxswimp[L; w; i + 1];curr]} -/ minsw:{[w;L]minswimp[L;w;1]} - -/ avgswimp:{[L;w;s;i] s:(s+L[i])-L[i-w];curr:s%((i+1)&w);$[i<(#L)-1; curr, avgswimp[L; w; s; i+1]; curr]} -/ avgsw:{[w;L] avgswimp[L;w;0;0]} - -/ sumswimp:{[L;w;s;i] s:(s+L[i])-L[i-w];$[i<(#L)-1; s, sumswimp[L; w; s; i+1]; s]} -/ sumsw:{[w;L] sumswimp[L;w;0;0]} - - -groupby0:{[L] - {[x;y] - x:$[(@x)=`i;(,(L[0]))!,(,0);x]; - k:(,(L[y]));gvk:x[k][0]; - found:$[(gvk[0]+gvk[1])>0;1;L[y] in !x]; - cg:(,L[y])!$[found;,gvk[0],y;,(,y)]; - (x,cg)}/!(#L)} - -groupBy:{[x]groupBySingle:{[a;x] - findAll:{[c;xx] - f:{[i;c]$[(c[0])[i]~c[1];i+1;0]}; - @[!#xx;!#xx;f;(#xx)#,(xx;c)]}; - z:findAll[a;x]; - b:(findAll[0;z]_(!(1+#z)))-1;(a;b)}; - x:+x;y:?x; - @[y;!#y;groupBySingle;(#y)#,x]} - -groupby:{[L] - L:^+L; - dimy:(#(L[0]))-1; - ((({[L;dim;x;y] - x:$[x~0;(,(dim#(L[0])),0);x]; - curr:dim#(L[y]); - $[(dim#*x)~curr;x;((,curr,y),x)]}[L;dimy])/!(#L));(+L)[dimy]) } - -lststr:{[L](+({[x;y] ($x,$y)}/L))[0]} -delist:{[L] $[(@L)in(`LL`LC`LG`L);delist[(,/L)];L]} -cntlist:{[L;i] $[(@L)in(`LL`LC`LG`L);cntlist[(,/L);i+1];i+1]} -flatRO:{[x]x[0],/:x[1]} -flatLO:{[x]x[0],\:x[1]} -flatBOTH:{[x],/(x[0],/:\:x[1])} - - -sumswkrl:{[L;w;x;y] ((x-L[y-w])+L[y])} -sumsw:{[w;L] $[(#L)=0;L;(sumswkrl[L;w])\@[!#L;0;L[file 0]]]} -avgswkrl:{[L;w;x;y] (x-(L[y-w]-L[y])%w)} -avgsw:{[w;L] $[(#L)=0;L;(avgswkrl[L;w])\@[!#L;0;L[0]]]} - -/ minsw:{[w;L] ({[L;w;x] min(L[$[x>w;(!w) + ((x-w)+1);!(x+1)]])}[L;w])'!#L} -import`mmw -minsw:{[w;L] ret:L; mmw[ret;((`g ($@ret)[0]), (#ret), w, 65536)];ret} -maxsw:{[w;L] ret:L; mmw[ret;((`g ($@ret)[0]), (#ret), w, 65537)];ret} -minswip:{[w;L] mmw[L;((`g ($@L)[0]), (#L), w, 65536)];} -maxswip:{[w;L] mmw[L;((`g ($@L)[0]), (#L), w, 65537)];} \ No newline at end of file diff --git a/join.k b/join.k deleted file mode 100644 index 0a220d8..0000000 --- a/join.k +++ /dev/null @@ -1 +0,0 @@ -{[I;L;R;i] t:&(I[i]);d:(#t);$[d=0;!0;+(Aa[t];Ab[t];d#(Bb[i]))]}[(Aa {x>(y*10)}/: Ba); Aa; Ba]'!#Aa \ No newline at end of file diff --git a/libcsv.a b/libcsv.a deleted file mode 100644 index 036fac9..0000000 Binary files a/libcsv.a and /dev/null differ diff --git a/mmw.dylib b/mmw.dylib deleted file mode 100644 index dea9d80..0000000 Binary files a/mmw.dylib and /dev/null differ diff --git a/mmw.so b/mmw.so deleted file mode 100644 index dd2900d..0000000 Binary files a/mmw.so and /dev/null differ diff --git a/moving_avg.a b/moving_avg.a index 2d4d5fa..f9b21e5 100644 --- a/moving_avg.a +++ b/moving_avg.a @@ -8,7 +8,7 @@ SELECT Month,avgs(3,sales) FROM sale ASSUMING ASC Month INTO OUTFILE "moving_avg_output.csv" -FIELDS TERMINATED BY "," +FIELDS TERMINATED BY ";" select sales, mins(2,Month) from sale group by sales into outfile "flatten.csv" \ No newline at end of file diff --git a/msc-plugin/lineage.hpp b/msc-plugin/lineage.hpp new file mode 100644 index 0000000..cb20820 --- /dev/null +++ b/msc-plugin/lineage.hpp @@ -0,0 +1,5 @@ +#pragma once +template +struct lineage { + +}; \ No newline at end of file diff --git a/msc-plugin/msc-plugin.vcxproj b/msc-plugin/msc-plugin.vcxproj index 78d05a4..682cfb8 100644 --- a/msc-plugin/msc-plugin.vcxproj +++ b/msc-plugin/msc-plugin.vcxproj @@ -93,6 +93,7 @@ true _CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) true + stdcpplatest Console @@ -108,6 +109,7 @@ true _CRT_SECURE_NO_WARNINGS;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) true + stdcpplatest Console @@ -123,6 +125,7 @@ true _CRT_SECURE_NO_WARNINGS;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) true + stdcpplatest Console @@ -138,6 +141,7 @@ true _CRT_SECURE_NO_WARNINGS;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) true + stdcpplatest Console diff --git a/msvs-py/msvs-py.pyproj b/msvs-py/msvs-py.pyproj new file mode 100644 index 0000000..b2fdde2 --- /dev/null +++ b/msvs-py/msvs-py.pyproj @@ -0,0 +1,93 @@ + + + Debug + 2.0 + ccc243f5-663e-45b7-a6de-b2468c58b3a7 + . + + + ..\msvs-py + . + . + msvs-py + msvs-py + + + true + false + + + true + false + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/out.cpp b/out.cpp index 8e4f407..03d68f4 100644 --- a/out.cpp +++ b/out.cpp @@ -1,60 +1,61 @@ +#include "./server/hasher.h" #include "./server/aggregations.h" +#include "csv.h" #include "./server/libaquery.h" +#include extern "C" int __DLLEXPORT__ dllmain(Context* cxt) { using namespace std; using namespace types; - auto stocks = new TableInfo("stocks", 2); -cxt->tables.insert({"stocks", stocks}); -auto& stocks_timestamp = *(ColRef *)(&stocks->colrefs[0]); -auto& stocks_price = *(ColRef *)(&stocks->colrefs[1]); -stocks_timestamp.init(); -stocks_price.init(); -stocks_timestamp.emplace_back(1); -stocks_price.emplace_back(15); -stocks_timestamp.emplace_back(2); -stocks_price.emplace_back(19); -stocks_timestamp.emplace_back(3); -stocks_price.emplace_back(16); -stocks_timestamp.emplace_back(4); -stocks_price.emplace_back(17); -stocks_timestamp.emplace_back(5); -stocks_price.emplace_back(15); -stocks_timestamp.emplace_back(6); -stocks_price.emplace_back(13); -stocks_timestamp.emplace_back(7); -stocks_price.emplace_back(5); -stocks_timestamp.emplace_back(8); -stocks_price.emplace_back(8); -stocks_timestamp.emplace_back(9); -stocks_price.emplace_back(7); -stocks_timestamp.emplace_back(10); -stocks_price.emplace_back(13); -stocks_timestamp.emplace_back(11); -stocks_price.emplace_back(11); -stocks_timestamp.emplace_back(12); -stocks_price.emplace_back(14); -stocks_timestamp.emplace_back(13); -stocks_price.emplace_back(10); -stocks_timestamp.emplace_back(14); -stocks_price.emplace_back(5); -stocks_timestamp.emplace_back(15); -stocks_price.emplace_back(2); -stocks_timestamp.emplace_back(16); -stocks_price.emplace_back(5); -auto out_ZPYh = new TableInfo>("out_ZPYh", 1); -cxt->tables.insert({"out_ZPYh", out_ZPYh}); -auto& out_ZPYh_maxstockspriceminstockstimestamp = *(ColRef> *)(&out_ZPYh->colrefs[0]); -out_ZPYh_maxstockspriceminstockstimestamp.init(); -out_ZPYh_maxstockspriceminstockstimestamp = max((stocks_price-min(stocks_timestamp))); -print(*out_ZPYh); - -auto out_1ac3 = new TableInfo>("out_1ac3", 1); -cxt->tables.insert({"out_1ac3", out_1ac3}); -auto& out_1ac3_maxstockspriceminsstocksprice = *(ColRef> *)(&out_1ac3->colrefs[0]); -out_1ac3_maxstockspriceminsstocksprice.init(); -out_1ac3_maxstockspriceminsstocksprice = max((stocks_price-mins(stocks_price))); -print(*out_1ac3); + + auto sale = new TableInfo("sale", 2); +cxt->tables.insert({"sale", sale}); +auto& sale_Month = *(ColRef *)(&sale->colrefs[0]); +auto& sale_sales = *(ColRef *)(&sale->colrefs[1]); +sale_Month.init(); +sale_sales.init(); +io::CSVReader<2> csv_reader_6T89Ll("moving_avg.csv"); +csv_reader_6T89Ll.read_header(io::ignore_extra_column, "Month","sales"); +int tmp_5vttJ2yV; +int tmp_2ckq15YU; +while(csv_reader_6T89Ll.read_row(tmp_5vttJ2yV,tmp_2ckq15YU)) { +sale_Month.emplace_back(tmp_5vttJ2yV); +sale_sales.emplace_back(tmp_2ckq15YU); +} +auto out_2UiD = new TableInfo>,value_type>>("out_2UiD", 2); +cxt->tables.insert({"out_2UiD", out_2UiD}); +auto& out_2UiD_Month = *(ColRef>> *)(&out_2UiD->colrefs[0]); +auto& out_2UiD_avgw3salesales = *(ColRef>> *)(&out_2UiD->colrefs[1]); +auto order_1NNZ9F = sale->order_by<0>(); +out_2UiD_Month.init(); +out_2UiD_Month = sale_Month[*order_1NNZ9F]; +out_2UiD_avgw3salesales.init(); +out_2UiD_avgw3salesales = avgw(3,sale_sales[*order_1NNZ9F]); +print(*out_2UiD); +FILE* fp_6xIJn4 = fopen("moving_avg_output.csv", "w"); +out_2UiD->printall(";", "\n", nullptr, fp_6xIJn4); +fclose(fp_6xIJn4); +typedef record record_type6Lepq5T; +unordered_map, transTypes> g4loWjmn; +for (uint32_t i5g = 0; i5g < sale_sales.size; ++i5g){ +g4loWjmn[forward_as_tuple(sale_sales[i5g])].emplace_back(i5g); +} +auto out_2YlO = new TableInfo>,decays>("out_2YlO", 2); +cxt->tables.insert({"out_2YlO", out_2YlO}); +auto& out_2YlO_sales = *(ColRef>> *)(&out_2YlO->colrefs[0]); +auto& out_2YlO_minw2saleMonth = *(ColRef> *)(&out_2YlO->colrefs[1]); +out_2YlO_sales.init(); +out_2YlO_minw2saleMonth.init(); +for(auto& iFU : g4loWjmn) { +auto &key_3AwvKMR = iFU.first; +auto &val_7jtE12E = iFU.second; +out_2YlO_sales.emplace_back(get<0>(key_3AwvKMR)); +out_2YlO_minw2saleMonth.emplace_back(minw(2,sale_Month[val_7jtE12E])); +} +print(*out_2YlO); +FILE* fp_45ld6S = fopen("flatten.csv", "w"); +out_2YlO->printall(",", "\n", nullptr, fp_45ld6S); +fclose(fp_45ld6S); return 0; } \ No newline at end of file diff --git a/out_attempt1.cpp b/out_attempt1.cpp new file mode 100644 index 0000000..de6ffb3 --- /dev/null +++ b/out_attempt1.cpp @@ -0,0 +1,60 @@ +#include "./server/libaquery.h" +#include +#include "./server/hasher.h" +#include "csv.h" +#include "./server/aggregations.h" + + extern "C" int __DLLEXPORT__ dllmain(Context* cxt) { + using namespace std; + using namespace types; + + auto sale = new TableInfo("sale", 2); +cxt->tables.insert({"sale", sale}); +auto& sale_Month = *(ColRef *)(&sale->colrefs[0]); +auto& sale_sales = *(ColRef *)(&sale->colrefs[1]); +sale_Month.init(); +sale_sales.init(); +io::CSVReader<2> csv_reader_53ychC("moving_avg.csv"); +csv_reader_53ychC.read_header(io::ignore_extra_column, "Month","sales"); +int tmp_7ttMnHd3; +int tmp_5nHjeAtP; +while(csv_reader_53ychC.read_row(tmp_7ttMnHd3,tmp_5nHjeAtP)) { + +sale_Month.emplace_back(tmp_7ttMnHd3); +sale_sales.emplace_back(tmp_5nHjeAtP); +} +auto out_3Xio = new TableInfo,decays>("out_3Xio", 2); +cxt->tables.insert({"out_3Xio", out_3Xio}); +auto& out_3Xio_Month = *(ColRef> *)(&out_3Xio->colrefs[0]); +auto& out_3Xio_avgsw3salesales = *(ColRef> *)(&out_3Xio->colrefs[1]); +out_3Xio_Month.init(); +out_3Xio_Month = sale_Month; +out_3Xio_avgsw3salesales.init(); +out_3Xio_avgsw3salesales = avgw(3,sale_sales); +// print(*out_3Xio); +FILE* fp_4nKGhD = fopen("moving_avg_output.csv", "w"); +out_3Xio->printall(",", "\n", nullptr, fp_4nKGhD); +fclose(fp_4nKGhD); +typedef record record_type1H2vDGL; +unordered_map, transTypes> g6Mjxfk5; +for (uint32_t i7u = 0; i7u < sale_sales.size; ++i7u){ +g6Mjxfk5[forward_as_tuple(sale_sales[i7u])].emplace_back(i7u); +} +auto out_2IU2 = new TableInfo,decays>("out_2IU2", 2); +cxt->tables.insert({"out_2IU2", out_2IU2}); +auto& out_2IU2_sales = *(ColRef> *)(&out_2IU2->colrefs[0]); +auto& out_2IU2_minsw2saleMonth = *(ColRef> *)(&out_2IU2->colrefs[1]); +out_2IU2_sales.init(); +out_2IU2_minsw2saleMonth.init(); +for(auto& i5J : g6Mjxfk5) { +auto &key_4jl5toH = i5J.first; +auto &val_VJGwVwH = i5J.second; +out_2IU2_sales.emplace_back(get<0>(key_4jl5toH)); +out_2IU2_minsw2saleMonth.emplace_back(minw(2,sale_Month[val_VJGwVwH])); +} +// print(*out_2IU2); +FILE* fp_18R4fY = fopen("flatten.csv", "w"); +out_2IU2->printall(",","\n", nullptr, fp_18R4fY); +fclose(fp_18R4fY); +return 0; +} \ No newline at end of file diff --git a/out_working.cpp b/out_working.cpp deleted file mode 100644 index 5fbb369..0000000 --- a/out_working.cpp +++ /dev/null @@ -1,46 +0,0 @@ -#include "./server/libaquery.h" -#include -#include -extern "C" int dllmain(Context *cxt) -{ - auto stocks = new TableInfo("stocks", 2); - cxt->tables.insert(std::make_pair("stocks", stocks)); - stocks->colrefs[0].ty = types::AINT; - stocks->colrefs[1].ty = types::AINT; - auto &stocks_0 = *(ColRef *)(&stocks->colrefs[0]); - auto &stocks_1 = *(ColRef *)(&stocks->colrefs[1]); - stocks_0.emplace_back(1); - stocks_1.emplace_back(15); - stocks_0.emplace_back(2); - stocks_1.emplace_back(19); - stocks_0.emplace_back(3); - stocks_1.emplace_back(16); - stocks_0.emplace_back(4); - stocks_1.emplace_back(17); - stocks_0.emplace_back(5); - stocks_1.emplace_back(15); - stocks_0.emplace_back(6); - stocks_1.emplace_back(13); - stocks_0.emplace_back(7); - stocks_1.emplace_back(5); - stocks_0.emplace_back(8); - stocks_1.emplace_back(8); - stocks_0.emplace_back(9); - stocks_1.emplace_back(7); - stocks_0.emplace_back(10); - stocks_1.emplace_back(13); - stocks_0.emplace_back(11); - stocks_1.emplace_back(11); - stocks_0.emplace_back(12); - stocks_1.emplace_back(14); - stocks_0.emplace_back(13); - stocks_1.emplace_back(10); - stocks_0.emplace_back(14); - stocks_1.emplace_back(5); - stocks_0.emplace_back(15); - stocks_1.emplace_back(2); - stocks_0.emplace_back(16); - stocks_1.emplace_back(5); - printf("%d\n", max(stocks_0 - min(stocks_1))); - return 0; -} \ No newline at end of file diff --git a/prompt.py b/prompt.py index 9333997..e956c3a 100644 --- a/prompt.py +++ b/prompt.py @@ -1,6 +1,8 @@ from concurrent.futures import thread import re import time + +from mo_parsing import ParseException import aquery_parser as parser import engine import subprocess @@ -90,6 +92,8 @@ cxt = None while test_parser: try: if server.poll() is not None: + mm.seek(0,os.SEEK_SET) + mm.write(b'\x01\x00') server = subprocess.Popen(["./server.bin", shm]) q = input() if q == 'exec': @@ -126,6 +130,15 @@ while test_parser: mm.seek(0,os.SEEK_SET) mm.write(b'\x01\x01') continue + elif q.startswith('save'): + filename = re.split(' |\t', q) + if (len(filename) > 1): + filename = filename[1] + else: + filename = f'out_{base62uuid(4)}.cpp' + with open(filename, 'wb') as outfile: + outfile.write((cxt.finalize()).encode('utf-8')) + continue trimed = ws.sub(' ', q.lower()).split(' ') if trimed[0].startswith('f'): fn = 'stock.a' if len(trimed) <= 1 or len(trimed[1]) == 0 \ @@ -137,7 +150,7 @@ while test_parser: continue stmts = parser.parse(q) print(stmts) - except (ValueError, FileNotFoundError) as e: + except (ValueError, FileNotFoundError, ParseException) as e: rm() print(type(e), e) diff --git a/q1.sql b/q1.sql index bd6d5fd..360dc2f 100644 --- a/q1.sql +++ b/q1.sql @@ -7,4 +7,4 @@ FIELDS TERMINATED BY "," SELECT sum(c), b, d FROM test group by a,b,d --- order by d DESC, b ASC +order by d DESC, b ASC diff --git a/server/aggregations.h b/server/aggregations.h index 71de8aa..7a5accc 100644 --- a/server/aggregations.h +++ b/server/aggregations.h @@ -39,10 +39,10 @@ T min(const VT& v) { return min_v; } template class VT> -VT mins(const VT& arr) { - const int& len = arr.size; +decayed_t mins(const VT& arr) { + const uint32_t& len = arr.size; std::deque> cache; - VT ret(len); + decayed_t ret(len); T min = std::numeric_limits::max(); for (int i = 0; i < len; ++i) { if (arr[i] < min) @@ -52,9 +52,9 @@ VT mins(const VT& arr) { return ret; } template class VT> -VT maxs(const VT& arr) { - const int& len = arr.size; - VT ret(len); +decayed_t maxs(const VT& arr) { + const uint32_t& len = arr.size; + decayed_t ret(len); T max = std::numeric_limits::min(); for (int i = 0; i < len; ++i) { if (arr[i] > max) @@ -65,9 +65,9 @@ VT maxs(const VT& arr) { } template class VT> -VT minw(const VT& arr, uint32_t w) { - const int& len = arr.size; - VT ret(len); +decayed_t minw(uint32_t w, const VT& arr) { + const uint32_t& len = arr.size; + decayed_t ret{len}; std::deque> cache; for (int i = 0; i < len; ++i) { if (!cache.empty() && cache.front().second == i - w) cache.pop_front(); @@ -79,9 +79,9 @@ VT minw(const VT& arr, uint32_t w) { } template class VT> -VT maxw(const VT& arr, uint32_t w) { - const int& len = arr.size; - VT ret(len); +decayed_t maxw(uint32_t w, const VT& arr) { + const uint32_t& len = arr.size; + decayed_t ret(len); std::deque> cache; for (int i = 0; i < len; ++i) { if (!cache.empty() && cache.front().second == i - w) cache.pop_front(); @@ -92,14 +92,66 @@ VT maxw(const VT& arr, uint32_t w) { return ret; } +template class VT> +decayed_t> sums(const VT& arr) { + const uint32_t& len = arr.size; + decayed_t> ret(len); + uint32_t i = 0; + if(len) ret[i++] = arr[0]; + for (; i < len; ++i) + ret[i] = ret[i-1] + arr[i]; + return ret; +} +template class VT> +decayed_t> avgs(const VT& arr) { + const uint32_t& len = arr.size; + typedef types::GetFPType FPType; + decayed_t ret(len); + uint32_t i = 0; + types::GetLongType s; + if(len) s = ret[i++] = arr[0]; + for (; i < len; ++i) + ret[i] = (s+=arr[i])/(FPType)(i+1); + return ret; +} +template class VT> +decayed_t> sumw(uint32_t w, const VT& arr) { + const uint32_t& len = arr.size; + decayed_t> ret(len); + uint32_t i = 0; + w = w > len ? len : w; + if(len) ret[i++] = arr[0]; + for (; i < w; ++i) + ret[i] = ret[i-1] + arr[i]; + for (; i < len; ++i) + ret[i] = ret[i-1] + arr[i] - arr[i-w]; + return ret; +} +template class VT> +decayed_t> avgw(uint32_t w, const VT& arr) { + typedef types::GetFPType FPType; + const uint32_t& len = arr.size; + decayed_t ret(len); + uint32_t i = 0; + types::GetLongType s; + w = w > len ? len : w; + if(len) s = ret[i++] = arr[0]; + for (; i < w; ++i) + ret[i] = (s += arr[i])/(FPType)(i+1); + for (; i < len; ++i) + ret[i] = ret[i-1] + (arr[i] - arr[i-w])/(FPType)w; + return ret; +} + +template constexpr inline T count(const T& v) { return 1; } template constexpr inline T max(const T& v) { return v; } template constexpr inline T min(const T& v) { return v; } template constexpr inline T avg(const T& v) { return v; } template constexpr inline T sum(const T& v) { return v; } -template constexpr inline T maxw(const T& v, uint32_t) { return v; } -template constexpr inline T minw(const T& v, uint32_t) { return v; } -template constexpr inline T avgw(const T& v, uint32_t) { return v; } -template constexpr inline T sumw(const T& v, uint32_t) { return v; } +template constexpr inline T maxw(uint32_t, const T& v) { return v; } +template constexpr inline T minw(uint32_t, const T& v) { return v; } +template constexpr inline T avgw(uint32_t, const T& v) { return v; } +template constexpr inline T sumw(uint32_t, const T& v) { return v; } template constexpr inline T maxs(const T& v) { return v; } template constexpr inline T mins(const T& v) { return v; } template constexpr inline T avgs(const T& v) { return v; } diff --git a/server/io.h b/server/io.h new file mode 100644 index 0000000..43be108 --- /dev/null +++ b/server/io.h @@ -0,0 +1,15 @@ +#pragma once +#include "types.h" +#include +#include +template +std::string generate_printf_string(const char* sep = " ", const char* end = "\n") { + std::string str; + ((str += types::printf_str[types::Types>::getType()], str += sep), ...); + const auto trim = str.size() - strlen(sep); + if (trim > 0) + str.resize(trim); + str += end; + return str; +} + diff --git a/server/priority_vector.hpp b/server/priority_vector.hpp new file mode 100644 index 0000000..f1c7d6f --- /dev/null +++ b/server/priority_vector.hpp @@ -0,0 +1,19 @@ +#pragma once +#include "vector_type.hpp" +#include +#include +template +class priority_vector : public vector_type { + const Comparator comp; +public: + priority_vector(Comparator comp = std::less{}) : + comp(comp), vector_type(0) {} + void emplace_back(T val) { + vector_type::emplace_back(val); + std::push_heap(container, container + size, comp); + } + void pop_back() { + std::pop_heap(container, container + size, comp); + --size; + } +}; \ No newline at end of file diff --git a/server/server.cpp b/server/server.cpp index 5ff0738..8a32f10 100644 --- a/server/server.cpp +++ b/server/server.cpp @@ -75,6 +75,7 @@ int main(int argc, char** argv) { shm.FreeMemoryMap(); return 0; } +#include "utils.h" int _main() { @@ -83,6 +84,7 @@ int _main() //t.emplace_back(2); //print(t); //return 0; + puts(cpp_17 ?"true":"false"); void* handle = dlopen("dll.so", RTLD_LAZY); printf("handle: %x\n", handle); Context* cxt = new Context(); @@ -96,7 +98,8 @@ int _main() } dlclose(handle); } - + //static_assert(std::is_same_v()), std::integer_sequence>, ""); return 0; + std::unordered_map a; } diff --git a/server/server.sln b/server/server.sln index e3b37e3..d22be11 100644 --- a/server/server.sln +++ b/server/server.sln @@ -8,32 +8,50 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "server", "server.vcxproj", {8081FDAA-4D13-4B7A-ADB2-8224AF7F1C81} = {8081FDAA-4D13-4B7A-ADB2-8224AF7F1C81} EndProjectSection EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "Project1", "..\Project1\Project1.vcxproj", "{8081FDAA-4D13-4B7A-ADB2-8224AF7F1C81}" +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "msc-plugin", "..\msc-plugin\msc-plugin.vcxproj", "{8081FDAA-4D13-4B7A-ADB2-8224AF7F1C81}" +EndProject +Project("{888888A0-9F3D-457C-B088-3A5042F75D52}") = "msvs-py", "..\msvs-py\msvs-py.pyproj", "{CCC243F5-663E-45B7-A6DE-B2468C58B3A7}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU Debug|x64 = Debug|x64 Debug|x86 = Debug|x86 + Release|Any CPU = Release|Any CPU Release|x64 = Release|x64 Release|x86 = Release|x86 EndGlobalSection GlobalSection(ProjectConfigurationPlatforms) = postSolution + {031352C2-AFBB-45AA-9518-DBC1F9EF2AF3}.Debug|Any CPU.ActiveCfg = Debug|x64 + {031352C2-AFBB-45AA-9518-DBC1F9EF2AF3}.Debug|Any CPU.Build.0 = Debug|x64 {031352C2-AFBB-45AA-9518-DBC1F9EF2AF3}.Debug|x64.ActiveCfg = Debug|x64 {031352C2-AFBB-45AA-9518-DBC1F9EF2AF3}.Debug|x64.Build.0 = Debug|x64 {031352C2-AFBB-45AA-9518-DBC1F9EF2AF3}.Debug|x86.ActiveCfg = Debug|Win32 {031352C2-AFBB-45AA-9518-DBC1F9EF2AF3}.Debug|x86.Build.0 = Debug|Win32 + {031352C2-AFBB-45AA-9518-DBC1F9EF2AF3}.Release|Any CPU.ActiveCfg = Release|x64 + {031352C2-AFBB-45AA-9518-DBC1F9EF2AF3}.Release|Any CPU.Build.0 = Release|x64 {031352C2-AFBB-45AA-9518-DBC1F9EF2AF3}.Release|x64.ActiveCfg = Release|x64 {031352C2-AFBB-45AA-9518-DBC1F9EF2AF3}.Release|x64.Build.0 = Release|x64 {031352C2-AFBB-45AA-9518-DBC1F9EF2AF3}.Release|x86.ActiveCfg = Release|Win32 {031352C2-AFBB-45AA-9518-DBC1F9EF2AF3}.Release|x86.Build.0 = Release|Win32 + {8081FDAA-4D13-4B7A-ADB2-8224AF7F1C81}.Debug|Any CPU.ActiveCfg = Debug|x64 + {8081FDAA-4D13-4B7A-ADB2-8224AF7F1C81}.Debug|Any CPU.Build.0 = Debug|x64 {8081FDAA-4D13-4B7A-ADB2-8224AF7F1C81}.Debug|x64.ActiveCfg = Debug|x64 {8081FDAA-4D13-4B7A-ADB2-8224AF7F1C81}.Debug|x64.Build.0 = Debug|x64 {8081FDAA-4D13-4B7A-ADB2-8224AF7F1C81}.Debug|x86.ActiveCfg = Debug|Win32 {8081FDAA-4D13-4B7A-ADB2-8224AF7F1C81}.Debug|x86.Build.0 = Debug|Win32 + {8081FDAA-4D13-4B7A-ADB2-8224AF7F1C81}.Release|Any CPU.ActiveCfg = Release|x64 + {8081FDAA-4D13-4B7A-ADB2-8224AF7F1C81}.Release|Any CPU.Build.0 = Release|x64 {8081FDAA-4D13-4B7A-ADB2-8224AF7F1C81}.Release|x64.ActiveCfg = Release|x64 {8081FDAA-4D13-4B7A-ADB2-8224AF7F1C81}.Release|x64.Build.0 = Release|x64 {8081FDAA-4D13-4B7A-ADB2-8224AF7F1C81}.Release|x86.ActiveCfg = Release|Win32 {8081FDAA-4D13-4B7A-ADB2-8224AF7F1C81}.Release|x86.Build.0 = Release|Win32 + {CCC243F5-663E-45B7-A6DE-B2468C58B3A7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {CCC243F5-663E-45B7-A6DE-B2468C58B3A7}.Debug|x64.ActiveCfg = Debug|Any CPU + {CCC243F5-663E-45B7-A6DE-B2468C58B3A7}.Debug|x86.ActiveCfg = Debug|Any CPU + {CCC243F5-663E-45B7-A6DE-B2468C58B3A7}.Release|Any CPU.ActiveCfg = Release|Any CPU + {CCC243F5-663E-45B7-A6DE-B2468C58B3A7}.Release|x64.ActiveCfg = Release|Any CPU + {CCC243F5-663E-45B7-A6DE-B2468C58B3A7}.Release|x86.ActiveCfg = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/server/server.vcxproj b/server/server.vcxproj index 221f1dd..0c38f23 100644 --- a/server/server.vcxproj +++ b/server/server.vcxproj @@ -23,32 +23,32 @@ Win32Proj {031352c2-afbb-45aa-9518-dbc1f9ef2af3} server - 10.0 + 10.0.19041.0 Application true - v143 + v141 Unicode Application false - v143 + v141 true Unicode Application true - v143 + v141 Unicode Application false - v143 + v141 true Unicode @@ -88,9 +88,9 @@ true _CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) true - stdcpp20 true true + stdcpplatest Console @@ -105,9 +105,9 @@ true _CRT_SECURE_NO_WARNINGS;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) true - stdcpp20 true true + stdcpplatest Console @@ -122,9 +122,9 @@ true _CRT_SECURE_NO_WARNINGS;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) true - stdcpp20 true true + stdcpplatest Console @@ -139,9 +139,9 @@ true _CRT_SECURE_NO_WARNINGS;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) true - stdcpp20 true true + stdcpplatest Console @@ -167,7 +167,9 @@ + + @@ -175,10 +177,4 @@ - - - - This project references NuGet package(s) that are missing on this computer. Use NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. - - \ No newline at end of file diff --git a/server/table.h b/server/table.h index 44a8448..0179ab8 100644 --- a/server/table.h +++ b/server/table.h @@ -1,9 +1,13 @@ +// TODO: Replace `cout, printf` with sprintf&fputs and custom buffers + #ifndef _TABLE_H #define _TABLE_H #include "types.h" #include "vector_type.hpp" #include +#include +#include "io.h" template class vector_type; template <> @@ -24,8 +28,10 @@ template class ColRef : public vector_type<_Ty> { public: + typedef ColRef<_Ty> Decayed_t; const char* name; types::Type_t ty = types::ERROR; + ColRef() : vector_type<_Ty>(0), name("") {} ColRef(const uint32_t& size, const char* name = "") : vector_type<_Ty>(size), name(name) {} ColRef(const char* name) : name(name) {} void init() { ty = types::Types<_Ty>::getType(); this->size = this->capacity = 0; this->container = 0; } @@ -35,6 +41,14 @@ public: ColView<_Ty> operator [](const vector_type&idxs) const { return ColView<_Ty>(*this, idxs); } + + void out(uint32_t n = 4, const char* sep = " ") const { + n = n > this->size ? this->size : n; + std::cout << '('; + for (uint32_t i = 0; i < n; ++i) + std::cout << this->operator[](i) << sep; + std::cout << ')'; + } template ColRef scast(); }; @@ -42,6 +56,7 @@ public: template class ColView { public: + typedef ColRef<_Ty> Decayed_t; const vector_type& idxs; const ColRef<_Ty>& orig; const uint32_t& size; @@ -75,8 +90,22 @@ public: Iterator_t end() const { return Iterator_t(idxs.end(), orig); } + void out(uint32_t n = 4, const char* sep = " ") const { + n = n > size ? size : n; + std::cout<<'('; + for (uint32_t i = 0; i < n; ++i) + std::cout << this->operator[](i)<< sep; + std::cout << ')'; + } }; - +template