From c46ce578d4dfe47d01254bd35174b36e8222fdb2 Mon Sep 17 00:00:00 2001 From: Bill Sun Date: Thu, 14 Apr 2022 01:53:11 +0800 Subject: [PATCH] Single Table query part 1 --- Makefile | 11 +- README.md | 247 ++-------------------------------- engine/ast.py | 46 ++++--- engine/ddl.py | 55 +++++--- engine/expr.py | 63 +++++---- engine/groupby.py | 65 ++++----- engine/projection.py | 36 ++--- engine/scan.py | 34 ++++- msc-plugin/msc-plugin.vcxproj | 168 +++++++++++++++++++++++ out.cpp | 30 +++-- prompt.py | 48 +++++-- q1.sql | 2 +- server/aggregations.h | 106 +++++++++++++++ server/hasher.h | 20 +++ server/libaquery.h | 8 ++ server/packages.config | 4 - server/server.cpp | 84 +++--------- server/server.sln | 13 ++ server/server.vcxproj | 30 +++-- server/table.h | 221 +++++++++++++++++++----------- server/types.h | 47 ++++++- server/utils.cpp | 18 +++ server/utils.h | 30 ++--- server/vector_type.hpp | 52 +++---- server/winhelper.cpp | 2 +- 25 files changed, 862 insertions(+), 578 deletions(-) create mode 100644 msc-plugin/msc-plugin.vcxproj create mode 100644 server/aggregations.h create mode 100644 server/hasher.h delete mode 100644 server/packages.config create mode 100644 server/utils.cpp diff --git a/Makefile b/Makefile index bac0dda..dd4dc9b 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,13 @@ +OS_SUPPORT = + +ifeq ($(OS),Windows_NT) + OS_SUPPORT += server/winhelper.cpp +endif +$(info $(OS_SUPPORT)) + +server.bin: + g++ server/server.cpp $(OS_SUPPORT) --std=c++1z -O3 -march=native -o server.bin snippet: - g++ -shared --std=c++1z out.cpp -O3 -march=native -o dll.so + g++ -shared -fPIC --std=c++1z out.cpp -O3 -march=native -o dll.so clean: rm *.shm -rf \ No newline at end of file diff --git a/README.md b/README.md index 8567239..099ba68 100644 --- a/README.md +++ b/README.md @@ -1,239 +1,20 @@ # AQuery Compiler -AQuery Compiler that compiles AQuery into [c](https://shakti.com/). +AQuery Compiler that compiles AQuery into C++11. Frontend built on top of [mo-sql-parsing](https://github.com/klahnakoski/mo-sql-parsing). ## Roadmap -- [x] SQL Parser -> AQuery Parser - - [ ] Data acquisition/output from/to csv file (By Jan. 21) -- -> AQuery-c Compiler - - Simple query (By Jan. 21) - - [ ] Nested queries (Jan. 28) +- [x] SQL Parser -> AQuery Parser (Front End) +- [ ] AQuery-C++ Compiler (Back End) + - [x] Schema and Data Model + - [x] Data acquisition/output from/to csv file + - [ ] Single table queries + - [x] Projections and Single Table Aggregations + - [x] Group by Aggregations + - [x] Filters + - [ ] Order by + - [ ] Assumption + - [ ] Multi-table + - [ ] Join + - [ ] Subqueries - [ ] -> Optimizing Compiler - -# Descriptions from mo-sql-parsing: - -Parse SQL into JSON so we can translate it for other datastores! - -[See changes](https://github.com/klahnakoski/mo-sql-parsing#version-changes) - - -## Problem Statement - -SQL is a familiar language used to access databases. Although, each database vendor has its quirky implementation, there is enough standardization that the average developer does not need to know of those quirks. This familiar core SQL (lowest common denominator, if you will) is useful enough to explore data in primitive ways. It is hoped that, once programmers have reviewed a datastore with basic SQL queries, and they see the value of that data, and they will be motivated to use the datastore's native query format. - -## Objectives - -The objective is to convert SQL queries to JSON-izable parse trees. This originally targeted MySQL, but has grown to include other database engines. *Please [paste some SQL into a new issue](https://github.com/klahnakoski/mo-sql-parsing/issues) if it does not work for you* - - -## Project Status - -November 2021 - There are [over 800 tests](https://app.travis-ci.com/github/klahnakoski/mo-sql-parsing). This parser is good enough for basic usage, including: - * inner queries, - * with clauses, - * window functions - * create/drop tables and views - * insert/update/delete statements - * lambda (`->`) functions - -## Install - - pip install mo-sql-parsing - -## Parsing SQL - - >>> from aquery_parser import parse - >>> parse("select count(1) from jobs") - {'select': {'value': {'count': 1}}, 'from': 'jobs'} - -Each SQL query is parsed to an object: Each clause is assigned to an object property of the same name. - - >>> parse("select a as hello, b as world from jobs") - {'select': [{'value': 'a', 'name': 'hello'}, {'value': 'b', 'name': 'world'}], 'from': 'jobs'} - -The `SELECT` clause is an array of objects containing `name` and `value` properties. - - -### SQL Flavours - -There are a few parsing modes you may be interested in: - - -#### SQLServer Identifiers (`[]`) - -SQLServer uses square brackets to delimit identifiers. For example - - SELECT [Timestamp] FROM [table] - -which conflicts with BigQuery array constructor (eg `[1, 2, 3, 4]`). You may use the SqlServer flavour with - - from aquery_parser import parse_sqlserver as parse - - -#### NULL is None - -The default output for this parser is to emit a null function `{"null":{}}` wherever `NULL` is encountered in the SQL. If you would like something different, you can replace nulls with `None` (or anything else for that matter): - - result = parse(sql, null=None) - -this has been implemented with a post-parse rewriting of the parse tree. - - -#### Normalized function call form - -The default behaviour of the parser is to output function calls in `simple_op` format: The operator being a key in the object; `{op: params}`. This form can be difficult to work with because the object must be scanned for known operators, or possible optional arguments, or at least distinguished from a query object. - -You can have the parser emit function calls in `normal_op` format - - >>> from aquery_parser import parse, normal_op - >>> parse("select trim(' ' from b+c)", calls=normal_op) - -which produces calls in a normalized format - - {"op": op, "args": args, "kwargs": kwargs} - -here is the pretty-printed JSON from the example above: - -``` -{'select': {'value': { - 'op': 'trim', - 'args': [{'op': 'add', 'args': ['b', 'c']}], - 'kwargs': {'characters': {'literal': ' '}} -}}} -``` - -#### MySQL literal strings - -MySQL uses both double quotes and single quotes to declare literal strings. This is not ansi behaviour, but it is more forgiving for programmers coming from other languages. A specific parse function is provided: - - result = parse_mysql(sql) - - -## Generating SQL - -You may also generate SQL from the a given JSON document. This is done by the formatter, which is in Alpha state (Oct2021). - - >>> from aquery_parser import format - >>> format({"from":"test", "select":["a.b", "c"]}) - 'SELECT a.b, c FROM test' - -## Contributing - -In the event that the parser is not working for you, you can help make this better but simply pasting your sql (or JSON) into a new issue. Extra points if you describe the problem. Even more points if you submit a PR with a test. If you also submit a fix, then you also have my gratitude. - - -### Run Tests - -See [the tests directory](https://github.com/klahnakoski/mo-sql-parsing/tree/dev/tests) for instructions running tests, or writing new ones. - -## More about implementation - -SQL queries are translated to JSON objects: Each clause is assigned to an object property of the same name. - - - # SELECT * FROM dual WHERE a>b ORDER BY a+b - { - "select": "*", - "from": "dual", - "where": {"gt": ["a", "b"]}, - "orderby": {"value": {"add": ["a", "b"]}} - } - -Expressions are also objects, but with only one property: The name of the operation, and the value holding (an array of) parameters for that operation. - - {op: parameters} - -and you can see this pattern in the previous example: - - {"gt": ["a","b"]} - -## Array Programming - -The `mo-sql-parsing.scrub()` method is used liberally throughout the code, and it "simplifies" the JSON. You may find this form a bit tedious to work with because the JSON property values can be values, lists of values, or missing. Please consider converting everything to arrays: - - -``` -def listwrap(value): - if value is None: - return [] - elif isinstance(value, list) - return value - else: - return [value] -``` - -then you may avoid all the is-it-a-list checks : - -``` -for select in listwrap(parsed_result.get('select')): - do_something(select) -``` - -## Version Changes - - - -### Version 8 - -*November 2021* - -* Prefer BigQuery `[]` (create array) over SQLServer `[]` (identity) -* Added basic DML (`INSERT`/`UPDATE`/`DELETE`) -* flatter `CREATE TABLE` structures. The `option` list in column definition has been flattened:
- **Old column format** - - {"create table": { - "columns": { - "name": "name", - "type": {"decimal": [2, 3]}, - "option": [ - "not null", - "check": {"lt": [{"length": "name"}, 10]} - ] - } - }} - - **New column format** - - {"create table": { - "columns": { - "name": "name", - "type": {"decimal": [2, 3]} - "nullable": False, - "check": {"lt": [{"length": "name"}, 10]} - } - }} - -### Version 7 - -*October 2021* - -* changed error reporting; still terrible -* upgraded mo-parsing library which forced version change - -### Version 6 - -*October 2021* - -* fixed `SELECT DISTINCT` parsing -* added `DISTINCT ON` parsing - -### Version 5 - -*August 2021* - -* remove inline module `mo-parsing` -* support `CREATE TABLE`, add SQL "flavours" emit `{null:{}}` for None - -### Version 4 - -*November 2021* - -* changed parse result of `SELECT DISTINCT` -* simpler `ORDER BY` clause in window functions - - - - - diff --git a/engine/ast.py b/engine/ast.py index 9e8dca9..0c2d257 100644 --- a/engine/ast.py +++ b/engine/ast.py @@ -1,10 +1,11 @@ +from operator import index from engine.utils import base62uuid - +from copy import copy # replace column info with this later. class ColRef: def __init__(self, cname, _ty, cobj, cnt, table:'TableInfo', name, id, compound = False): - self.cname = cname - self.cxt_name = None + self.cname = cname # column object location + self.cxt_name = None # column object in context self.type = _ty self.cobj = cobj self.cnt = cnt @@ -28,6 +29,7 @@ class ColRef: base_name = f'{base_name}_{counter}' self.cxt_name = base_name cxt.columns_in_context[self] = base_name + # TODO: change this to cname; cxt.emit(f'auto& {base_name} = *(ColRef<{self.type}> *)(&{self.table.cxt_name}->colrefs[{self.id}]);') elif self.cxt_name is None: self.cxt_name = cxt.columns_in_context[self] @@ -44,7 +46,9 @@ class ColRef: self.__arr__[key] = value def __str__(self): - return self.cname + return self.reference() + def __repr__(self): + return self.reference() class TableInfo: @@ -87,6 +91,7 @@ class TableInfo: self.cxt.emit(f'auto& {base_name} = *(TableInfo{type_tags} *)(cxt->tables[{self.table_name}]);') def refer_all(self): + self.reference() for c in self.columns: c.reference() def add_cols(self, cols, new = True): @@ -95,12 +100,13 @@ class TableInfo: def add_col(self, c, new = True, i = 0): _ty = c['type'] if new: - cname =f'{self.table_name}->colrefs[{i}].scast()' + cname =f'get<{i}>({self.table_name})' _ty = _ty if type(c) is ColRef else list(_ty.keys())[0] col_object = ColRef(cname, _ty, c, 1, self,c['name'], len(self.columns)) else: col_object = c cname = c.cname + c.table = self self.cxt.ccols_byname[cname] = col_object self.columns_byname[c['name']] = col_object self.columns.append(col_object) @@ -156,7 +162,7 @@ class TableInfo: self.cxt.tables_byname[alias] = self self.alias.add(alias) - def parse_tablenames(self, colExpr, materialize = True): + def parse_tablenames(self, colExpr, materialize = True, raw = False): self.get_col = self.get_col if materialize else self.get_col_d parsedColExpr = colExpr.split('.') @@ -168,13 +174,12 @@ class TableInfo: if datasource is None: raise ValueError(f'Table name/alias not defined{parsedColExpr[0]}') else: - ret = datasource.get_col(parsedColExpr[1]) - if self.groupinfo is not None and ret: - ret = f"{ret.reference()}[{'start' if ret in self.groupinfo.referenced else 'range'}]" - else: - ret = ret.reference() - return ret - + ret = datasource.parse_tablenames(parsedColExpr[1], raw) + from engine.expr import index_expr + string = ret.reference() + index_expr + if self.groupinfo is not None and ret and ret in self.groupinfo.raw_groups: + string = f'get<{self.groupinfo.raw_groups.index(ret)}>({{y}})' + return string, ret if raw else string class View: def __init__(self, context, table = None, tmp = True): self.table: TableInfo = table @@ -187,7 +192,11 @@ class View: self.context.emit(f'{self.name}:()') class Context: - function_head = 'extern \"C\" int dllmain(Context* cxt){ \n' + function_head = ''' + extern "C" int __DLLEXPORT__ dllmain(Context* cxt) { + using namespace std; + using namespace types; + ''' def __init__(self): self.tables:List[TableInfo] = [] self.tables_byname = dict() @@ -208,7 +217,8 @@ class Context: # and will be deactivated when the `from' is out of scope self.datasource = None self.ds_stack = [] - + self.scans = [] + self.removing_scan = False def add_table(self, table_name, cols): tbl = TableInfo(table_name, cols, self) self.tables.append(tbl) @@ -257,6 +267,10 @@ class Context: return ds else: return None + def remove_scan(self, scan, str_scan): + self.emit(str_scan) + self.scans.remove(scan) + def finalize(self): if not self.finalized: headers = '' @@ -282,8 +296,6 @@ class ast_node: self.context = parent.context if context is None else context self.parent = parent self.datasource = None - for h in self.header: - self.context.headers.add(h) self.init(node) self.produce(node) self.spawn(node) diff --git a/engine/ddl.py b/engine/ddl.py index 04ddc49..b3e9f3a 100644 --- a/engine/ddl.py +++ b/engine/ddl.py @@ -1,21 +1,35 @@ # code-gen for data decl languages -from engine.ast import ColRef, TableInfo, ast_node, include +from engine.ast import ColRef, TableInfo, ast_node, Context, include from engine.utils import base62uuid + class create_table(ast_node): name = 'create_table' + def __init__(self, parent: "ast_node", node, context: Context = None, cexpr = None): + self.cexpr = cexpr + super().__init__(parent, node, context) def produce(self, node): - ct = node[self.name] - tbl = self.context.add_table(ct['name'], ct['columns']) + if type(node) is not TableInfo: + ct = node[self.name] + tbl = self.context.add_table(ct['name'], ct['columns']) + else: + tbl = node + + col_type_str = ','.join([c.type for c in tbl.columns]) # create tables in c - self.emit(f"auto {tbl.table_name} = new TableInfo(\"{tbl.table_name}\", {tbl.n_cols});") + self.emit(f"auto {tbl.table_name} = new TableInfo<{col_type_str}>(\"{tbl.table_name}\", {tbl.n_cols});") self.emit("cxt->tables.insert({\"" + tbl.table_name + f"\", {tbl.table_name}"+"});") self.context.tables_in_context[tbl] = tbl.table_name tbl.cxt_name = tbl.table_name - for i, c in enumerate(ct['columns']): - # TODO: more self awareness - self.emit(f"{tbl.table_name}->colrefs[{i}].ty = types::AINT;") - + tbl.refer_all() + if self.cexpr is None: + for c in tbl.columns: + self.emit(f"{c.cxt_name}.init();") + else: + for i, c in enumerate(tbl.columns): + self.emit(f"{c.cxt_name}.init();") + self.emit(f"{c.cxt_name} = {self.cexpr[i]()};") + class insert(ast_node): name = 'insert' def produce(self, node): @@ -42,18 +56,25 @@ class c(ast_node): class load(ast_node): name="load" def produce(self, node): + self.context.headers.add('"csv.h"') node = node[self.name] table:TableInfo = self.context.tables_byname[node['table']] - n_keys = len(table.columns) - keys = '' - for _ in range(n_keys): - keys+='`tk'+base62uuid(6) - tablename = 'l'+base62uuid(7) - - self.emit(f"{tablename}:({keys}!(+(`csv ? 1:\"{node['file']['literal']}\")))[{keys}]") - + table.refer_all() + csv_reader_name = 'csv_reader_' + base62uuid(6) + col_types = [c.type for c in table.columns] + col_tmp_names = ['tmp_'+base62uuid(8) for _ in range(len(table.columns))] + # col_type_str = ",".join(col_types) + col_names = ','.join([f'"{c.name}"' for c in table.columns]) + + self.emit(f'io::CSVReader<{len(col_types)}> {csv_reader_name}("{node["file"]["literal"]}");') + self.emit(f'{csv_reader_name}.read_header(io::ignore_extra_column, {col_names});') + for t, n in zip(col_types, col_tmp_names): + self.emit(f'{t} {n};') + self.emit(f'while({csv_reader_name}.read_row({",".join(col_tmp_names)})) {{ \n') for i, c in enumerate(table.columns): - self.emit(f'{c.cname}:{tablename}[{i}]') + self.emit(f'{c.cxt_name}.emplace_back({col_tmp_names[i]});') + self.emit('}') + class outfile(ast_node): name="_outfile" diff --git a/engine/expr.py b/engine/expr.py index 4877377..ae9769f 100644 --- a/engine/expr.py +++ b/engine/expr.py @@ -1,5 +1,7 @@ -from engine.ast import ast_node - +from engine.ast import ast_node, ColRef +start_expr = 'f"' +index_expr = '{\'\' if x is None and y is None else f\'[{x}]\'}' +end_expr = '"' class expr(ast_node): name='expr' @@ -8,7 +10,7 @@ class expr(ast_node): 'min': 'min', 'avg': 'avg', 'sum': 'sum', - 'mod':'mod', + 'count' : 'count', 'mins': ['mins', 'minsw'], 'maxs': ['maxs', 'maxsw'], 'avgs': ['avgs', 'avgsw'], @@ -19,29 +21,32 @@ class expr(ast_node): 'sub':'-', 'add':'+', 'mul':'*', - 'div':'%', + 'div':'/', + 'mod':'%', 'and':'&', 'or':'|', + 'xor' : '^', 'gt':'>', 'lt':'<', + 'le':'<=', + 'gt':'>=' } compound_ops = { - 'ge' : [2, lambda x: f'~({x[0]}<{x[1]})'], - 'le' : [2, lambda x: f'~({x[0]}>{x[1]})'], - 'count' : [1, lambda x: f'#({x[0]})'] } unary_ops = { 'neg' : '-', - 'not' : '~' + 'not' : '!' } coumpound_generating_ops = ['mod', 'mins', 'maxs', 'sums'] + \ list( binary_ops.keys()) + list(compound_ops.keys()) + list(unary_ops.keys() ) - def __init__(self, parent, node, materialize_cols = True): + def __init__(self, parent, node, materialize_cols = True, abs_col = False): self.materialize_cols = materialize_cols + self.raw_col = None + self.__abs = abs_col ast_node.__init__(self, parent, node, None) def init(self, _): @@ -54,7 +59,8 @@ class expr(ast_node): else: self.datasource = self.context.datasource self.udf_map = parent.context.udf_map - self.cexpr = '' + self._expr = '' + self.cexpr = None self.func_maps = {**self.udf_map, **self.builtin_func_maps} def produce(self, node): @@ -62,30 +68,31 @@ class expr(ast_node): for key, val in node.items(): if key in self.func_maps: # if type(val) in [dict, str]: + self.context.headers.add('"./server/aggregations.h"') if type(val) is list and len(val) > 1: cfunc = self.func_maps[key] cfunc = cfunc[len(val) - 1] if type(cfunc) is list else cfunc - self.cexpr += f"{cfunc}(" + self._expr += f"{cfunc}(" for i, p in enumerate(val): - self.cexpr += expr(self, p).cexpr + (';'if i {self.group_type};') + self.emit(f'unordered_map<{self.group_type}, vector_type, ' + f'transTypes<{self.group_type}, hasher>> {self.group};') self.n_grps = len(node) - if self.n_grps <= 1: - self.emit(f'{self.group}:={self.group}') - else: - self.emit(f'{self.group}:groupby[({self.group},(,!(#({first_col}))))]') + self.scanner = scan(self, None, expr.toCExpr(first_col)()+'.size') + self.scanner.add(f'{self.group}[forward_as_tuple({g_contents(self.scanner.it_ver)})].emplace_back({self.scanner.it_ver});') + def consume(self, _): self.referenced = self.datasource.rec self.datasource.rec = None - return super().consume(_) + self.scanner.finalize() - def finalize(self, ret, out): - self.groupby_function = 'fgrp'+base62uuid(4) - grp = self.group - if self.n_grps <= 1: - cfn = "{[range] start:*range;"+ ret + "}" - self.emit(f'{out}:(({cfn}\'{grp})[!{grp}])') - self.parent.inv = False - else: - cfn = "{[ids;grps;ll;dim;x] " + \ - "start:grps[x][dim];" + \ - "end:$[x=0;ll;grps[x-1][dim]];" + \ - "range:(end-start)#((start-ll)#ids);" + \ - "start:ids[start];" + \ - ret + '}' - self.emit(f'{self.groupby_function}:{cfn}') - self.emit(f'{out}:+({self.groupby_function}' + \ - f'[{grp}[1];{grp}[0];(#{grp}[0])+1;(#({grp}[0][0]))-1]\'!(#({grp}[0])))') \ No newline at end of file + def finalize(self, cexprs, out:TableInfo): + gscanner = scan(self, self.group) + key_var = 'key_'+base62uuid(7) + val_var = 'val_'+base62uuid(7) + + gscanner.add(f'auto &{key_var} = {gscanner.it_ver}.first;') + gscanner.add(f'auto &{val_var} = {gscanner.it_ver}.second;') + gscanner.add(';\n'.join([f'{out.columns[i].reference()}.emplace_back({ce(x=val_var, y=key_var)})' for i, ce in enumerate(cexprs)])+';') + + gscanner.finalize() \ No newline at end of file diff --git a/engine/projection.py b/engine/projection.py index 311c2d8..088bde0 100644 --- a/engine/projection.py +++ b/engine/projection.py @@ -5,7 +5,7 @@ from engine.expr import expr from engine.orderby import orderby from engine.scan import filter from engine.utils import base62uuid, enlist, base62alp -from engine.ddl import outfile +from engine.ddl import create_table, outfile import copy class projection(ast_node): @@ -75,7 +75,7 @@ class projection(ast_node): if self.group_node is not None: # There is group by; has_groupby = True - cexpr = f'(' + cexprs = [] flatten = False cols = [] self.out_table = TableInfo('out_'+base62uuid(4), [], self.context) @@ -89,27 +89,23 @@ class projection(ast_node): if type(proj) is dict: if 'value' in proj: e = proj['value'] - if type(e) is str: - cname = e # TODO: deal w/ alias - cexpr += (f"{self.datasource.parse_tablenames(proj['value'])}") - elif type(e) is dict: - p_expr = expr(self, e) - cname = p_expr.cexpr - compound = True - cexpr += f"{cname}" - cname = ''.join([a if a in base62alp else '' for a in cname]) - cexpr += ';'if i < len(self.projections)-1 else '' + sname = expr(self, e)._expr + fname = expr.toCExpr(sname) + absname = expr(self, e, abs_col=True)._expr + compound = True + cexprs.append(fname) + cname = ''.join([a if a in base62alp else '' for a in fname()]) compound = compound and has_groupby and self.datasource.rec not in self.group_node.referenced - cols.append(ColRef(f'{disp_varname}[{i}]', 'generic', self.out_table, 0, None, cname, i, compound=compound)) + cols.append(ColRef(cname, expr.toCExpr(f'decays')(0), self.out_table, 0, None, cname, i, compound=compound)) self.out_table.add_cols(cols, False) - cexpr += ')' if has_groupby: - self.group_node.finalize(cexpr, disp_varname) + create_table(self, self.out_table) + self.group_node.finalize(cexprs, self.out_table) else: - self.emit(f'auto {disp_varname} = {cexpr};') + create_table(self, self.out_table, cexpr = cexprs) self.datasource.group_node = None has_orderby = 'orderby' in node @@ -122,12 +118,8 @@ class projection(ast_node): self.emit_no_ln(f"{f'{disp_varname}:+' if flatten else ''}(") if self.disp or has_orderby: - if len(self.projections) > 1: - self.emit_no_ln(f"{'+' if self.inv else ''}{disp_varname}") - else: - self.emit_no_ln(f'print({disp_varname});') - if flatten: - self.emit_no_ln(f'{disp_varname}') + self.emit(f'print(*{self.out_table.cxt_name});') + if has_orderby: self.emit(f')[{orderby_node.view}]') else: diff --git a/engine/scan.py b/engine/scan.py index 1042d6b..f567829 100644 --- a/engine/scan.py +++ b/engine/scan.py @@ -1,11 +1,41 @@ from xmlrpc.client import Boolean -from engine.ast import ColRef, TableInfo, View, ast_node +from engine.ast import ColRef, TableInfo, View, ast_node, Context from engine.utils import base62uuid from engine.expr import expr class scan(ast_node): name = 'scan' - + def __init__(self, parent: "ast_node", node, size = None, context: Context = None): + self.type = type + self.size = size + super().__init__(parent, node, context) + def init(self, _): + self.datasource = self.context.datasource + self.start = '' + self.body = '' + self.end = '}' + self.filter = None + scan_vars = set(s.it_var for s in self.context.scans) + self.it_ver = 'i' + base62uuid(2) + while(self.it_ver in scan_vars): + self.it_ver = 'i' + base62uuid(6) + self.parent.context.scans.append(self) + def produce(self, node): + if type(node) is ColRef: + if self.size is None: + self.start += f'for (auto& {self.it_ver} : {node.reference()}) {{\n' + else: + self.start += f"for (uint32_t {self.it_ver} = 0; {self.it_ver} < {node.reference()}.size; ++{self.it_ver}){{\\n" + elif type(node) is str: + self.start+= f'for(auto& {self.it_ver} : {node}) {{\n' + else: + self.start += f"for (uint32_t {self.it_ver} = 0; {self.it_ver} < {self.size}; ++{self.it_ver}){{\n" + + def add(self, stmt): + self.body+=stmt + '\n' + def finalize(self): + self.context.remove_scan(self, self.start + self.body + self.end) + class filter(ast_node): name = 'filter' def __init__(self, parent: "ast_node", node, materialize = False, context = None): diff --git a/msc-plugin/msc-plugin.vcxproj b/msc-plugin/msc-plugin.vcxproj new file mode 100644 index 0000000..78d05a4 --- /dev/null +++ b/msc-plugin/msc-plugin.vcxproj @@ -0,0 +1,168 @@ + + + + + Debug + Win32 + + + Release + Win32 + + + Debug + x64 + + + Release + x64 + + + + 16.0 + Win32Proj + {8081fdaa-4d13-4b7a-adb2-8224af7f1c81} + Project1 + 10.0 + msc-plugin + + + + DynamicLibrary + true + v143 + Unicode + + + DynamicLibrary + false + v143 + true + Unicode + + + DynamicLibrary + true + v143 + Unicode + + + DynamicLibrary + false + v143 + true + Unicode + + + + + + + + + + + + + + + + + + + + + + dll.so + + + + dll.so + + + + dll.so + + + + dll.so + + + + Level3 + true + _CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) + true + + + Console + true + $(ProjectDir)\..\dll.so + + + + + Level3 + true + true + true + _CRT_SECURE_NO_WARNINGS;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + true + + + Console + true + true + true + $(ProjectDir)\..\dll.so + + + + + Level3 + true + _CRT_SECURE_NO_WARNINGS;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) + true + + + Console + true + $(ProjectDir)\..\dll.so + + + + + Level3 + true + true + true + _CRT_SECURE_NO_WARNINGS;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + true + + + Console + true + true + true + $(ProjectDir)\..\dll.so + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/out.cpp b/out.cpp index 137e13f..6c5b5f5 100644 --- a/out.cpp +++ b/out.cpp @@ -1,11 +1,15 @@ #include "./server/libaquery.h" -extern "C" int dllmain(Context* cxt){ -auto stocks = new TableInfo("stocks", 2); +#include "./server/aggregations.h" + + extern "C" int __DLLEXPORT__ dllmain(Context* cxt) { + using namespace std; + using namespace types; + auto stocks = new TableInfo("stocks", 2); cxt->tables.insert({"stocks", stocks}); -stocks->colrefs[0].ty = types::AINT; -stocks->colrefs[1].ty = types::AINT; auto& stocks_timestamp = *(ColRef *)(&stocks->colrefs[0]); auto& stocks_price = *(ColRef *)(&stocks->colrefs[1]); +stocks_timestamp.init(); +stocks_price.init(); stocks_timestamp.emplace_back(1); stocks_price.emplace_back(15); stocks_timestamp.emplace_back(2); @@ -38,9 +42,19 @@ stocks_timestamp.emplace_back(15); stocks_price.emplace_back(2); stocks_timestamp.emplace_back(16); stocks_price.emplace_back(5); -auto d2PxTIVW = (max((stocks_price-min(stocks_timestamp)))); -print(d2PxTIVW); -auto d2dVVnjL = (max((stocks_price-mins(stocks_price)))); -print(d2dVVnjL); +auto out_6gPn = new TableInfo>("out_6gPn", 1); +cxt->tables.insert({"out_6gPn", out_6gPn}); +auto& out_6gPn_maxstockspriceminstockstimestamp = *(ColRef> *)(&out_6gPn->colrefs[0]); +out_6gPn_maxstockspriceminstockstimestamp.init(); +out_6gPn_maxstockspriceminstockstimestamp = max((stocks_price-min(stocks_timestamp))); +print(*out_6gPn); + +auto out_7a2d = new TableInfo>("out_7a2d", 1); +cxt->tables.insert({"out_7a2d", out_7a2d}); +auto& out_7a2d_maxstockspriceminsstocksprice = *(ColRef> *)(&out_7a2d->colrefs[0]); +out_7a2d_maxstockspriceminsstocksprice.init(); +out_7a2d_maxstockspriceminsstocksprice = max((stocks_price-mins(stocks_price))); +print(*out_7a2d); + return 0; } \ No newline at end of file diff --git a/prompt.py b/prompt.py index 9e6b00c..84607ae 100644 --- a/prompt.py +++ b/prompt.py @@ -1,3 +1,4 @@ +from concurrent.futures import thread import re import time import aquery_parser as parser @@ -8,12 +9,35 @@ import sys import os from engine.utils import base62uuid import atexit +try: + os.remove('server.bin') +except Exception as e: + print(type(e), e) +subprocess.call(['make', 'server.bin']) +cleanup = True def rm(): - files = os.listdir('.') - for f in files: - if f.endswith('.shm'): - os.remove(f) + if cleanup: + mm.seek(0,os.SEEK_SET) + mm.write(b'\x00\x00') + mm.flush() + time.sleep(.001) + server.kill() + files = os.listdir('.') + for f in files: + if f.endswith('.shm'): + os.remove(f) + mm.close() + cleanup = False + +def proc_alive(pid): + try: + os.kill(pid, 0) + except OSError: + return False + else: + return True + atexit.register(rm) shm = base62uuid() @@ -35,7 +59,8 @@ else: mm = mmap.mmap(0, 2, shm) mm.write(b'\x01\x00') mm.flush() -subprocess.Popen(["./server.bin", shm]) +server = subprocess.Popen(["./server.bin", shm]) + test_parser = True # code to test parser @@ -64,6 +89,8 @@ keep = False cxt = None while test_parser: try: + if not proc_alive(server.pid): + server = subprocess.Popen(["./server.bin", shm]) q = input() if q == 'exec': if not keep or cxt is None: @@ -90,11 +117,13 @@ while test_parser: elif q == 'keep': keep = not keep continue + elif q=='format' or q == 'fmt': + subprocess.call(['clang-format', 'out.cpp']) elif q == 'exit': break elif q == 'r': if subprocess.call(['make', 'snippet']) == 0: - mm.seek(0,os.SEEK_SET) + mm.seek(0,os.SEEK_SET) mm.write(b'\x01\x01') continue trimed = ws.sub(' ', q.lower()).split(' ') @@ -108,9 +137,8 @@ while test_parser: continue stmts = parser.parse(q) print(stmts) - except (ValueError) as e: + except (ValueError, FileNotFoundError) as e: + rm() print(type(e), e) -mm.close() -handle.close() -os.remove(shm) \ No newline at end of file +rm() \ No newline at end of file diff --git a/q1.sql b/q1.sql index 2acaf11..bb55cd6 100644 --- a/q1.sql +++ b/q1.sql @@ -7,4 +7,4 @@ FIELDS TERMINATED BY "," SELECT sum(c), b, d FROM test group by a,b,d -order by d DESC, b ASC +-- order by d DESC, b ASC diff --git a/server/aggregations.h b/server/aggregations.h new file mode 100644 index 0000000..71de8aa --- /dev/null +++ b/server/aggregations.h @@ -0,0 +1,106 @@ +#pragma once +#include "types.h" +#include +#include +#include +template class VT> +size_t count(const VT& v) { + return v.size; +} + +template +constexpr static inline size_t count(const T&) { return 1; } + +// TODO: Specializations for dt/str/none +template class VT> +types::GetLongType sum(const VT& v) { + types::GetLongType ret = 0; + for (const auto& _v : v) + ret += _v; + return ret; +} +template class VT> +types::GetFPType avg(const VT& v) { + return static_cast>( + sum(v) / static_cast(v.size)); +} +template class VT> +T max(const VT& v) { + T max_v = std::numeric_limits::min(); + for (const auto& _v : v) + max_v = max_v > _v ? max_v : _v; + return max_v; +} +template class VT> +T min(const VT& v) { + T min_v = std::numeric_limits::max(); + for (const auto& _v : v) + min_v = min_v < _v ? min_v : _v; + return min_v; +} +template class VT> +VT mins(const VT& arr) { + const int& len = arr.size; + std::deque> cache; + VT ret(len); + T min = std::numeric_limits::max(); + for (int i = 0; i < len; ++i) { + if (arr[i] < min) + min = arr[i]; + ret[i] = min; + } + return ret; +} +template class VT> +VT maxs(const VT& arr) { + const int& len = arr.size; + VT ret(len); + T max = std::numeric_limits::min(); + for (int i = 0; i < len; ++i) { + if (arr[i] > max) + max = arr[i]; + ret[i] = max; + } + return ret; +} + +template class VT> +VT minw(const VT& arr, uint32_t w) { + const int& len = arr.size; + VT ret(len); + std::deque> cache; + for (int i = 0; i < len; ++i) { + if (!cache.empty() && cache.front().second == i - w) cache.pop_front(); + while (!cache.empty() && cache.back().first > arr[i]) cache.pop_back(); + cache.push_back({ arr[i], i }); + ret[i] = cache.front().first; + } + return ret; +} + +template class VT> +VT maxw(const VT& arr, uint32_t w) { + const int& len = arr.size; + VT ret(len); + std::deque> cache; + for (int i = 0; i < len; ++i) { + if (!cache.empty() && cache.front().second == i - w) cache.pop_front(); + while (!cache.empty() && cache.back().first > arr[i]) cache.pop_back(); + cache.push_back({ arr[i], i }); + arr[i] = cache.front().first; + } + return ret; +} + +template constexpr inline T max(const T& v) { return v; } +template constexpr inline T min(const T& v) { return v; } +template constexpr inline T avg(const T& v) { return v; } +template constexpr inline T sum(const T& v) { return v; } +template constexpr inline T maxw(const T& v, uint32_t) { return v; } +template constexpr inline T minw(const T& v, uint32_t) { return v; } +template constexpr inline T avgw(const T& v, uint32_t) { return v; } +template constexpr inline T sumw(const T& v, uint32_t) { return v; } +template constexpr inline T maxs(const T& v) { return v; } +template constexpr inline T mins(const T& v) { return v; } +template constexpr inline T avgs(const T& v) { return v; } +template constexpr inline T sums(const T& v) { return v; } diff --git a/server/hasher.h b/server/hasher.h new file mode 100644 index 0000000..780d24c --- /dev/null +++ b/server/hasher.h @@ -0,0 +1,20 @@ +#pragma once + +#include +#include +template +struct hasher { + template typename std::enable_if< i == sizeof...(Types), + size_t>::type hashi(const std::tuple& record) const { + return 0; + } + + template typename std::enable_if< i < sizeof ...(Types), + size_t>::type hashi(const std::tuple& record) const { + using current_type = typename std::decay>::type>::type; + return std::hash()(std::get(record)) ^ hashi(record); + } + size_t operator()(const std::tuple& record) const { + return hashi(record); + } +}; diff --git a/server/libaquery.h b/server/libaquery.h index 9369ac9..3288a03 100644 --- a/server/libaquery.h +++ b/server/libaquery.h @@ -3,8 +3,16 @@ #include "table.h" #include + struct Context{ std::unordered_map tables; std::unordered_map cols; }; + +#ifdef _MSC_VER +#define __DLLEXPORT__ __declspec(dllexport) __stdcall +#else +#define __DLLEXPORT__ +#endif + #endif \ No newline at end of file diff --git a/server/packages.config b/server/packages.config deleted file mode 100644 index acb6d36..0000000 --- a/server/packages.config +++ /dev/null @@ -1,4 +0,0 @@ - - - - \ No newline at end of file diff --git a/server/server.cpp b/server/server.cpp index 4ff2501..5b40ab8 100644 --- a/server/server.cpp +++ b/server/server.cpp @@ -3,6 +3,7 @@ #include //#include #include + #include "libaquery.h" #ifdef _WIN32 #include "winhelper.h" @@ -32,6 +33,7 @@ struct thread_context{ void daemon(thread_context* c) { } +#include "aggregations.h" typedef int (*code_snippet)(void*); int _main(); int main(int argc, char** argv) { @@ -51,7 +53,7 @@ int main(int argc, char** argv) { printf("running: %s\n", running? "true":"false"); printf("ready: %s\n", ready? "true":"false"); while (running) { - std::this_thread::sleep_for(1us); + std::this_thread::sleep_for(100us); if(ready){ printf("running: %s\n", running? "true":"false"); printf("ready: %s\n", ready? "true":"false"); @@ -75,72 +77,26 @@ int main(int argc, char** argv) { } int _main() { - using string = std::string; - using std::cout; - using std::thread; - void* handle = dlopen("dll.so", RTLD_LAZY); - io::CSVReader<4> in("../test.csv"); - in.read_header(io::ignore_extra_column, "a", "b", "c", "d"); - int a, b, cc, d; - string tp, tpdf, st; - while (in.read_row(a, b, cc, d)) - cout << a<< ' ' << b <<' ' << cc <<' ' << d << '\n'; - - //code_snippet c = reinterpret_cast(dlsym(handle, "dlmain")); - //printf("%d", c(0)); - //dlclose(handle); - vector_type t1{ 1.2, 3.4, .2, 1e-5, 1, 3, 5, 4, 5}; - vector_type t2{ 2, 4, 3, 5, 0, 2, 6, 1, 2}; - printf("x: "); - const auto& ret = t1 - t2; - for (const auto& x : ret) { - printf("%lf ", x); - } - puts(""); + //vector_type t; + //t = 1; + //t.emplace_back(2); + //print(t); + //return 0; + void* handle = dlopen("dll.so", RTLD_LAZY); + printf("handle: %x\n", handle); Context* cxt = new Context(); - auto stocks = TableInfo("stocks", 2); - cxt->tables.insert(std::make_pair("stocks", &stocks)); - - auto& stocks_0 = get<0>(stocks); - auto& stocks_1 = get<1>(stocks); - - stocks_0.init(); - stocks_1.init(); + if (handle) { + printf("inner\n"); + code_snippet c = reinterpret_cast(dlsym(handle, "dllmain")); + printf("routine: %x\n", c); + if (c) { + printf("inner\n"); + printf("return: %d\n", c(cxt)); + } + dlclose(handle); + } - stocks_0.emplace_back(1); - stocks_1.emplace_back(15); - stocks_0.emplace_back(2); - stocks_1.emplace_back(19); - stocks_0.emplace_back(3); - stocks_1.emplace_back(16); - stocks_0.emplace_back(4); - stocks_1.emplace_back(17); - stocks_0.emplace_back(5); - stocks_1.emplace_back(15); - stocks_0.emplace_back(6); - stocks_1.emplace_back(13); - stocks_0.emplace_back(7); - stocks_1.emplace_back(5); - stocks_0.emplace_back(8); - stocks_1.emplace_back(8); - stocks_0.emplace_back(9); - stocks_1.emplace_back(7); - stocks_0.emplace_back(10); - stocks_1.emplace_back(13); - stocks_0.emplace_back(11); - stocks_1.emplace_back(11); - stocks_0.emplace_back(12); - stocks_1.emplace_back(14); - stocks_0.emplace_back(13); - stocks_1.emplace_back(10); - stocks_0.emplace_back(14); - stocks_1.emplace_back(5); - stocks_0.emplace_back(15); - stocks_1.emplace_back(2); - stocks_0.emplace_back(16); - stocks_1.emplace_back(5); - printf("%d\n", max(stocks_0 - mins(stocks_1))); return 0; } diff --git a/server/server.sln b/server/server.sln index 0da24c2..e3b37e3 100644 --- a/server/server.sln +++ b/server/server.sln @@ -4,6 +4,11 @@ Microsoft Visual Studio Solution File, Format Version 12.00 VisualStudioVersion = 17.2.32210.308 MinimumVisualStudioVersion = 10.0.40219.1 Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "server", "server.vcxproj", "{031352C2-AFBB-45AA-9518-DBC1F9EF2AF3}" + ProjectSection(ProjectDependencies) = postProject + {8081FDAA-4D13-4B7A-ADB2-8224AF7F1C81} = {8081FDAA-4D13-4B7A-ADB2-8224AF7F1C81} + EndProjectSection +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "Project1", "..\Project1\Project1.vcxproj", "{8081FDAA-4D13-4B7A-ADB2-8224AF7F1C81}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -21,6 +26,14 @@ Global {031352C2-AFBB-45AA-9518-DBC1F9EF2AF3}.Release|x64.Build.0 = Release|x64 {031352C2-AFBB-45AA-9518-DBC1F9EF2AF3}.Release|x86.ActiveCfg = Release|Win32 {031352C2-AFBB-45AA-9518-DBC1F9EF2AF3}.Release|x86.Build.0 = Release|Win32 + {8081FDAA-4D13-4B7A-ADB2-8224AF7F1C81}.Debug|x64.ActiveCfg = Debug|x64 + {8081FDAA-4D13-4B7A-ADB2-8224AF7F1C81}.Debug|x64.Build.0 = Debug|x64 + {8081FDAA-4D13-4B7A-ADB2-8224AF7F1C81}.Debug|x86.ActiveCfg = Debug|Win32 + {8081FDAA-4D13-4B7A-ADB2-8224AF7F1C81}.Debug|x86.Build.0 = Debug|Win32 + {8081FDAA-4D13-4B7A-ADB2-8224AF7F1C81}.Release|x64.ActiveCfg = Release|x64 + {8081FDAA-4D13-4B7A-ADB2-8224AF7F1C81}.Release|x64.Build.0 = Release|x64 + {8081FDAA-4D13-4B7A-ADB2-8224AF7F1C81}.Release|x86.ActiveCfg = Release|Win32 + {8081FDAA-4D13-4B7A-ADB2-8224AF7F1C81}.Release|x86.Build.0 = Release|Win32 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/server/server.vcxproj b/server/server.vcxproj index 92ea8c3..221f1dd 100644 --- a/server/server.vcxproj +++ b/server/server.vcxproj @@ -1,6 +1,5 @@  - Debug @@ -90,6 +89,8 @@ _CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) true stdcpp20 + true + true Console @@ -105,6 +106,8 @@ _CRT_SECURE_NO_WARNINGS;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) true stdcpp20 + true + true Console @@ -120,6 +123,8 @@ _CRT_SECURE_NO_WARNINGS;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) true stdcpp20 + true + true Console @@ -135,6 +140,8 @@ _CRT_SECURE_NO_WARNINGS;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) true stdcpp20 + true + true Console @@ -144,35 +151,34 @@ + + - + + + + + - + + - - - - - - - + This project references NuGet package(s) that are missing on this computer. Use NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. - - \ No newline at end of file diff --git a/server/table.h b/server/table.h index 5e8fd88..44a8448 100644 --- a/server/table.h +++ b/server/table.h @@ -3,7 +3,7 @@ #include "types.h" #include "vector_type.hpp" - +#include template class vector_type; template <> @@ -18,22 +18,64 @@ namespace types { struct Coercion; } #endif - +template +class ColView; template class ColRef : public vector_type<_Ty> { public: const char* name; types::Type_t ty = types::ERROR; - ColRef(uint32_t size, const char* name = "") : vector_type<_Ty>(size), name(name) {} + ColRef(const uint32_t& size, const char* name = "") : vector_type<_Ty>(size), name(name) {} ColRef(const char* name) : name(name) {} void init() { ty = types::Types<_Ty>::getType(); this->size = this->capacity = 0; this->container = 0; } ColRef(const char* name, types::Type_t ty) : name(name), ty(ty) {} - + using vector_type<_Ty>::operator[]; + using vector_type<_Ty>::operator=; + ColView<_Ty> operator [](const vector_type&idxs) const { + return ColView<_Ty>(*this, idxs); + } template ColRef scast(); }; +template +class ColView { +public: + const vector_type& idxs; + const ColRef<_Ty>& orig; + const uint32_t& size; + ColView(const ColRef<_Ty>& orig, const vector_type& idxs) : orig(orig), idxs(idxs), size(idxs.size) {} + ColView(const ColView<_Ty>& orig, const vector_type& idxs) : orig(orig.orig), idxs(idxs), size(idxs.size) { + for (uint32_t i = 0; i < size; ++i) + idxs[i] = orig.idxs[idxs[i]]; + } + _Ty& operator [](const uint32_t& i) const { + return orig[idxs[i]]; + } + struct Iterator_t { + const uint32_t* val; + const ColRef<_Ty>& orig; + constexpr Iterator_t(const uint32_t* val, const ColRef<_Ty>& orig) noexcept : val(val), orig(orig) {} + _Ty& operator*() { return orig[*val]; } + bool operator != (const Iterator_t& rhs) { return rhs.val != val; } + Iterator_t& operator++ () { + ++val; + return *this; + } + Iterator_t operator++ (int) { + Iterator_t tmp = *this; + ++val; + return tmp; + } + }; + Iterator_t begin() const { + return Iterator_t(idxs.begin(), orig); + } + Iterator_t end() const { + return Iterator_t(idxs.end(), orig); + } +}; template template @@ -47,10 +89,27 @@ template struct TableInfo { const char* name; ColRef* colrefs; - uint32_t n_cols, n_rows; - //void print(); + uint32_t n_cols; + void print(const char* __restrict sep, const char* __restrict end) const; + typedef std::tuple tuple_type; + template + typename std::enable_if::type print_impl(const uint32_t& i, const char* __restrict sep = " ") const; + template + typename std::enable_if::type print_impl(const uint32_t& i, const char* __restrict sep = " ") const; + template + struct GetTypes { + typedef typename std::tuple::type ...> type; + }; + template + using getRecordType = typename GetTypes::type; TableInfo(const char* name, uint32_t n_cols); }; + +template +constexpr inline ColRef>>& get(const TableInfo<_Types...>& table) noexcept { + return *(ColRef>> *) & (table.colrefs[_Index]); +} + template constexpr static inline bool is_vector(const ColRef&) { return true; @@ -68,100 +127,110 @@ TableInfo::TableInfo(const char* name, uint32_t n_cols) : name(name), this->colrefs = (ColRef*)malloc(sizeof(ColRef) * n_cols); } -template -ColRef::type> operator -(const ColRef& lhs, const ColRef& rhs) { - auto ret = ColRef::type>(lhs.size, ""); +template +template +inline typename std::enable_if::type + TableInfo::print_impl(const uint32_t& i, const char* __restrict sep) const { + std::cout << (get(*this))[i]; +} + +template +template +inline typename std::enable_if::type + TableInfo::print_impl(const uint32_t& i, const char* __restrict sep) const +{ + std::cout << (get(*this))[i] << sep; + print_impl(i, sep); +} + +template +inline void TableInfo::print(const char* __restrict sep, const char* __restrict end) const { + int n_rows = 0; + if (n_cols > 0 && colrefs[0].size > 0) + n_rows = colrefs[0].size; + for (int i = 0; i < n_rows; ++i) { + print_impl(i); + std::cout << end; + } +} +template class VT> +VT::type> operator -(const VT& lhs, const VT& rhs) { + auto ret = VT::type>(lhs.size, ""); for (int i = 0; i < lhs.size; ++i) ret.container[i] = lhs.container[i] - rhs.container[i]; return ret; } -template -ColRef::type> operator -(const ColRef& lhs, const T2& rhs) { - auto ret = ColRef::type>(lhs.size, ""); +template class VT> +VT::type> operator -(const VT& lhs, const T2& rhs) { + auto ret = VT::type>(lhs.size, ""); for (int i = 0; i < lhs.size; ++i) ret.container[i] = lhs.container[i] - rhs; return ret; } - -template -constexpr ColRef>>& get(TableInfo<_Types...>& table) noexcept { - return *(ColRef< std::tuple_element_t<_Index, std::tuple<_Types...>>> *) &(table.colrefs[_Index]); +template class VT> +VT::type> operator +(const VT& lhs, const VT& rhs) { + auto ret = VT::type>(lhs.size, ""); + for (int i = 0; i < lhs.size; ++i) + ret.container[i] = lhs.container[i] + rhs.container[i]; + return ret; } - - -//void TableInfo::print() -//{ -// for (int i = 0; i < n_rows; ++i) { -// for (int j = 0; j < n_cols; ++j) // TODO: Deal with date/time -// printf(types::printf_str[colrefs[j].ty], colrefs[j].get(i, colrefs[j].ty)); -// puts(""); -// } -//} -template -ColRef mins(const ColRef& arr) { - const int& len = arr.size; - std::deque> cache; - ColRef ret(len); - T min = std::numeric_limits::max(); - for (int i = 0; i < len; ++i) { - if (arr[i] < min) - min = arr[i]; - ret[i] = min; - } +template class VT> +VT::type> operator +(const VT& lhs, const T2& rhs) { + auto ret = VT::type>(lhs.size, ""); + for (int i = 0; i < lhs.size; ++i) + ret.container[i] = lhs.container[i] + rhs; return ret; } -template -ColRef maxs(const ColRef& arr) { - const int& len = arr.size; - ColRef ret(len); - T max = std::numeric_limits::min(); - for (int i = 0; i < len; ++i) { - if (arr[i] > max) - max = arr[i]; - ret[i] = max; - } +template class VT> +VT::type> operator *(const VT& lhs, const VT& rhs) { + auto ret = VT::type>(lhs.size, ""); + for (int i = 0; i < lhs.size; ++i) + ret.container[i] = lhs.container[i] * rhs.container[i]; return ret; } - -template -ColRef minw(const ColRef& arr, uint32_t w) { - const int& len = arr.size; - ColRef ret(len); - std::deque> cache; - for (int i = 0; i < len; ++i) { - if (!cache.empty() && cache.front().second == i - w) cache.pop_front(); - while (!cache.empty() && cache.back().first > arr[i]) cache.pop_back(); - cache.push_back({ arr[i], i }); - ret[i] = cache.front().first; - } +template class VT> +VT::type> operator *(const VT& lhs, const T2& rhs) { + auto ret = VT::type>(lhs.size, ""); + for (int i = 0; i < lhs.size; ++i) + ret.container[i] = lhs.container[i] * rhs; return ret; } - -template -ColRef maxw(const ColRef& arr, uint32_t w) { - const int& len = arr.size; - ColRef ret(len); - std::deque> cache; - for (int i = 0; i < len; ++i) { - if (!cache.empty() && cache.front().second == i - w) cache.pop_front(); - while (!cache.empty() && cache.back().first > arr[i]) cache.pop_back(); - cache.push_back({ arr[i], i }); - arr[i] = cache.front().first; - } +template class VT> +VT::type> operator /(const VT& lhs, const VT& rhs) { + auto ret = VT::type>(lhs.size, ""); + for (int i = 0; i < lhs.size; ++i) + ret.container[i] = lhs.container[i] / rhs.container[i]; + return ret; +} +template class VT> +VT::type> operator /(const VT& lhs, const T2& rhs) { + auto ret = VT::type>(lhs.size, ""); + for (int i = 0; i < lhs.size; ++i) + ret.container[i] = lhs.container[i] / rhs; return ret; } +template +void print(const TableInfo& v, const char* delimiter = " ", const char* endline = "\n") { + v.print(delimiter, endline); +} template -void print(const T& v) { +void print(const T& v, const char* delimiter = " ") { printf(types::printf_str[types::Types::getType()], v); } - template -void print(const ColRef& v, const char* delimiter) { - +void inline print_impl(const T& v, const char* delimiter, const char* endline) { for (const auto& vi : v) { print(vi); - puts(delimiter); + printf("%s", delimiter); } + printf("%s", endline); +} + +template class VT> +typename std::enable_if, TableInfo>::value>::type +print(const VT& v, const char* delimiter = " ", const char* endline = "\n") { + print_impl(v, delimiter, endline); } + #endif \ No newline at end of file diff --git a/server/types.h b/server/types.h index b4ed73c..6d6c4b9 100644 --- a/server/types.h +++ b/server/types.h @@ -4,7 +4,8 @@ #include #include #include -#include +#include + #ifdef _MSC_VER #define __restrict__ __restrict #endif @@ -72,13 +73,51 @@ namespace types { }; #define __Eq(x) (sizeof(T) == sizeof(x)) template - struct GetFPType { + struct GetFPTypeImpl { using type = Cond(__Eq(float), float, Cond(__Eq(double), double, long double)); }; template - struct GetLongType - { + using GetFPType = typename GetFPTypeImpl::type; + template + struct GetLongTypeImpl { using type = Cond(_U(T), unsigned long long, Cond(Fp(T), long double, long long)); }; + template + using GetLongType = typename GetLongTypeImpl::type; } + +#define getT(i, t) std::tuple_element_t> +template class T, typename ...Types> +struct applyTemplates { + using type = T; +}; + +template class rT> +struct transTypes_s; +template class lT, typename ...T, template class rT> +struct transTypes_s, rT> { + using type = rT; +}; + +// static_assert(std::is_same, std::unordered_map>, std::unordered_map>::value); +template class rT> +using transTypes = typename transTypes_s::type; + +template +struct record_types {}; + +template +using record = std::tuple; + +template +struct decayS { + using type = typename std::decay::type; +}; +template class T, typename ...Types> +struct decayS >{ + using type = T::type ...>; +}; +template +using decays = typename decayS::type; + #endif // !_TYPES_H diff --git a/server/utils.cpp b/server/utils.cpp new file mode 100644 index 0000000..5ac8bc1 --- /dev/null +++ b/server/utils.cpp @@ -0,0 +1,18 @@ +#include "utils.h" +#include +#include +using std::string; +string base62uuid(int l = 8) { + using namespace std; + constexpr static const char* base62alp = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"; + static mt19937_64 engine(chrono::system_clock::now().time_since_epoch().count()); + static uniform_int_distribution u(0x10000, 0xfffff); + uint64_t uuid = (u(engine) << 32ull) + (chrono::system_clock::now().time_since_epoch().count() & 0xffffffff); + printf("%llx\n", uuid); + string ret; + while (uuid && l-- >= 0) { + ret = string("") + base62alp[uuid % 62] + ret; + uuid /= 62; + } + return ret; +} \ No newline at end of file diff --git a/server/utils.h b/server/utils.h index 8a53238..0e1a4ef 100644 --- a/server/utils.h +++ b/server/utils.h @@ -1,18 +1,18 @@ #pragma once #include #include -#include -using std::string; -string base62uuid(int l = 8) { - constexpr static const char* base62alp = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"; - static mt19937_64 engine(chrono::system_clock::now().time_since_epoch().count()); - static uniform_int_distribution u(0x100000000, 0xfffffffff); - uint64_t uuid = (u(engine)<<16ull) + (time(0)&0xffff); - printf("%lx\n", uuid); - string ret; - while (uuid && l-- >= 0){ - ret = string("") + base62alp[uuid % 62] + ret; - uuid /= 62; - } - return ret; -} \ No newline at end of file + +template +struct const_range { + int arr[cnt]; + constexpr const_range() { + for (int i = begin, n = 0; n < cnt; ++n, i += interval) + arr[n] = i; + } + const int* begin() const { + return arr; + } + const int* end() const { + return arr + cnt; + } +}; \ No newline at end of file diff --git a/server/vector_type.hpp b/server/vector_type.hpp index a307c4e..05fe03c 100644 --- a/server/vector_type.hpp +++ b/server/vector_type.hpp @@ -41,7 +41,7 @@ public: _Ty* container; uint32_t size, capacity; typedef _Ty* iterator_t; - vector_type(uint32_t size) : size(size), capacity(size) { + vector_type(const uint32_t& size) : size(size), capacity(size) { container = (_Ty*)malloc(size * sizeof(_Ty)); } constexpr vector_type(std::initializer_list<_Ty> _l) { @@ -51,13 +51,22 @@ public: *(_container++) = l; } } - constexpr vector_type() noexcept {}; + constexpr vector_type() noexcept : size(0), capacity(0), container(0) {}; constexpr vector_type(vector_type<_Ty>& vt) noexcept { _copy(vt); } constexpr vector_type(vector_type<_Ty>&& vt) noexcept { _move(std::move(vt)); } + vector_type<_Ty> operator =(const _Ty& vt) { + if (!container) { + container = (_Ty*)malloc(sizeof(_Ty)); + capacity = 1; + } + size = 1; + container[0] = vt; + return *this; + } vector_type<_Ty> operator =(vector_type<_Ty>& vt) { _copy(vt); return *this; @@ -102,7 +111,7 @@ public: return curr; } - _Ty& operator[](const std::size_t _i) const { + _Ty& operator[](const uint32_t _i) const { return container[_i]; } @@ -240,42 +249,11 @@ public: } constexpr vector_type() : size(0), capacity(0), container(0) {}; void *get(uint32_t i, types::Type_t atype){ - return static_cast(static_cast(container) + (i * types::AType_sizes[atype])); } + void operator[](const uint32_t& i) { + + } }; -#pragma pack(pop) - - - - -// TODO: Specializations for dt/str/none -template -types::GetLongType sum(const vector_type &v) { - types::GetLongType ret = 0; - for (const auto& _v : v) - ret += _v; - return ret; -} -template -types::GetFPType avg(const vector_type &v) { - return static_cast>( - sum(v)/static_cast(v.size)); -} -template -T max(const vector_type& v) { - T max_v = std::numeric_limits::min(); - for (const auto& _v : v) - max_v = max_v > _v ? max_v : _v; - return max_v; -} -template -T min(const vector_type& v) { - T min_v = std::numeric_limits::max(); - for (const auto& _v : v) - min_v = min_v < _v ? min_v : _v; - return min_v; -} - #endif diff --git a/server/winhelper.cpp b/server/winhelper.cpp index 1ed27c7..fd534bf 100644 --- a/server/winhelper.cpp +++ b/server/winhelper.cpp @@ -9,7 +9,7 @@ void* dlopen(const char* lib, int) void* dlsym(void* handle, const char* proc) { - return GetProcAddress(static_cast(handle), proc); + return reinterpret_cast(GetProcAddress(static_cast(handle), proc)); } int dlclose(void* handle)