update structure, vis application
This commit is contained in:
@@ -1,37 +0,0 @@
|
||||
from engine.ast import Context, ast_node
|
||||
saved_cxt = None
|
||||
|
||||
def initialize(cxt = None, keep = False):
|
||||
global saved_cxt
|
||||
if cxt is None or not keep or type(cxt) is not Context:
|
||||
if saved_cxt is None or not keep:
|
||||
cxt = Context()
|
||||
saved_cxt = cxt
|
||||
else:
|
||||
cxt = saved_cxt
|
||||
cxt.new()
|
||||
|
||||
return cxt
|
||||
|
||||
def generate(ast, cxt):
|
||||
for k in ast.keys():
|
||||
if k in ast_node.types.keys():
|
||||
root = ast_node.types[k](None, ast, cxt)
|
||||
|
||||
def exec(stmts, cxt = None, keep = None):
|
||||
cxt = initialize(cxt, keep)
|
||||
stmts_stmts = stmts['stmts']
|
||||
if type(stmts_stmts) is list:
|
||||
for s in stmts_stmts:
|
||||
generate(s, cxt)
|
||||
else:
|
||||
generate(stmts_stmts, cxt)
|
||||
|
||||
cxt.Info(cxt.ccode)
|
||||
with open('out.cpp', 'wb') as outfile:
|
||||
outfile.write((cxt.finalize()).encode('utf-8'))
|
||||
|
||||
return cxt
|
||||
|
||||
|
||||
__all__ = ["initialize", "generate", "exec", "saved_cxt"]
|
||||
-373
@@ -1,373 +0,0 @@
|
||||
from engine.utils import base62uuid
|
||||
from copy import copy
|
||||
from typing import *
|
||||
# replace column info with this later.
|
||||
class ColRef:
|
||||
def __init__(self, cname, _ty, cobj, cnt, table:'TableInfo', name, id, compound = False):
|
||||
self.cname = cname # column object location
|
||||
self.cxt_name = None # column object in context
|
||||
self.type = _ty
|
||||
self.cobj = cobj
|
||||
self.cnt = cnt
|
||||
self.table = table
|
||||
self.name = name
|
||||
self.id = id # position in table
|
||||
self.order_pending = None # order_pending
|
||||
self.compound = compound # compound field (list as a field)
|
||||
self.views = []
|
||||
self.aux_columns = [] # columns for temperary calculations
|
||||
# e.g. order by, group by, filter by expressions
|
||||
|
||||
self.__arr__ = (cname, _ty, cobj, cnt, table, name, id)
|
||||
|
||||
def reference(self):
|
||||
cxt = self.table.cxt
|
||||
self.table.reference()
|
||||
if self not in cxt.columns_in_context:
|
||||
counter = 0
|
||||
base_name = self.table.table_name + '_' + self.name
|
||||
if base_name in cxt.columns_in_context.values():
|
||||
while (f'{base_name}_{counter}') in cxt.columns_in_context.values():
|
||||
counter += 1
|
||||
base_name = f'{base_name}_{counter}'
|
||||
self.cxt_name = base_name
|
||||
cxt.columns_in_context[self] = base_name
|
||||
# TODO: change this to cname;
|
||||
cxt.emit(f'auto& {base_name} = *(ColRef<{self.type}> *)(&{self.table.cxt_name}->colrefs[{self.id}]);')
|
||||
elif self.cxt_name is None:
|
||||
self.cxt_name = cxt.columns_in_context[self]
|
||||
|
||||
return self.cxt_name
|
||||
|
||||
def __getitem__(self, key):
|
||||
if type(key) is str:
|
||||
return getattr(self, key)
|
||||
else:
|
||||
return self.__arr__[key]
|
||||
|
||||
def __setitem__(self, key, value):
|
||||
self.__arr__[key] = value
|
||||
|
||||
def __str__(self):
|
||||
return self.reference()
|
||||
def __repr__(self):
|
||||
return self.reference()
|
||||
|
||||
class TableInfo:
|
||||
|
||||
def __init__(self, table_name, cols, cxt:'Context'):
|
||||
# statics
|
||||
self.table_name = table_name
|
||||
self.alias = set([table_name])
|
||||
self.columns_byname = dict() # column_name, type
|
||||
self.columns = []
|
||||
self.cxt = cxt
|
||||
self.cxt_name = None
|
||||
self.views = set()
|
||||
#keep track of temp vars
|
||||
self.local_vars = dict()
|
||||
self.rec = None # a hook on get_col_d to record tables being referenced in the process
|
||||
self.groupinfo = None
|
||||
self.add_cols(cols)
|
||||
# runtime
|
||||
self.n_rows = 0 # number of cols
|
||||
self.order = [] # assumptions
|
||||
|
||||
cxt.tables_byname[self.table_name] = self # construct reverse map
|
||||
def reference(self):
|
||||
if self not in self.cxt.tables_in_context:
|
||||
counter = 0
|
||||
base_name = self.table_name
|
||||
if base_name in self.cxt.tables_in_context.values():
|
||||
while (f'{base_name}_{counter}') in self.cxt.tables_in_context.values():
|
||||
counter += 1
|
||||
base_name = f'{base_name}_{counter}'
|
||||
self.cxt_name = base_name
|
||||
self.cxt.tables_in_context[self] = base_name
|
||||
|
||||
type_tags = '<'
|
||||
for c in self.columns:
|
||||
type_tags += c.type + ','
|
||||
if type_tags.endswith(','):
|
||||
type_tags = type_tags[:-1]
|
||||
type_tags += '>'
|
||||
|
||||
self.cxt.emit(f'auto& {base_name} = *(TableInfo{type_tags} *)(cxt->tables["{self.table_name}"]);')
|
||||
return self.cxt_name
|
||||
def refer_all(self):
|
||||
self.reference()
|
||||
for c in self.columns:
|
||||
c.reference()
|
||||
def add_cols(self, cols, new = True):
|
||||
for i, c in enumerate(cols):
|
||||
self.add_col(c, new, i)
|
||||
def add_col(self, c, new = True, i = 0):
|
||||
_ty = c['type']
|
||||
if new:
|
||||
cname =f'get<{i}>({self.table_name})'
|
||||
_ty = _ty if type(c) is ColRef else list(_ty.keys())[0]
|
||||
col_object = ColRef(cname, _ty, c, 1, self,c['name'], len(self.columns))
|
||||
else:
|
||||
col_object = c
|
||||
cname = c.cname
|
||||
c.table = self
|
||||
self.cxt.ccols_byname[cname] = col_object
|
||||
self.columns_byname[c['name']] = col_object
|
||||
self.columns.append(col_object)
|
||||
def get_size(self):
|
||||
size_tmp = 'tmp_sz_'+base62uuid(6)
|
||||
self.cxt.emit(f'const auto& {size_tmp} = {self.columns[0].reference()}.size;')
|
||||
return size_tmp
|
||||
@property
|
||||
def n_cols(self):
|
||||
return len(self.columns)
|
||||
|
||||
def materialize_orderbys(self):
|
||||
view_stack = ''
|
||||
stack_name = ''
|
||||
for o in self.order:
|
||||
o.materialize()
|
||||
if len(view_stack) == 0:
|
||||
view_stack = o.view.name
|
||||
stack_name = view_stack
|
||||
else:
|
||||
view_stack = view_stack+'['+ o.view.name +']'
|
||||
# TODO: Optimize by doing everything in a stmt
|
||||
if len(view_stack) > 0:
|
||||
if len(self.order) > 1:
|
||||
self.cxt.emit(f'{stack_name}:{view_stack}')
|
||||
for c in self.columns:
|
||||
c.order_pending = stack_name
|
||||
self.order[0].node.view = stack_name
|
||||
self.order.clear()
|
||||
|
||||
def get_col_d(self, col_name):
|
||||
col = self.columns_byname[col_name]
|
||||
if type(self.rec) is set:
|
||||
self.rec.add(col)
|
||||
return col
|
||||
|
||||
def get_ccolname_d(self, col_name):
|
||||
return self.get_col_d(col_name).cname
|
||||
|
||||
def get_col(self, col_name):
|
||||
self.materialize_orderbys()
|
||||
col = self.get_col_d(col_name)
|
||||
if type(col.order_pending) is str:
|
||||
self.cxt.emit_no_flush(f'{col.cname}:{col.cname}[{col.order_pending}]')
|
||||
col.order_pending = None
|
||||
return col
|
||||
def get_ccolname(self, col_name):
|
||||
return self.get_col(col_name).cname
|
||||
|
||||
def add_alias(self, alias):
|
||||
# TODO: Scoping of alias should be constrainted in the query.
|
||||
if alias in self.cxt.tables_byname.keys():
|
||||
print("Error: table alias already exists")
|
||||
return
|
||||
self.cxt.tables_byname[alias] = self
|
||||
self.alias.add(alias)
|
||||
|
||||
def parse_col_names(self, colExpr, materialize = True, raw = False):
|
||||
# get_col = self.get_col if materialize else self.get_col_d
|
||||
|
||||
parsedColExpr = colExpr.split('.')
|
||||
ret = None
|
||||
if len(parsedColExpr) <= 1:
|
||||
ret = self.get_col_d(colExpr)
|
||||
else:
|
||||
datasource = self.cxt.tables_byname[parsedColExpr[0]]
|
||||
if datasource is None:
|
||||
raise ValueError(f'Table name/alias not defined{parsedColExpr[0]}')
|
||||
else:
|
||||
ret = datasource.parse_col_names(parsedColExpr[1], raw)
|
||||
from engine.expr import index_expr
|
||||
string = ret.reference() + index_expr
|
||||
if self.groupinfo is not None and ret and ret in self.groupinfo.raw_groups:
|
||||
string = f'get<{self.groupinfo.raw_groups.index(ret)}>({{y}})'
|
||||
return string, ret if raw else string
|
||||
|
||||
class View:
|
||||
def __init__(self, context, table = None, tmp = True):
|
||||
self.table: TableInfo = table
|
||||
self.name = 'v'+base62uuid(7)
|
||||
if type(table) is TableInfo:
|
||||
table.views.add(self)
|
||||
self.context = context
|
||||
|
||||
def construct(self):
|
||||
self.context.emit(f'{self.name}:()')
|
||||
|
||||
class Context:
|
||||
function_head = '''
|
||||
extern "C" int __DLLEXPORT__ dllmain(Context* cxt) {
|
||||
using namespace std;
|
||||
using namespace types;
|
||||
|
||||
'''
|
||||
LOG_INFO = 'INFO'
|
||||
LOG_ERROR = 'ERROR'
|
||||
LOG_SILENT = 'SILENT'
|
||||
from engine.types import Types
|
||||
type_table : Dict[str, Types] = dict()
|
||||
|
||||
def new(self):
|
||||
self.tmp_names = set()
|
||||
self.udf_map = dict()
|
||||
self.headers = set(['\"./server/libaquery.h\"'])
|
||||
self.finalized = False
|
||||
# read header
|
||||
self.ccode = str()
|
||||
self.ccodelet = str()
|
||||
with open('header.cxx', 'r') as outfile:
|
||||
self.ccode = outfile.read()
|
||||
# datasource will be availible after `from' clause is parsed
|
||||
# and will be deactivated when the `from' is out of scope
|
||||
self.datasource = None
|
||||
self.ds_stack = []
|
||||
self.scans = []
|
||||
self.removing_scan = False
|
||||
|
||||
def __init__(self):
|
||||
self.tables:list[TableInfo] = []
|
||||
self.tables_byname = dict()
|
||||
self.ccols_byname = dict()
|
||||
self.gc_name = 'gc_' + base62uuid(4)
|
||||
self.tmp_names = set()
|
||||
self.udf_map = dict()
|
||||
self.headers = set(['\"./server/libaquery.h\"'])
|
||||
self.finalized = False
|
||||
self.log_level = Context.LOG_SILENT
|
||||
self.print = print
|
||||
# read header
|
||||
self.ccode = str()
|
||||
self.ccodelet = str()
|
||||
self.columns_in_context = dict()
|
||||
self.tables_in_context = dict()
|
||||
with open('header.cxx', 'r') as outfile:
|
||||
self.ccode = outfile.read()
|
||||
# datasource will be availible after `from' clause is parsed
|
||||
# and will be deactivated when the `from' is out of scope
|
||||
self.datasource = None
|
||||
self.ds_stack = []
|
||||
self.scans = []
|
||||
self.removing_scan = False
|
||||
def add_table(self, table_name, cols):
|
||||
tbl = TableInfo(table_name, cols, self)
|
||||
self.tables.append(tbl)
|
||||
return tbl
|
||||
|
||||
def gen_tmptable(self):
|
||||
from engine.utils import base62uuid
|
||||
return f't{base62uuid(7)}'
|
||||
def reg_tmp(self, name, f):
|
||||
self.tmp_names.add(name)
|
||||
self.emit(f"{self.gc_name}.reg({{{name}, 0,0{'' if f is None else ',{f}'}}});")
|
||||
|
||||
def define_tmp(self, typename, isPtr = True, f = None):
|
||||
name = 'tmp_' + base62uuid()
|
||||
if isPtr:
|
||||
self.emit(f'auto* {name} = new {typename};')
|
||||
self.reg_tmp(name, f)
|
||||
else:
|
||||
self.emit(f'auto {name} = {typename};')
|
||||
return name
|
||||
def emit(self, codelet):
|
||||
self.ccode += self.ccodelet + codelet + '\n'
|
||||
self.ccodelet = ''
|
||||
def emit_no_flush(self, codelet):
|
||||
self.ccode += codelet + '\n'
|
||||
def emit_flush(self):
|
||||
self.ccode += self.ccodelet + '\n'
|
||||
self.ccodelet = ''
|
||||
def emit_nonewline(self, codelet):
|
||||
self.ccodelet += codelet
|
||||
|
||||
def datsource_top(self):
|
||||
if len(self.ds_stack) > 0:
|
||||
return self.ds_stack[-1]
|
||||
else:
|
||||
return None
|
||||
def datasource_pop(self):
|
||||
if len(self.ds_stack) > 0:
|
||||
self.ds_stack.pop()
|
||||
return self.ds_stack[-1]
|
||||
else:
|
||||
return None
|
||||
def datasource_push(self, ds):
|
||||
if type(ds) is TableInfo:
|
||||
self.ds_stack.append(ds)
|
||||
return ds
|
||||
else:
|
||||
return None
|
||||
def remove_scan(self, scan, str_scan):
|
||||
self.emit(str_scan)
|
||||
self.scans.remove(scan)
|
||||
|
||||
def Info(self, msg):
|
||||
if self.log_level.upper() == Context.LOG_INFO:
|
||||
self.print(msg)
|
||||
def Error(self, msg):
|
||||
if self.log_level.upper() == Context.LOG_ERROR:
|
||||
self.print(msg)
|
||||
else:
|
||||
self.Info(self, msg)
|
||||
|
||||
|
||||
def finalize(self):
|
||||
if not self.finalized:
|
||||
headers = ''
|
||||
for h in self.headers:
|
||||
if h[0] != '"':
|
||||
headers += '#include <' + h + '>\n'
|
||||
else:
|
||||
headers += '#include ' + h + '\n'
|
||||
self.ccode = headers + self.function_head + self.ccode + 'return 0;\n}'
|
||||
self.headers = set()
|
||||
return self.ccode
|
||||
def __str__(self):
|
||||
self.finalize()
|
||||
return self.ccode
|
||||
def __repr__(self) -> str:
|
||||
return self.__str__()
|
||||
|
||||
|
||||
class ast_node:
|
||||
types = dict()
|
||||
header = []
|
||||
def __init__(self, parent:"ast_node", node, context:Context = None):
|
||||
self.context = parent.context if context is None else context
|
||||
self.parent = parent
|
||||
self.datasource = None
|
||||
self.init(node)
|
||||
self.produce(node)
|
||||
self.spawn(node)
|
||||
self.consume(node)
|
||||
|
||||
def emit(self, code):
|
||||
self.context.emit(code)
|
||||
def emit_no_ln(self, code):
|
||||
self.context.emit_nonewline(code)
|
||||
|
||||
name = 'null'
|
||||
|
||||
# each ast node has 3 stages.
|
||||
# `produce' generates info for child nodes
|
||||
# `spawn' populates child nodes
|
||||
# `consume' consumes info from child nodes and finalizes codegen
|
||||
# For simple operators, there may not be need for some of these stages
|
||||
def init(self, _):
|
||||
pass
|
||||
def produce(self, _):
|
||||
pass
|
||||
def spawn(self, _):
|
||||
pass
|
||||
def consume(self, _):
|
||||
pass
|
||||
|
||||
# include classes in module as first order operators
|
||||
def include(objs):
|
||||
import inspect
|
||||
for _, cls in inspect.getmembers(objs):
|
||||
if inspect.isclass(cls) and issubclass(cls, ast_node) and not cls.name.startswith('_'):
|
||||
ast_node.types[cls.name] = cls
|
||||
-128
@@ -1,128 +0,0 @@
|
||||
# code-gen for data decl languages
|
||||
|
||||
from engine.orderby import orderby
|
||||
from engine.ast import ColRef, TableInfo, ast_node, Context, include
|
||||
from engine.scan import scan
|
||||
from engine.utils import base62uuid
|
||||
|
||||
class create_table(ast_node):
|
||||
name = 'create_table'
|
||||
def __init__(self, parent: "ast_node", node, context: Context = None, cexprs = None, lineage = False):
|
||||
self.cexprs = cexprs
|
||||
self.lineage = lineage
|
||||
super().__init__(parent, node, context)
|
||||
def produce(self, node):
|
||||
if type(node) is not TableInfo:
|
||||
ct = node[self.name]
|
||||
tbl = self.context.add_table(ct['name'], ct['columns'])
|
||||
else:
|
||||
tbl = node
|
||||
|
||||
col_type_str = ','.join([c.type for c in tbl.columns])
|
||||
# create tables in c
|
||||
self.emit(f"auto {tbl.table_name} = new TableInfo<{col_type_str}>(\"{tbl.table_name}\", {tbl.n_cols});")
|
||||
self.emit("cxt->tables.insert({\"" + tbl.table_name + f"\", {tbl.table_name}"+"});")
|
||||
self.context.tables_in_context[tbl] = tbl.table_name
|
||||
tbl.cxt_name = tbl.table_name
|
||||
tbl.refer_all()
|
||||
# create an empty new table
|
||||
if self.cexprs is None:
|
||||
for c in tbl.columns:
|
||||
self.emit(f'{c.cxt_name}.init("{c.name}");')
|
||||
# create an output table
|
||||
else:
|
||||
# 1 to 1 lineage.
|
||||
if len(self.context.scans) == 0:
|
||||
if self.lineage:
|
||||
order = 'order_' + base62uuid(6)
|
||||
self.emit(f'auto {order} = {self.parent.datasource.cxt_name}->order_by<{orderby(self.parent, self.parent.assumptions).result()}>();')
|
||||
self.lineage = '*' + order
|
||||
else:
|
||||
self.lineage = None
|
||||
for i, c in enumerate(tbl.columns):
|
||||
self.emit(f'{c.cxt_name}.init("{c.name}");')
|
||||
self.emit(f"{c.cxt_name} = {self.cexprs[i](self.lineage)};")
|
||||
self.lineage = None
|
||||
self.parent.assumptions = None
|
||||
else:
|
||||
scanner:scan = self.context.scans[-1]
|
||||
if self.lineage:
|
||||
lineage_var = 'lineage_' + base62uuid(6)
|
||||
counter_var = 'counter_' + base62uuid(6)
|
||||
scanner.add(f'auto {lineage_var} = {self.datasource.cxt_name}->bind({tbl.cxt_name});', "init")
|
||||
scanner.add(f'auto {counter_var} = 0;', "init")
|
||||
scanner.add(f"{lineage_var}.emplace_back({counter_var}++);", "front")
|
||||
self.lineage = f"{lineage_var}.rid"
|
||||
for i, c in enumerate(tbl.columns):
|
||||
scanner.add(f'{c.cxt_name}.init("{c.name}");', "init")
|
||||
scanner.add(f"{c.cxt_name} = {self.cexprs[i](scanner.it_var)};")
|
||||
|
||||
class insert(ast_node):
|
||||
name = 'insert'
|
||||
def produce(self, node):
|
||||
ct = node[self.name]
|
||||
table:TableInfo = self.context.tables_byname[ct]
|
||||
|
||||
values = node['query']['select']
|
||||
if len(values) != table.n_cols:
|
||||
raise ValueError("Column Mismatch")
|
||||
table.refer_all()
|
||||
for i, s in enumerate(values):
|
||||
if 'value' in s:
|
||||
cname = table.columns[i].cxt_name
|
||||
self.emit(f"{cname}.emplace_back({s['value']});")
|
||||
else:
|
||||
# subquery, dispatch to select astnode
|
||||
pass
|
||||
|
||||
class c(ast_node):
|
||||
name='c'
|
||||
def produce(self, node):
|
||||
self.emit(node[self.name])
|
||||
|
||||
class load(ast_node):
|
||||
name="load"
|
||||
def produce(self, node):
|
||||
self.context.headers.add('"csv.h"')
|
||||
node = node[self.name]
|
||||
table:TableInfo = self.context.tables_byname[node['table']]
|
||||
table.refer_all()
|
||||
csv_reader_name = 'csv_reader_' + base62uuid(6)
|
||||
col_types = [c.type for c in table.columns]
|
||||
col_tmp_names = ['tmp_'+base62uuid(8) for _ in range(len(table.columns))]
|
||||
# col_type_str = ",".join(col_types)
|
||||
col_names = ','.join([f'"{c.name}"' for c in table.columns])
|
||||
|
||||
self.emit(f'io::CSVReader<{len(col_types)}> {csv_reader_name}("{node["file"]["literal"]}");')
|
||||
self.emit(f'{csv_reader_name}.read_header(io::ignore_extra_column, {col_names});')
|
||||
for t, n in zip(col_types, col_tmp_names):
|
||||
self.emit(f'{t} {n};')
|
||||
self.emit(f'while({csv_reader_name}.read_row({",".join(col_tmp_names)})) {{ \n')
|
||||
for i, c in enumerate(table.columns):
|
||||
self.emit(f'{c.cxt_name}.emplace_back({col_tmp_names[i]});')
|
||||
self.emit('}')
|
||||
|
||||
|
||||
class outfile(ast_node):
|
||||
name="_outfile"
|
||||
def produce(self, node):
|
||||
out_table:TableInfo = self.parent.out_table
|
||||
filename = node['loc']['literal'] if 'loc' in node else node['literal']
|
||||
sep = ',' if 'term' not in node else node['term']['literal']
|
||||
file_pointer = 'fp_' + base62uuid(6)
|
||||
self.emit(f'FILE* {file_pointer} = fopen("{filename}", "wb");')
|
||||
self.emit(f'{out_table.cxt_name}->printall("{sep}", "\\n", nullptr, {file_pointer});')
|
||||
self.emit(f'fclose({file_pointer});')
|
||||
# self.context.headers.add('fstream')
|
||||
# cout_backup_buffer = 'stdout_' + base62uuid(4)
|
||||
# ofstream = 'ofstream_' + base62uuid(6)
|
||||
# self.emit(f'auto {cout_backup_buffer} = cout.rdbuf();')
|
||||
# self.emit(f'auto {ofstream} = ofstream("{filename}");')
|
||||
# self.emit(f'cout.rdbuf({ofstream}.rdbuf());')
|
||||
# TODO: ADD STMTS.
|
||||
# self.emit(f'cout.rdbuf({cout_backup_buffer});')
|
||||
# self.emit(f'{ofstream}.close();')
|
||||
|
||||
|
||||
import sys
|
||||
include(sys.modules[__name__])
|
||||
-135
@@ -1,135 +0,0 @@
|
||||
from engine.ast import ast_node, ColRef
|
||||
start_expr = 'f"'
|
||||
index_expr = '{\'\' if x is None and y is None else f\'[{x}]\'}'
|
||||
end_expr = '"'
|
||||
|
||||
class expr(ast_node):
|
||||
name='expr'
|
||||
builtin_func_maps = {
|
||||
'max': 'max',
|
||||
'min': 'min',
|
||||
'avg': 'avg',
|
||||
'sum': 'sum',
|
||||
'count' : 'count',
|
||||
'mins': ['mins', 'minw'],
|
||||
'maxs': ['maxs', 'maxw'],
|
||||
'avgs': ['avgs', 'avgw'],
|
||||
'sums': ['sums', 'sumw'],
|
||||
}
|
||||
|
||||
binary_ops = {
|
||||
'sub':'-',
|
||||
'add':'+',
|
||||
'mul':'*',
|
||||
'div':'/',
|
||||
'mod':'%',
|
||||
'and':'&&',
|
||||
'or':'||',
|
||||
'xor' : '^',
|
||||
'gt':'>',
|
||||
'lt':'<',
|
||||
'lte':'<=',
|
||||
'gte':'>=',
|
||||
'neq':'!=',
|
||||
'eq':'=='
|
||||
}
|
||||
|
||||
compound_ops = {
|
||||
'missing' : ['missing', lambda x: f'{x[0]} == nullval<decays<decltype({x[0]})>>'],
|
||||
}
|
||||
|
||||
unary_ops = {
|
||||
'neg' : '-',
|
||||
'not' : '!'
|
||||
}
|
||||
|
||||
coumpound_generating_ops = ['avgs', 'mins', 'maxs', 'sums'] + \
|
||||
list( binary_ops.keys()) + list(compound_ops.keys()) + list(unary_ops.keys() )
|
||||
|
||||
def __init__(self, parent, node, materialize_cols = True, abs_col = False):
|
||||
self.materialize_cols = materialize_cols
|
||||
self.raw_col = None
|
||||
self.__abs = abs_col
|
||||
self.inside_agg = False
|
||||
if(type(parent) is expr):
|
||||
self.inside_agg = parent.inside_agg
|
||||
self.__abs = parent.__abs
|
||||
ast_node.__init__(self, parent, node, None)
|
||||
|
||||
def init(self, _):
|
||||
from engine.projection import projection
|
||||
parent = self.parent
|
||||
self.isvector = parent.isvector if type(parent) is expr else False
|
||||
self.is_compound = parent.is_compound if type(parent) is expr else False
|
||||
if type(parent) in [projection, expr]:
|
||||
self.datasource = parent.datasource
|
||||
else:
|
||||
self.datasource = self.context.datasource
|
||||
self.udf_map = parent.context.udf_map
|
||||
self._expr = ''
|
||||
self.cexpr = None
|
||||
self.func_maps = {**self.udf_map, **self.builtin_func_maps}
|
||||
|
||||
def produce(self, node):
|
||||
if type(node) is dict:
|
||||
for key, val in node.items():
|
||||
if key in self.func_maps:
|
||||
# TODO: distinguish between UDF agg functions and other UDF functions.
|
||||
self.inside_agg = True
|
||||
self.context.headers.add('"./server/aggregations.h"')
|
||||
if type(val) is list and len(val) > 1:
|
||||
cfunc = self.func_maps[key]
|
||||
cfunc = cfunc[len(val) - 1] if type(cfunc) is list else cfunc
|
||||
self._expr += f"{cfunc}("
|
||||
for i, p in enumerate(val):
|
||||
self._expr += expr(self, p)._expr + (','if i<len(val)-1 else '')
|
||||
else:
|
||||
funcname = self.func_maps[key]
|
||||
funcname = funcname[0] if type(funcname) is list else funcname
|
||||
self._expr += f"{funcname}("
|
||||
self._expr += expr(self, val)._expr
|
||||
self._expr += ')'
|
||||
self.inside_agg = False
|
||||
elif key in self.binary_ops:
|
||||
l = expr(self, val[0])._expr
|
||||
r = expr(self, val[1])._expr
|
||||
self._expr += f'({l}{self.binary_ops[key]}{r})'
|
||||
elif key in self.compound_ops:
|
||||
x = []
|
||||
if type(val) is list:
|
||||
for v in val:
|
||||
x.append(expr(self, v)._expr)
|
||||
self._expr = self.compound_ops[key][1](x)
|
||||
elif key in self.unary_ops:
|
||||
self._expr += f'{self.unary_ops[key]}({expr(self, val)._expr})'
|
||||
else:
|
||||
self.context.Error(f'Undefined expr: {key}{val}')
|
||||
|
||||
if key in self.coumpound_generating_ops and not self.is_compound:
|
||||
self.is_compound = True
|
||||
p = self.parent
|
||||
while type(p) is expr and not p.is_compound:
|
||||
p.is_compound = True
|
||||
p = p.parent
|
||||
|
||||
elif type(node) is str:
|
||||
p = self.parent
|
||||
while type(p) is expr and not p.isvector:
|
||||
p.isvector = True
|
||||
p = p.parent
|
||||
|
||||
self._expr, self.raw_col = self.datasource.parse_col_names(node, self.materialize_cols, True)
|
||||
self.raw_col = self.raw_col if type(self.raw_col) is ColRef else None
|
||||
if self.__abs and self.raw_col:
|
||||
self._expr = self.raw_col.reference() + ("" if self.inside_agg else index_expr)
|
||||
elif type(node) is bool:
|
||||
self._expr = '1' if node else '0'
|
||||
else:
|
||||
self._expr = f'{node}'
|
||||
def toCExpr(_expr):
|
||||
return lambda x = None, y = None : eval(start_expr + _expr + end_expr)
|
||||
def consume(self, _):
|
||||
self.cexpr = expr.toCExpr(self._expr)
|
||||
def __str__(self):
|
||||
return self.cexpr
|
||||
|
||||
@@ -1,72 +0,0 @@
|
||||
from engine.ast import ColRef, TableInfo, ast_node
|
||||
from engine.orderby import assumption
|
||||
from engine.scan import scan
|
||||
from engine.utils import base62uuid
|
||||
from engine.expr import expr
|
||||
|
||||
class groupby(ast_node):
|
||||
name = '_groupby'
|
||||
def init(self, _):
|
||||
self.context.headers.add('"./server/hasher.h"')
|
||||
self.context.headers.add('unordered_map')
|
||||
self.group = 'g' + base62uuid(7)
|
||||
self.group_type = 'record_type' + base62uuid(7)
|
||||
self.datasource = self.parent.datasource
|
||||
self.scanner = None
|
||||
self.datasource.rec = set()
|
||||
self.raw_groups = []
|
||||
def produce(self, node):
|
||||
|
||||
if type(node) is not list:
|
||||
node = [node]
|
||||
g_contents = ''
|
||||
g_contents_list = []
|
||||
first_col = ''
|
||||
for i, g in enumerate(node):
|
||||
v = g['value']
|
||||
e = expr(self, v)
|
||||
if type(e.raw_col) is ColRef:
|
||||
self.raw_groups.append(e.raw_col)
|
||||
e = e._expr
|
||||
# if v is compound expr, create tmp cols
|
||||
if type(v) is not str:
|
||||
tmpcol = 't' + base62uuid(7)
|
||||
self.emit(f'auto {tmpcol} = {e};')
|
||||
e = tmpcol
|
||||
if i == 0:
|
||||
first_col = e
|
||||
g_contents_list.append(e)
|
||||
g_contents_decltype = [f'decltype({c})' for c in g_contents_list]
|
||||
g_contents = expr.toCExpr(','.join(g_contents_list))
|
||||
self.emit(f'typedef record<{expr.toCExpr(",".join(g_contents_decltype))(0)}> {self.group_type};')
|
||||
self.emit(f'unordered_map<{self.group_type}, vector_type<uint32_t>, '
|
||||
f'transTypes<{self.group_type}, hasher>> {self.group};')
|
||||
self.n_grps = len(node)
|
||||
self.scanner = scan(self, self.datasource, expr.toCExpr(first_col)()+'.size')
|
||||
self.scanner.add(f'{self.group}[forward_as_tuple({g_contents(self.scanner.it_var)})].emplace_back({self.scanner.it_var});')
|
||||
|
||||
|
||||
def consume(self, _):
|
||||
self.referenced = self.datasource.rec
|
||||
self.datasource.rec = None
|
||||
self.scanner.finalize()
|
||||
|
||||
def deal_with_assumptions(self, assumption:assumption, out:TableInfo):
|
||||
gscanner = scan(self, self.group)
|
||||
val_var = 'val_'+base62uuid(7)
|
||||
gscanner.add(f'auto &{val_var} = {gscanner.it_var}.second;')
|
||||
gscanner.add(f'{self.datasource.cxt_name}->order_by<{assumption.result()}>(&{val_var});')
|
||||
gscanner.finalize()
|
||||
|
||||
def finalize(self, cexprs, out:TableInfo):
|
||||
gscanner = scan(self, self.group)
|
||||
key_var = 'key_'+base62uuid(7)
|
||||
val_var = 'val_'+base62uuid(7)
|
||||
|
||||
gscanner.add(f'auto &{key_var} = {gscanner.it_var}.first;')
|
||||
gscanner.add(f'auto &{val_var} = {gscanner.it_var}.second;')
|
||||
gscanner.add(';\n'.join([f'{out.columns[i].reference()}.emplace_back({ce(x=val_var, y=key_var)})' for i, ce in enumerate(cexprs)])+';')
|
||||
|
||||
gscanner.finalize()
|
||||
|
||||
self.datasource.groupinfo = None
|
||||
@@ -1,6 +0,0 @@
|
||||
from engine.ast import ast_node
|
||||
|
||||
|
||||
class join(ast_node):
|
||||
name='join'
|
||||
|
||||
@@ -1,65 +0,0 @@
|
||||
from engine.ast import ColRef, TableInfo, View, ast_node, Context
|
||||
from engine.utils import base62uuid, seps
|
||||
from engine.expr import expr
|
||||
|
||||
class order_item:
|
||||
def __init__(self, name, node, order = True):
|
||||
self.name = name
|
||||
self.order = order
|
||||
self.node = node
|
||||
self.materialized = False
|
||||
|
||||
def materialize(self):
|
||||
if not self.materialized:
|
||||
self.name = expr(self.node, self.name, False).cexpr
|
||||
self.materialized = True
|
||||
return ('' if self.order else '-') + f'({self.name})'
|
||||
|
||||
def __str__(self):
|
||||
return self.name
|
||||
def __repr__(self):
|
||||
return self.__str__()
|
||||
|
||||
class orderby(ast_node):
|
||||
name = '_orderby'
|
||||
def __init__(self, parent: "ast_node", node, context: Context = None):
|
||||
self.col_list = []
|
||||
super().__init__(parent, node, context)
|
||||
def init(self, _):
|
||||
self.datasource = self.parent.datasource
|
||||
self.order = []
|
||||
self.view = ''
|
||||
def produce(self, node):
|
||||
if type(node) is not list:
|
||||
node = [node]
|
||||
for n in node:
|
||||
order = not ('sort' in n and n['sort'] == 'desc')
|
||||
col_id = self.datasource.columns_byname[n['value']].id
|
||||
col_id = col_id if order else -col_id-1
|
||||
if col_id not in self.col_list:
|
||||
self.col_list.append(col_id)
|
||||
self.order.append(order_item(n['value'], self, order))
|
||||
|
||||
def merge(self, node):
|
||||
self.produce(node)
|
||||
|
||||
def finalize(self, references):
|
||||
self.order = [ o for o in self.order if o.name in references ]
|
||||
|
||||
def result(self, sep:str = ','):
|
||||
return sep.join([f"{c}" for c in self.col_list])
|
||||
|
||||
class assumption(orderby):
|
||||
name = '_assumption'
|
||||
def __init__(self, parent: "ast_node", node, context: Context = None, exclude = []):
|
||||
self.exclude = exclude
|
||||
super().__init__(parent, node, context)
|
||||
|
||||
def produce(self, node):
|
||||
if type(node) is not list:
|
||||
node = [node]
|
||||
[n for n in node if n not in self.exclude]
|
||||
return super().produce(node)
|
||||
|
||||
def empty(self):
|
||||
return len(self.col_list) == 0
|
||||
@@ -1,180 +0,0 @@
|
||||
from engine.ast import ColRef, TableInfo, ast_node, Context, include
|
||||
from engine.groupby import groupby
|
||||
from engine.join import join
|
||||
from engine.expr import expr
|
||||
from engine.orderby import assumption, orderby
|
||||
from engine.scan import filter
|
||||
from engine.utils import base62uuid, enlist, base62alp, has_other
|
||||
from engine.ddl import create_table, outfile
|
||||
import copy
|
||||
|
||||
class projection(ast_node):
|
||||
name='select'
|
||||
def __init__(self, parent:ast_node, node, context:Context = None, outname = None, disp = True):
|
||||
self.disp = disp
|
||||
self.outname = outname
|
||||
self.group_node = None
|
||||
self.assumptions = None
|
||||
self.where = None
|
||||
ast_node.__init__(self, parent, node, context)
|
||||
def init(self, _):
|
||||
if self.outname is None:
|
||||
self.outname = self.context.gen_tmptable()
|
||||
|
||||
def produce(self, node):
|
||||
p = node['select']
|
||||
self.projections = p if type(p) is list else [p]
|
||||
self.context.Info(node)
|
||||
|
||||
def spawn(self, node):
|
||||
self.datasource = None
|
||||
if 'from' in node:
|
||||
from_clause = node['from']['table_source']
|
||||
if type(from_clause) is list:
|
||||
# from joins
|
||||
join(self, from_clause)
|
||||
elif type(from_clause) is dict:
|
||||
if 'value' in from_clause:
|
||||
value = from_clause['value']
|
||||
if type(value) is dict:
|
||||
if 'select' in value:
|
||||
# from subquery
|
||||
projection(self, from_clause, disp = False)
|
||||
else:
|
||||
# TODO: from func over table
|
||||
print(f'from func over table{node}')
|
||||
elif type(value) is str:
|
||||
self.datasource = self.context.tables_byname[value]
|
||||
if 'name' in value:
|
||||
self.datasource.add_alias(value['name'])
|
||||
if 'assuming' in node['from']:
|
||||
self.assumptions = enlist(node['from']['assuming'])
|
||||
|
||||
elif type(from_clause) is str:
|
||||
self.datasource = self.context.tables_byname[from_clause]
|
||||
|
||||
if self.datasource is None:
|
||||
raise ValueError('spawn error: from clause')
|
||||
|
||||
if self.datasource is not None:
|
||||
self.datasource_changed = True
|
||||
self.prev_datasource = self.context.datasource
|
||||
self.context.datasource = self.datasource
|
||||
if 'where' in node:
|
||||
self.where = filter(self, node['where'], True)
|
||||
# self.datasource = filter(self, node['where'], True).output
|
||||
# self.context.datasource = self.datasource
|
||||
|
||||
if 'groupby' in node:
|
||||
self.group_node = groupby(self, node['groupby'])
|
||||
self.datasource = copy.copy(self.datasource) # shallow copy
|
||||
self.datasource.groupinfo = self.group_node
|
||||
else:
|
||||
self.group_node = None
|
||||
|
||||
def consume(self, node):
|
||||
self.inv = True
|
||||
disp_varname = 'd'+base62uuid(7)
|
||||
has_groupby = self.group_node is not None
|
||||
cexprs = []
|
||||
flatten = False
|
||||
cols = []
|
||||
self.out_table = TableInfo('out_'+base62uuid(4), [], self.context)
|
||||
if 'outfile' in node:
|
||||
flatten = True
|
||||
|
||||
new_names = []
|
||||
proj_raw_cols = []
|
||||
for i, proj in enumerate(self.projections):
|
||||
cname = ''
|
||||
compound = False
|
||||
self.datasource.rec = set()
|
||||
if type(proj) is dict:
|
||||
if 'value' in proj:
|
||||
e = proj['value']
|
||||
sname = expr(self, e)
|
||||
if type(sname.raw_col) is ColRef:
|
||||
proj_raw_cols.append(sname.raw_col)
|
||||
sname = sname._expr
|
||||
fname = expr.toCExpr(sname) # fastest access method at innermost context
|
||||
absname = expr(self, e, abs_col=True)._expr # absolute name at function scope
|
||||
# TODO: Make it single pass here.
|
||||
compound = True # compound column
|
||||
cexprs.append(fname)
|
||||
cname = e if type(e) is str else ''.join([a if a in base62alp else '' for a in expr.toCExpr(absname)()])
|
||||
if 'name' in proj: # renaming column by AS keyword
|
||||
cname = proj['name']
|
||||
new_names.append(cname)
|
||||
elif type(proj) is str:
|
||||
col = self.datasource.get_col_d(proj)
|
||||
if type(col) is ColRef:
|
||||
col.reference()
|
||||
|
||||
compound = compound and has_groupby and has_other(self.datasource.rec, self.group_node.referenced)
|
||||
self.datasource.rec = None
|
||||
|
||||
typename = f'decays<decltype({absname})>'
|
||||
if not compound:
|
||||
typename = f'value_type<{typename}>'
|
||||
|
||||
cols.append(ColRef(cname, expr.toCExpr(typename)(), self.out_table, 0, None, cname, i, compound=compound))
|
||||
|
||||
self.out_table.add_cols(cols, False)
|
||||
|
||||
lineage = None
|
||||
|
||||
if has_groupby:
|
||||
create_table(self, self.out_table) # creates empty out_table.
|
||||
if self.assumptions is not None:
|
||||
self.assumptions = assumption(self, self.assumptions, exclude=self.group_node.raw_groups)
|
||||
if not self.assumptions.empty():
|
||||
self.group_node.deal_with_assumptions(self.assumptions, self.out_table)
|
||||
self.assumptions = None
|
||||
self.group_node.finalize(cexprs, self.out_table)
|
||||
else:
|
||||
# if all assumptions in projections, treat as orderby
|
||||
lineage = self.assumptions is not None and has_other(self.assumptions, proj_raw_cols)
|
||||
spawn = create_table(self, self.out_table, cexprs = cexprs, lineage = lineage) # create and populate out_table.
|
||||
if lineage and type(spawn.lineage) is str:
|
||||
lineage = spawn.lineage
|
||||
self.assumptions = orderby(self, self.assumptions) # do not exclude proj_raw_cols
|
||||
else:
|
||||
lineage = None
|
||||
if self.where is not None:
|
||||
self.where.finalize()
|
||||
|
||||
if type(lineage) is str:
|
||||
order = 'order_' + base62uuid(6)
|
||||
self.emit(f'auto {order} = {self.datasource.cxt_name}->order_by<{self.assumptions.result()}>({lineage});')
|
||||
self.emit(f'{self.out_table.cxt_name}->materialize(*{order});')
|
||||
self.assumptions = None
|
||||
|
||||
if self.assumptions is not None:
|
||||
orderby_node = orderby(self, self.assumptions)
|
||||
else:
|
||||
orderby_node = None
|
||||
|
||||
if 'orderby' in node:
|
||||
self.datasource = self.out_table
|
||||
self.context.datasource = self.out_table # discard current ds
|
||||
orderbys = node['orderby']
|
||||
orderby_node = orderby(self, orderbys) if orderby_node is None else orderby_node.merge(orderbys)
|
||||
|
||||
if orderby_node is not None:
|
||||
self.emit(f'auto {disp_varname} = {self.out_table.reference()}->order_by_view<{orderby_node.result()}>();')
|
||||
else:
|
||||
disp_varname = f'*{self.out_table.cxt_name}'
|
||||
|
||||
if self.disp:
|
||||
self.emit(f'print({disp_varname});')
|
||||
|
||||
|
||||
if flatten:
|
||||
outfile(self, node['outfile'])
|
||||
|
||||
if self.datasource_changed:
|
||||
self.context.datasource = self.prev_datasource
|
||||
|
||||
|
||||
import sys
|
||||
include(sys.modules[__name__])
|
||||
@@ -1,99 +0,0 @@
|
||||
from xmlrpc.client import Boolean
|
||||
from engine.ast import ColRef, TableInfo, View, ast_node, Context
|
||||
from engine.utils import base62uuid
|
||||
from engine.expr import expr
|
||||
|
||||
class scan(ast_node):
|
||||
name = 'scan'
|
||||
def __init__(self, parent: "ast_node", node, size = None, context: Context = None, const = False):
|
||||
self.type = type
|
||||
self.size = size
|
||||
self.const = "const " if const else ""
|
||||
super().__init__(parent, node, context)
|
||||
def init(self, _):
|
||||
self.datasource = self.context.datasource
|
||||
self.initializers = ''
|
||||
self.start = ''
|
||||
self.front = ''
|
||||
self.body = ''
|
||||
self.end = '}'
|
||||
self.mode = None
|
||||
self.filters = []
|
||||
scan_vars = set(s.it_var for s in self.context.scans)
|
||||
self.it_var = 'i' + base62uuid(2)
|
||||
while(self.it_var in scan_vars):
|
||||
self.it_var = 'i' + base62uuid(6)
|
||||
self.parent.context.scans.append(self)
|
||||
def produce(self, node):
|
||||
if type(node) is ColRef:
|
||||
self.colref = node
|
||||
if self.size is None:
|
||||
self.mode = ["col", node.table]
|
||||
self.start += f'for ({self.const}auto& {self.it_var} : {node.reference()}) {{\n'
|
||||
else:
|
||||
self.mode = ["idx", node.table]
|
||||
self.start += f"for (uint32_t {self.it_var} = 0; {self.it_var} < {node.reference()}.size; ++{self.it_var}){{\\n"
|
||||
elif type(node) is str:
|
||||
self.mode = ["idx", None]
|
||||
self.start+= f'for({self.const}auto& {self.it_var} : {node}) {{\n'
|
||||
else:
|
||||
self.mode = ["idx", node] # Node is the TableInfo
|
||||
self.start += f"for (uint32_t {self.it_var} = 0; {self.it_var} < {self.size}; ++{self.it_var}){{\n"
|
||||
|
||||
def add(self, stmt, position = "body"):
|
||||
if position == "body":
|
||||
self.body += stmt + '\n'
|
||||
elif position == "init":
|
||||
self.initializers += stmt + '\n'
|
||||
else:
|
||||
self.front += stmt + '\n'
|
||||
|
||||
def finalize(self):
|
||||
for f in self.filters:
|
||||
self.start += f
|
||||
self.end += '}'
|
||||
self.context.remove_scan(self, self.initializers + self.start + self.front + self.body + self.end)
|
||||
|
||||
class filter(ast_node):
|
||||
name = 'filter'
|
||||
def __init__(self, parent: "ast_node", node, materialize = False, context = None):
|
||||
self.materialize = materialize
|
||||
super().__init__(parent, node, context)
|
||||
def init(self, _):
|
||||
self.datasource = self.context.datasource
|
||||
self.view = View(self.context, self.datasource)
|
||||
self.value = None
|
||||
|
||||
def spawn(self, node):
|
||||
# TODO: deal with subqueries
|
||||
self.modified_node = node
|
||||
return super().spawn(node)
|
||||
def __materialize__(self):
|
||||
if self.materialize:
|
||||
cols = [] if self.datasource is None else self.datasource.columns
|
||||
self.output = TableInfo('tn'+base62uuid(6), cols, self.context)
|
||||
self.output.construct()
|
||||
if type(self.value) is View: # cond filtered on tables.
|
||||
self.emit(f'{self.value.name}:&{self.value.name}')
|
||||
for o, c in zip(self.output.columns,self.value.table.columns):
|
||||
self.emit(f'{o.cname}:{c.cname}[{self.value.name}]')
|
||||
elif self.value is not None: # cond is scalar
|
||||
tmpVar = 't'+base62uuid(7)
|
||||
self.emit(f'{tmpVar}:{self.value}')
|
||||
for o, c in zip(self.output.columns, self.datasource.columns):
|
||||
self.emit(f'{o.cname}:$[{tmpVar};{c.cname};()]')
|
||||
|
||||
def finalize(self):
|
||||
self.scanner.finalize()
|
||||
def consume(self, _):
|
||||
# TODO: optimizations after converting expr to cnf
|
||||
self.scanner = None
|
||||
for s in self.context.scans:
|
||||
if self.datasource == s.mode[1]:
|
||||
self.scanner = s
|
||||
break
|
||||
if self.scanner is None:
|
||||
self.scanner = scan(self, self.datasource, self.datasource.get_size())
|
||||
self.expr = expr(self, self.modified_node)
|
||||
self.scanner.filters.append(f'if ({self.expr.cexpr(self.scanner.it_var)}) {{\n')
|
||||
|
||||
-386
@@ -1,386 +0,0 @@
|
||||
from copy import deepcopy
|
||||
from typing import Dict, List
|
||||
|
||||
from aquery_config import have_hge
|
||||
from engine.utils import base62uuid, defval
|
||||
|
||||
type_table: Dict[str, "Types"] = {}
|
||||
|
||||
class Types:
|
||||
def init_any(self):
|
||||
self.name : str = 'Any'
|
||||
self.sqlname : str = 'Int'
|
||||
self.cname : str = 'void*'
|
||||
self.ctype_name : str = "types::NONE"
|
||||
self.null_value = 0
|
||||
self.priority : int= 0
|
||||
self.cast_to_dict = dict()
|
||||
self.cast_from_dict = dict()
|
||||
|
||||
def __init__(self, priority = 0, *,
|
||||
name = None, cname = None, sqlname = None,
|
||||
ctype_name = None, null_value = None,
|
||||
fp_type = None, long_type = None, is_fp = False,
|
||||
cast_to = None, cast_from = None
|
||||
):
|
||||
|
||||
self.is_fp = is_fp
|
||||
if name is None:
|
||||
self.init_any()
|
||||
else:
|
||||
self.name = name
|
||||
self.cname = defval(cname, name.lower() + '_t')
|
||||
self.sqlname = defval(sqlname, name.upper())
|
||||
self.ctype_name = defval(ctype_name, f'types::{name.upper()}')
|
||||
self.null_value = defval(null_value, 0)
|
||||
self.cast_to_dict = defval(cast_to, dict())
|
||||
self.cast_from_dict = defval(cast_from, dict())
|
||||
self.priority = priority
|
||||
|
||||
self.long_type = defval(long_type, self)
|
||||
self.fp_type = defval(fp_type, self)
|
||||
|
||||
global type_table
|
||||
type_table[name] = self
|
||||
|
||||
def cast_to(self, ty : "Types"):
|
||||
if ty in self.cast_to_dict:
|
||||
return self.cast_to_dict[ty.name](ty)
|
||||
else:
|
||||
raise Exception(f'Illeagal cast: from {self.name} to {ty.name}.')
|
||||
def cast_from(self, ty : "Types"):
|
||||
if ty in self.cast_from_dict:
|
||||
return self.cast_from_dict[ty.name](ty)
|
||||
else:
|
||||
raise Exception(f'Illeagal cast: from {ty.name} to {self.name}.')
|
||||
|
||||
def __call__(self, args):
|
||||
arg_str = ', '.join([a.__str__() for a in args])
|
||||
ret = deepcopy(self)
|
||||
ret.sqlname = self.sqlname + f'({arg_str})'
|
||||
return ret
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return self.sqlname
|
||||
def __str__(self) -> str:
|
||||
return self.sqlname
|
||||
|
||||
@staticmethod
|
||||
def decode(aquery_type : str, vector_type:str = 'vector_type') -> "Types":
|
||||
if (aquery_type.lower().startswith('vec')):
|
||||
return VectorT(Types.decode(aquery_type[3:]), vector_type)
|
||||
return type_table[aquery_type.lower()]
|
||||
|
||||
class TypeCollection:
|
||||
def __init__(self, sz, deftype, fptype = None, utype = None, *, collection = None) -> None:
|
||||
self.size = sz
|
||||
self.type = deftype
|
||||
self.fptype = fptype
|
||||
self.utype = utype
|
||||
self.all_types = [deftype]
|
||||
if fptype is not None:
|
||||
self.all_types.append(fptype)
|
||||
if utype is not None:
|
||||
self.all_types.append(utype)
|
||||
if collection is not None:
|
||||
for ty in collection:
|
||||
self.all_types.append(ty)
|
||||
|
||||
type_table = dict()
|
||||
AnyT = Types(-1)
|
||||
LazyT = Types(240, name = 'Lazy', cname = '', sqlname = '', ctype_name = '')
|
||||
DateT = Types(200, name = 'DATE', cname = 'types::date_t', sqlname = 'DATE', ctype_name = 'types::ADATE')
|
||||
TimeT = Types(201, name = 'TIME', cname = 'types::time_t', sqlname = 'TIME', ctype_name = 'types::ATIME')
|
||||
TimeStampT = Types(202, name = 'TIMESTAMP', cname = 'types::timestamp_t', sqlname = 'TIMESTAMP', ctype_name = 'ATIMESTAMP')
|
||||
DoubleT = Types(17, name = 'double', cname='double', sqlname = 'DOUBLE', is_fp = True)
|
||||
LDoubleT = Types(18, name = 'long double', cname='long double', sqlname = 'LDOUBLE', is_fp = True)
|
||||
FloatT = Types(16, name = 'float', cname = 'float', sqlname = 'REAL',
|
||||
long_type = DoubleT, is_fp = True)
|
||||
HgeT = Types(9, name = 'int128',cname='__int128_t', sqlname = 'HUGEINT', fp_type = DoubleT)
|
||||
UHgeT = Types(10, name = 'uint128', cname='__uint128_t', sqlname = 'HUGEINT', fp_type = DoubleT)
|
||||
LongT = Types(4, name = 'int64', sqlname = 'BIGINT', fp_type = DoubleT)
|
||||
BoolT = Types(0, name = 'bool', cname='bool', sqlname = 'BOOL', long_type=LongT, fp_type=FloatT)
|
||||
ByteT = Types(1, name = 'int8', sqlname = 'TINYINT', long_type=LongT, fp_type=FloatT)
|
||||
ShortT = Types(2, name = 'int16', sqlname='SMALLINT', long_type=LongT, fp_type=FloatT)
|
||||
IntT = Types(3, name = 'int', cname = 'int', long_type=LongT, fp_type=FloatT)
|
||||
ULongT = Types(8, name = 'uint64', sqlname = 'UINT64', fp_type=DoubleT)
|
||||
UIntT = Types(7, name = 'uint32', sqlname = 'UINT32', long_type=ULongT, fp_type=FloatT)
|
||||
UShortT = Types(6, name = 'uint16', sqlname = 'UINT16', long_type=ULongT, fp_type=FloatT)
|
||||
UByteT = Types(5, name = 'uint8', sqlname = 'UINT8', long_type=ULongT, fp_type=FloatT)
|
||||
StrT = Types(200, name = 'str', cname = 'string_view', sqlname='TEXT', ctype_name = 'types::ASTR')
|
||||
TextT = Types(200, name = 'text', cname = 'string_view', sqlname='TEXT', ctype_name = 'types::ASTR')
|
||||
VarcharT = Types(200, name = 'varchar', cname = 'string_view', sqlname='VARCHAR', ctype_name = 'types::ASTR')
|
||||
VoidT = Types(200, name = 'void', cname = 'void', sqlname='Null', ctype_name = 'types::None')
|
||||
|
||||
class VectorT(Types):
|
||||
def __init__(self, inner_type : Types, vector_type:str = 'vector_type'):
|
||||
self.inner_type = inner_type
|
||||
self.vector_type = vector_type
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return f'{self.vector_type}<{self.inner_type.name}>'
|
||||
@property
|
||||
def sqlname(self) -> str:
|
||||
return 'HUGEINT' # Store vector_type into 16 bit integers
|
||||
@property
|
||||
def cname(self) -> str:
|
||||
return f'{self.vector_type}<{self.inner_type.cname}>'
|
||||
@property
|
||||
def fp_type(self) -> Types:
|
||||
return VectorT(self.inner_type.fp_type, self.vector_type)
|
||||
@property
|
||||
def long_type(self):
|
||||
return VectorT(self.inner_type.long_type, self.vector_type)
|
||||
|
||||
|
||||
def _ty_make_dict(fn : str, *ty : Types):
|
||||
return {eval(fn):t for t in ty}
|
||||
|
||||
int_types : Dict[str, Types] = _ty_make_dict('t.sqlname.lower()', LongT, ByteT, ShortT, IntT)
|
||||
uint_types : Dict[str, Types] = _ty_make_dict('t.sqlname.lower()', ULongT, UByteT, UShortT, UIntT)
|
||||
fp_types : Dict[str, Types] = _ty_make_dict('t.sqlname.lower()', FloatT, DoubleT)
|
||||
temporal_types : Dict[str, Types] = _ty_make_dict('t.sqlname.lower()', DateT, TimeT, TimeStampT)
|
||||
builtin_types : Dict[str, Types] = {
|
||||
'string' : StrT,
|
||||
**_ty_make_dict('t.sqlname.lower()', AnyT, TextT, VarcharT, HgeT),
|
||||
**int_types, **fp_types, **temporal_types}
|
||||
|
||||
def get_int128_support():
|
||||
for t in int_types.values():
|
||||
t.long_type = HgeT
|
||||
for t in uint_types.values():
|
||||
t.long_type = UHgeT
|
||||
int_types['int128'] = HgeT
|
||||
uint_types['uint128'] = UHgeT
|
||||
|
||||
def revert_int128_support():
|
||||
for t in int_types.values():
|
||||
t.long_type = LongT
|
||||
for t in uint_types.values():
|
||||
t.long_type = ULongT
|
||||
int_types.pop('int128', None)
|
||||
uint_types.pop('uint128', None)
|
||||
|
||||
|
||||
type_bylength : Dict[int, TypeCollection] = {}
|
||||
type_bylength[1] = TypeCollection(1, ByteT)
|
||||
type_bylength[2] = TypeCollection(2, ShortT)
|
||||
type_bylength[4] = TypeCollection(4, IntT, FloatT)
|
||||
type_bylength[8] = TypeCollection(8, LongT, DoubleT, collection=[AnyT])
|
||||
|
||||
class OperatorBase:
|
||||
def extending_type(ops:Types):
|
||||
return ops.long_type
|
||||
def fraction_type (ops:Types):
|
||||
return ops.fp_type
|
||||
def __init__(self, opname, n_ops, return_fx, * ,
|
||||
optypes = None, cname = None, sqlname = None,
|
||||
call = None):
|
||||
self.name = opname
|
||||
self.cname = defval(cname, opname)
|
||||
self.sqlname = defval(sqlname, opname.upper())
|
||||
self.n_ops = n_ops
|
||||
self.optypes = optypes
|
||||
self.return_type = defval(return_fx, lambda: self.optypes[0])
|
||||
self.call = defval(call, lambda _, c_code = False, *args:
|
||||
f'{self.cname if c_code else self.sqlname}({", ". join(args)})')
|
||||
def __call__(self, c_code = False, *args) -> str:
|
||||
return self.call(self, c_code, *args)
|
||||
|
||||
def get_return_type(self, *inputs):
|
||||
return self.return_type(*inputs)
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return self.name
|
||||
def __str__(self) -> str:
|
||||
return self.name
|
||||
|
||||
# TODO: Type checks, Type catagories, e.g.: value type, etc.
|
||||
|
||||
|
||||
# return type deduction
|
||||
def auto_extension(*args : Types) -> Types:
|
||||
final_type = AnyT
|
||||
is_fp = False
|
||||
for a in args:
|
||||
if not is_fp and a.is_fp:
|
||||
is_fp = True
|
||||
final_type = final_type.fp_type
|
||||
elif is_fp:
|
||||
a = a.fp_type
|
||||
final_type = a if a.priority > final_type.priority else final_type
|
||||
return final_type
|
||||
|
||||
def auto_extension_int(*args : Types) -> Types:
|
||||
final_type = AnyT
|
||||
for a in args:
|
||||
final_type = a if a.priority > final_type.priority else final_type
|
||||
return final_type
|
||||
def ty_clamp(fn, l:int = None, r:int = None):
|
||||
return lambda *args : fn(*args[l: r])
|
||||
def logical(*_ : Types) -> Types:
|
||||
return ByteT
|
||||
def int_return(*_ : Types) -> Types:
|
||||
return IntT
|
||||
def long_return(*_ : Types) -> Types:
|
||||
return LongT.long_type
|
||||
def lfp_return(*_ : Types) -> Types:
|
||||
return LongT.fp_type
|
||||
def pack_return(*args : Types) -> Types:
|
||||
if len(args) == 0:
|
||||
raise ValueError('0 arguments in pack expression')
|
||||
inner_ty = args[0]
|
||||
for a in args:
|
||||
if a != inner_ty:
|
||||
raise ValueError('pack expression with different types')
|
||||
if isinstance(inner_ty, VectorT):
|
||||
print('warning: packing vector types')
|
||||
inner_ty = inner_ty.inner_type
|
||||
return VectorT(args[0])
|
||||
def as_is (t: Types) -> Types:
|
||||
return t
|
||||
|
||||
def fp (fx):
|
||||
return lambda *args : fx(*args).fp_type
|
||||
def ext (fx):
|
||||
return lambda *args : fx(*args).long_type
|
||||
|
||||
|
||||
# operator call behavior
|
||||
def binary_op_behavior(op:OperatorBase, c_code, *xs):
|
||||
name = op.cname if c_code else op.sqlname
|
||||
return f'({f" {name} ".join(xs)})'
|
||||
|
||||
def unary_op_behavior(op:OperatorBase, c_code, x):
|
||||
name = op.cname if c_code else op.sqlname
|
||||
return f'({name} {x})'
|
||||
|
||||
def fn_behavior(op:OperatorBase, c_code, *x):
|
||||
name = op.cname if c_code else op.sqlname
|
||||
return f'{name}({", ".join([f"{xx}" for xx in x])})'
|
||||
|
||||
def count_behavior(op:OperatorBase, c_code, x, distinct = False):
|
||||
if not c_code:
|
||||
return f'{op.sqlname}({"distinct " if distinct else ""}{x})'
|
||||
elif distinct:
|
||||
return f'({x}).distinct_size()'
|
||||
else:
|
||||
return '{count()}'
|
||||
|
||||
def distinct_behavior(op:OperatorBase, c_code, x):
|
||||
if not c_code:
|
||||
return f'{op.sqlname}({x})'
|
||||
else:
|
||||
return f'({x}).distinct()'
|
||||
|
||||
def windowed_fn_behavor(op: OperatorBase, c_code, *x):
|
||||
if not c_code:
|
||||
return f'{op.sqlname}({", ".join([f"{xx}" for xx in x])})'
|
||||
else:
|
||||
name = op.cname if len(x) == 1 else op.cname[:-1] + 'w'
|
||||
return f'{name}({", ".join([f"{xx}" for xx in x])})'
|
||||
|
||||
def pack_behavior(op: OperatorBase, c_code, *x):
|
||||
if len(x) == 0:
|
||||
raise ValueError('0 arguments in pack expression')
|
||||
|
||||
if not c_code:
|
||||
return f'{op.sqlname}({", ".join([f"{xx}" for xx in x])})'
|
||||
else:
|
||||
return f'decltype({x[0]})::pack({len(x)}, {", ".join([f"{xx}.s()" for xx in x])})'
|
||||
|
||||
# arithmetic
|
||||
opadd = OperatorBase('add', 2, auto_extension, cname = '+', sqlname = '+', call = binary_op_behavior)
|
||||
# monetdb wont extend int division to fp type
|
||||
# opdiv = OperatorBase('div', 2, fp(auto_extension), cname = '/', sqlname = '/', call = binary_op_behavior)
|
||||
opdiv = OperatorBase('div', 2, auto_extension, cname = '/', sqlname = '/', call = binary_op_behavior)
|
||||
opmul = OperatorBase('mul', 2, auto_extension, cname = '*', sqlname = '*', call = binary_op_behavior)
|
||||
opsub = OperatorBase('sub', 2, auto_extension, cname = '-', sqlname = '-', call = binary_op_behavior)
|
||||
opmod = OperatorBase('mod', 2, auto_extension_int, cname = '%', sqlname = '%', call = binary_op_behavior)
|
||||
opneg = OperatorBase('neg', 1, as_is, cname = '-', sqlname = '-', call = unary_op_behavior)
|
||||
# logical
|
||||
opand = OperatorBase('and', 2, logical, cname = '&&', sqlname = ' AND ', call = binary_op_behavior)
|
||||
opor = OperatorBase('or', 2, logical, cname = '||', sqlname = ' OR ', call = binary_op_behavior)
|
||||
opxor = OperatorBase('xor', 2, logical, cname = '^', sqlname = ' XOR ', call = binary_op_behavior)
|
||||
opgt = OperatorBase('gt', 2, logical, cname = '>', sqlname = '>', call = binary_op_behavior)
|
||||
oplt = OperatorBase('lt', 2, logical, cname = '<', sqlname = '<', call = binary_op_behavior)
|
||||
opgte = OperatorBase('gte', 2, logical, cname = '>=', sqlname = '>=', call = binary_op_behavior)
|
||||
oplte = OperatorBase('lte', 2, logical, cname = '<=', sqlname = '<=', call = binary_op_behavior)
|
||||
opneq = OperatorBase('neq', 2, logical, cname = '!=', sqlname = '!=', call = binary_op_behavior)
|
||||
opeq = OperatorBase('eq', 2, logical, cname = '==', sqlname = '=', call = binary_op_behavior)
|
||||
opnot = OperatorBase('not', 1, logical, cname = '!', sqlname = 'NOT', call = unary_op_behavior)
|
||||
opdistinct = OperatorBase('distinct', 1, as_is, cname = '.distinct()', sqlname = 'distinct', call = distinct_behavior)
|
||||
# functional
|
||||
fnprev = OperatorBase('prev', 1, as_is, cname = 'prev', sqlname = 'PREV', call = fn_behavior)
|
||||
fnnext = OperatorBase('next', 1, as_is, cname = 'aggnext', sqlname = 'NEXT', call = fn_behavior)
|
||||
fnmax = OperatorBase('max', 1, as_is, cname = 'max', sqlname = 'MAX', call = fn_behavior)
|
||||
fnmin = OperatorBase('min', 1, as_is, cname = 'min', sqlname = 'MIN', call = fn_behavior)
|
||||
fndeltas = OperatorBase('deltas', 1, as_is, cname = 'deltas', sqlname = 'DELTAS', call = fn_behavior)
|
||||
fnratios = OperatorBase('ratios', [1, 2], fp(ty_clamp(as_is, -1)), cname = 'ratios', sqlname = 'RATIOS', call = windowed_fn_behavor)
|
||||
fnlast = OperatorBase('last', 1, as_is, cname = 'last', sqlname = 'LAST', call = fn_behavior)
|
||||
fnfirst = OperatorBase('first', 1, as_is, cname = 'frist', sqlname = 'FRIST', call = fn_behavior)
|
||||
#fnsum = OperatorBase('sum', 1, ext(auto_extension), cname = 'sum', sqlname = 'SUM', call = fn_behavior)
|
||||
#fnavg = OperatorBase('avg', 1, fp(ext(auto_extension)), cname = 'avg', sqlname = 'AVG', call = fn_behavior)
|
||||
fnsum = OperatorBase('sum', 1, long_return, cname = 'sum', sqlname = 'SUM', call = fn_behavior)
|
||||
fnavg = OperatorBase('avg', 1, lfp_return, cname = 'avg', sqlname = 'AVG', call = fn_behavior)
|
||||
fnvar = OperatorBase('var', 1, lfp_return, cname = 'var', sqlname = 'VAR_POP', call = fn_behavior)
|
||||
fnstd = OperatorBase('stddev', 1, lfp_return, cname = 'stddev', sqlname = 'STDDEV_POP', call = fn_behavior)
|
||||
fnmaxs = OperatorBase('maxs', [1, 2], ty_clamp(as_is, -1), cname = 'maxs', sqlname = 'MAXS', call = windowed_fn_behavor)
|
||||
fnmins = OperatorBase('mins', [1, 2], ty_clamp(as_is, -1), cname = 'mins', sqlname = 'MINS', call = windowed_fn_behavor)
|
||||
fnsums = OperatorBase('sums', [1, 2], ext(ty_clamp(auto_extension, -1)), cname = 'sums', sqlname = 'SUMS', call = windowed_fn_behavor)
|
||||
fnavgs = OperatorBase('avgs', [1, 2], fp(ext(ty_clamp(auto_extension, -1))), cname = 'avgs', sqlname = 'AVGS', call = windowed_fn_behavor)
|
||||
fnvars = OperatorBase('vars', [1, 2], fp(ext(ty_clamp(auto_extension, -1))), cname = 'vars', sqlname = 'VARS', call = windowed_fn_behavor)
|
||||
fnstds = OperatorBase('stddevs', [1, 2], fp(ext(ty_clamp(auto_extension, -1))), cname = 'stddevs', sqlname = 'STDDEVS', call = windowed_fn_behavor)
|
||||
fncnt = OperatorBase('count', 1, int_return, cname = 'count', sqlname = 'COUNT', call = count_behavior)
|
||||
fnpack = OperatorBase('pack', -1, pack_return, cname = 'pack', sqlname = 'PACK', call = pack_behavior)
|
||||
# special
|
||||
def is_null_call_behavior(op:OperatorBase, c_code : bool, x : str):
|
||||
if c_code :
|
||||
return f'{x} == nullval<decays<decltype({x})>>'
|
||||
else :
|
||||
return f'{x} IS NULL'
|
||||
spnull = OperatorBase('missing', 1, logical, cname = "", sqlname = "", call = is_null_call_behavior)
|
||||
|
||||
# cstdlib
|
||||
# If in aggregation functions, using monetdb builtins. If in nested agg, inside udfs, using cstdlib.
|
||||
fntrunc = OperatorBase('truncate', 2, ty_clamp(as_is, 0, 1), cname = 'truncate', sqlname = 'TRUNCATE', call = fn_behavior)
|
||||
fnsqrt = OperatorBase('sqrt', 1, lambda *_ : DoubleT, cname = 'sqrt', sqlname = 'SQRT', call = fn_behavior)
|
||||
fnlog = OperatorBase('log', 2, lambda *_ : DoubleT, cname = 'log', sqlname = 'LOG', call = fn_behavior)
|
||||
fnsin = OperatorBase('sin', 1, lambda *_ : DoubleT, cname = 'sin', sqlname = 'SIN', call = fn_behavior)
|
||||
fncos = OperatorBase('cos', 1, lambda *_ : DoubleT, cname = 'cos', sqlname = 'COS', call = fn_behavior)
|
||||
fntan = OperatorBase('tan', 1, lambda *_ : DoubleT, cname = 'tan', sqlname = 'TAN', call = fn_behavior)
|
||||
fnpow = OperatorBase('pow', 2, lambda *_ : DoubleT, cname = 'pow', sqlname = 'POW', call = fn_behavior)
|
||||
|
||||
# type collections
|
||||
def _op_make_dict(*items : OperatorBase):
|
||||
return { i.name: i for i in items}
|
||||
#binary op
|
||||
builtin_binary_arith = _op_make_dict(opadd, opdiv, opmul, opsub, opmod)
|
||||
builtin_binary_logical = _op_make_dict(opand, opor, opxor, opgt, oplt,
|
||||
opgte, oplte, opneq, opeq)
|
||||
builtin_binary_ops = {**builtin_binary_arith, **builtin_binary_logical}
|
||||
#unary op
|
||||
builtin_unary_logical = _op_make_dict(opnot)
|
||||
builtin_unary_arith = _op_make_dict(opneg)
|
||||
builtin_unary_special = _op_make_dict(spnull, opdistinct)
|
||||
# functions
|
||||
builtin_cstdlib = _op_make_dict(fnsqrt, fnlog, fnsin, fncos, fntan, fnpow)
|
||||
builtin_aggfunc = _op_make_dict(fnmax, fnmin, fnsum, fnavg,
|
||||
fnlast, fnfirst, fncnt, fnvar, fnstd)
|
||||
builtin_vecfunc = _op_make_dict(fnmaxs,
|
||||
fnmins, fndeltas, fnratios, fnsums, fnavgs,
|
||||
fnpack, fntrunc, fnprev, fnnext, fnvars, fnstds)
|
||||
builtin_vecfunc = {**builtin_vecfunc, **builtin_cstdlib}
|
||||
builtin_func = {**builtin_vecfunc, **builtin_aggfunc}
|
||||
|
||||
user_module_func = {}
|
||||
|
||||
builtin_operators : Dict[str, OperatorBase] = {**builtin_binary_arith, **builtin_binary_logical,
|
||||
**builtin_unary_arith, **builtin_unary_logical, **builtin_unary_special, **builtin_func, **builtin_cstdlib,
|
||||
**user_module_func}
|
||||
|
||||
type_table = {**builtin_types, **type_table}
|
||||
|
||||
# Additional Aliases for type names
|
||||
type_table['boolean'] = BoolT
|
||||
-192
@@ -1,192 +0,0 @@
|
||||
import uuid
|
||||
from collections import OrderedDict
|
||||
from collections.abc import Mapping, MutableMapping
|
||||
|
||||
lower_alp = 'abcdefghijklmnopqrstuvwxyz'
|
||||
upper_alp = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
|
||||
nums = '0123456789'
|
||||
base62alp = nums + lower_alp + upper_alp
|
||||
|
||||
reserved_monet = ['month']
|
||||
session_context = None
|
||||
|
||||
class CaseInsensitiveDict(MutableMapping):
|
||||
def __init__(self, data=None, **kwargs):
|
||||
self._store = OrderedDict()
|
||||
if data is None:
|
||||
data = {}
|
||||
self.update(data, **kwargs)
|
||||
|
||||
def __setitem__(self, key, value):
|
||||
# Use the lowercased key for lookups, but store the actual
|
||||
# key alongside the value.
|
||||
self._store[key.lower()] = (key, value)
|
||||
|
||||
def __getitem__(self, key):
|
||||
return self._store[key.lower()][1]
|
||||
|
||||
def __delitem__(self, key):
|
||||
del self._store[key.lower()]
|
||||
|
||||
def __iter__(self):
|
||||
return (casedkey for casedkey, mappedvalue in self._store.values())
|
||||
|
||||
def __len__(self):
|
||||
return len(self._store)
|
||||
|
||||
def lower_items(self):
|
||||
"""Like iteritems(), but with all lowercase keys."""
|
||||
return ((lowerkey, keyval[1]) for (lowerkey, keyval) in self._store.items())
|
||||
|
||||
def __eq__(self, other):
|
||||
if isinstance(other, Mapping):
|
||||
other = CaseInsensitiveDict(other)
|
||||
else:
|
||||
return NotImplemented
|
||||
# Compare insensitively
|
||||
return dict(self.lower_items()) == dict(other.lower_items())
|
||||
|
||||
# Copy is required
|
||||
def copy(self):
|
||||
return CaseInsensitiveDict(self._store.values())
|
||||
|
||||
def __repr__(self):
|
||||
return str(dict(self.items()))
|
||||
|
||||
def base62uuid(crop=8):
|
||||
_id = uuid.uuid4().int
|
||||
ret = ''
|
||||
|
||||
while _id:
|
||||
ret = base62alp[_id % 62] + ret
|
||||
_id //= 62
|
||||
|
||||
return ret[:crop] if len(ret) else '0'
|
||||
|
||||
def get_legal_name(name, lower = True):
|
||||
if name is not None:
|
||||
if lower:
|
||||
name = name.lower()
|
||||
name = ''.join([n for n in name if n in base62alp or n == '_'])
|
||||
|
||||
if name is None or len(name) == 0 or set(name) == set('_'):
|
||||
name = base62uuid(8)
|
||||
if(name[0] in nums):
|
||||
name = '_' + name
|
||||
|
||||
return name
|
||||
|
||||
def check_legal_name(name):
|
||||
all_underscores = True
|
||||
for c in name:
|
||||
if c not in base62alp and c != '_':
|
||||
return False
|
||||
if c != '_':
|
||||
all_underscores = False
|
||||
if all_underscores:
|
||||
return False
|
||||
if name[0] in nums:
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
def enlist(l):
|
||||
return l if type(l) is list else [l]
|
||||
|
||||
def seps(s, i, l):
|
||||
return s if i < len(l) - 1 else ''
|
||||
|
||||
def has_other(a, b):
|
||||
for ai in a:
|
||||
if ai not in b:
|
||||
return True
|
||||
return False
|
||||
|
||||
def defval(val, default):
|
||||
return default if val is None else val
|
||||
|
||||
# escape must be readonly
|
||||
from typing import Mapping, Set
|
||||
|
||||
|
||||
def remove_last(pattern : str, string : str, escape : Set[str] = set()) -> str:
|
||||
idx = string.rfind(pattern)
|
||||
if idx == -1:
|
||||
return string
|
||||
else:
|
||||
if set(string[idx:]).difference(escape):
|
||||
return string
|
||||
else:
|
||||
return string[:idx] + string[idx+1:]
|
||||
|
||||
class _Counter:
|
||||
def __init__(self, cnt):
|
||||
self.cnt = cnt
|
||||
def inc(self, cnt = 1):
|
||||
self.cnt += cnt
|
||||
cnt = self.cnt - cnt
|
||||
return cnt
|
||||
|
||||
import re
|
||||
|
||||
ws = re.compile(r'\s+')
|
||||
import os
|
||||
|
||||
|
||||
def add_dll_dir(dll: str):
|
||||
import sys
|
||||
if sys.version_info.major >= 3 and sys.version_info.minor >7 and os.name == 'nt':
|
||||
os.add_dll_directory(dll)
|
||||
else:
|
||||
os.environ['PATH'] = os.path.abspath(dll) + os.pathsep + os.environ['PATH']
|
||||
|
||||
nullstream = open(os.devnull, 'w')
|
||||
|
||||
|
||||
def clamp(val, minval, maxval):
|
||||
return min(max(val, minval), maxval)
|
||||
|
||||
def escape_qoutes(string : str):
|
||||
return re.sub(r'^\'', r'\'',re.sub(r'([^\\])\'', r'\1\'', string))
|
||||
|
||||
def get_innermost(sl):
|
||||
if sl and type(sl) is dict:
|
||||
if 'literal' in sl and type(sl['literal']) is str:
|
||||
return f"'{get_innermost(sl['literal'])}'"
|
||||
return get_innermost(next(iter(sl.values()), None))
|
||||
elif sl and type(sl) is list:
|
||||
return get_innermost(sl[0])
|
||||
else:
|
||||
return sl
|
||||
|
||||
|
||||
def send_to_server(payload : str):
|
||||
from prompt import PromptState
|
||||
cxt : PromptState = session_context
|
||||
if cxt is None:
|
||||
raise RuntimeError("Error! no session specified.")
|
||||
else:
|
||||
from ctypes import c_char_p
|
||||
cxt.payload = (c_char_p*1)(c_char_p(bytes(payload, 'utf-8')))
|
||||
cxt.cfg.has_dll = 0
|
||||
cxt.send(1, cxt.payload)
|
||||
cxt.set_ready()
|
||||
|
||||
def get_storedproc(name : str):
|
||||
from prompt import PromptState, StoredProcedure
|
||||
cxt : PromptState = session_context
|
||||
if cxt is None:
|
||||
raise RuntimeError("Error! no session specified.")
|
||||
else:
|
||||
ret : StoredProcedure = cxt.get_storedproc(bytes(name, 'utf-8'))
|
||||
if (
|
||||
ret.name and
|
||||
ret.name.decode('utf-8') != name
|
||||
):
|
||||
print(f'Procedure {name} mismatch in server {ret.name.value}')
|
||||
return None
|
||||
else:
|
||||
return ret
|
||||
|
||||
def execute_procedure(proc):
|
||||
pass
|
||||
Reference in New Issue
Block a user