udf support for monetdbe backend
This commit is contained in:
+30
-5
@@ -1,12 +1,37 @@
|
||||
from engine.ast import Context, ast_node
|
||||
import engine.ddl, engine.projection
|
||||
saved_cxt = None
|
||||
|
||||
def initialize():
|
||||
return Context()
|
||||
def initialize(cxt = None, keep = False):
|
||||
global saved_cxt
|
||||
if cxt is None or not keep or type(cxt) is not Context:
|
||||
if saved_cxt is None or not keep:
|
||||
cxt = Context()
|
||||
saved_cxt = cxt
|
||||
else:
|
||||
cxt = saved_cxt
|
||||
cxt.new()
|
||||
|
||||
return cxt
|
||||
|
||||
def generate(ast, cxt):
|
||||
for k in ast.keys():
|
||||
if k in ast_node.types.keys():
|
||||
root = ast_node.types[k](None, ast, cxt)
|
||||
|
||||
__all__ = ["initialize", "generate"]
|
||||
|
||||
def exec(stmts, cxt = None, keep = None):
|
||||
cxt = initialize(cxt, keep)
|
||||
stmts_stmts = stmts['stmts']
|
||||
if type(stmts_stmts) is list:
|
||||
for s in stmts_stmts:
|
||||
generate(s, cxt)
|
||||
else:
|
||||
generate(stmts_stmts, cxt)
|
||||
|
||||
cxt.Info(cxt.ccode)
|
||||
with open('out.cpp', 'wb') as outfile:
|
||||
outfile.write((cxt.finalize()).encode('utf-8'))
|
||||
|
||||
return cxt
|
||||
|
||||
|
||||
__all__ = ["initialize", "generate", "exec", "saved_cxt"]
|
||||
|
||||
+9
-5
@@ -1,5 +1,6 @@
|
||||
from engine.utils import base62uuid
|
||||
from copy import copy
|
||||
from typing import *
|
||||
# replace column info with this later.
|
||||
class ColRef:
|
||||
def __init__(self, cname, _ty, cobj, cnt, table:'TableInfo', name, id, compound = False):
|
||||
@@ -65,7 +66,7 @@ class TableInfo:
|
||||
self.views = set()
|
||||
#keep track of temp vars
|
||||
self.local_vars = dict()
|
||||
self.rec = None
|
||||
self.rec = None # a hook on get_col_d to record tables being referenced in the process
|
||||
self.groupinfo = None
|
||||
self.add_cols(cols)
|
||||
# runtime
|
||||
@@ -207,14 +208,17 @@ class Context:
|
||||
LOG_INFO = 'INFO'
|
||||
LOG_ERROR = 'ERROR'
|
||||
LOG_SILENT = 'SILENT'
|
||||
from engine.types import Types
|
||||
type_table : Dict[str, Types] = dict()
|
||||
|
||||
def new(self):
|
||||
self.tmp_names = set()
|
||||
self.udf_map = dict()
|
||||
self.headers = set(['\"./server/libaquery.h\"'])
|
||||
self.finalized = False
|
||||
# read header
|
||||
self.ccode = ''
|
||||
self.ccodelet = ''
|
||||
self.ccode = str()
|
||||
self.ccodelet = str()
|
||||
with open('header.cxx', 'r') as outfile:
|
||||
self.ccode = outfile.read()
|
||||
# datasource will be availible after `from' clause is parsed
|
||||
@@ -236,8 +240,8 @@ class Context:
|
||||
self.log_level = Context.LOG_SILENT
|
||||
self.print = print
|
||||
# read header
|
||||
self.ccode = ''
|
||||
self.ccodelet = ''
|
||||
self.ccode = str()
|
||||
self.ccodelet = str()
|
||||
self.columns_in_context = dict()
|
||||
self.tables_in_context = dict()
|
||||
with open('header.cxx', 'r') as outfile:
|
||||
|
||||
+5
-2
@@ -28,11 +28,14 @@ class expr(ast_node):
|
||||
'xor' : '^',
|
||||
'gt':'>',
|
||||
'lt':'<',
|
||||
'le':'<=',
|
||||
'gt':'>='
|
||||
'lte':'<=',
|
||||
'gte':'>=',
|
||||
'neq':'!=',
|
||||
'eq':'=='
|
||||
}
|
||||
|
||||
compound_ops = {
|
||||
'missing' : ['missing', lambda x: f'{x[0]} == nullval<decays<decltype({x[0]})>>'],
|
||||
}
|
||||
|
||||
unary_ops = {
|
||||
|
||||
+222
-26
@@ -1,34 +1,230 @@
|
||||
from engine.ast import Context
|
||||
from engine.utils import defval
|
||||
from typing import Dict, List
|
||||
|
||||
type_table: Dict[str, "Types"] = {}
|
||||
|
||||
class Types:
|
||||
name = 'Any'
|
||||
cname = 'void*'
|
||||
ctype_name = "types::NONE"
|
||||
def __init__(self, context:Context):
|
||||
self.cxt = context
|
||||
def cast_to(self, *_):
|
||||
return self
|
||||
def cast_from(self, *_):
|
||||
return self
|
||||
def init_any(self):
|
||||
self.name : str = 'Any'
|
||||
self.sqlname : str = 'Int'
|
||||
self.cname : str = 'void*'
|
||||
self.ctype_name : str = "types::NONE"
|
||||
self.null_value = 0
|
||||
self.priority : int= 0
|
||||
self.cast_to_dict = dict()
|
||||
self.cast_from_dict = dict()
|
||||
def __init__(self, priority = 0, *,
|
||||
name = None, cname = None, sqlname = None,
|
||||
ctype_name = None, null_value = None,
|
||||
fp_type = None, long_type = None, is_fp = False,
|
||||
cast_to = None, cast_from = None
|
||||
):
|
||||
|
||||
self.is_fp = is_fp
|
||||
if name is None:
|
||||
self.init_any()
|
||||
else:
|
||||
self.name = name
|
||||
self.cname = defval(cname, name.lower() + '_t')
|
||||
self.sqlname = defval(sqlname, name.upper())
|
||||
self.ctype_name = defval(ctype_name, f'types::{name.upper()}')
|
||||
self.null_value = defval(null_value, 0)
|
||||
self.cast_to_dict = defval(cast_to, dict())
|
||||
self.cast_from_dict = defval(cast_from, dict())
|
||||
self.priority = priority
|
||||
|
||||
self.long_type = defval(long_type, self)
|
||||
self.fp_type = defval(fp_type, self)
|
||||
|
||||
global type_table
|
||||
type_table[name] = self
|
||||
|
||||
def cast_to(self, ty : "Types"):
|
||||
if ty in self.cast_to_dict:
|
||||
return self.cast_to_dict[ty.name](ty)
|
||||
else:
|
||||
raise Exception(f'Illeagal cast: from {self.name} to {ty.name}.')
|
||||
def cast_from(self, ty : "Types"):
|
||||
if ty in self.cast_from_dict:
|
||||
return self.cast_from_dict[ty.name](ty)
|
||||
else:
|
||||
raise Exception(f'Illeagal cast: from {ty.name} to {self.name}.')
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return self.cname
|
||||
return self.sqlname
|
||||
def __str__(self) -> str:
|
||||
return self.sqlname
|
||||
class TypeCollection:
|
||||
def __init__(self, sz, deftype, fptype = None, utype = None, *, collection = None) -> None:
|
||||
self.size = sz
|
||||
self.type = deftype
|
||||
self.fptype = fptype
|
||||
self.utype = utype
|
||||
self.all_types = [deftype]
|
||||
if fptype is not None:
|
||||
self.all_types.append(fptype)
|
||||
if utype is not None:
|
||||
self.all_types.append(utype)
|
||||
if collection is not None:
|
||||
for ty in collection:
|
||||
self.all_types.append(ty)
|
||||
|
||||
type_table = dict()
|
||||
AnyT = Types(0)
|
||||
LazyT = Types(240, name = 'Lazy', cname = '', sqlname = '', ctype_name = '')
|
||||
DoubleT = Types(17, name = 'double', cname='double', sqlname = 'DOUBLE', is_fp = True)
|
||||
FloatT = Types(16, name = 'float', cname = 'float', sqlname = 'REAL',
|
||||
long_type = DoubleT, is_fp = True)
|
||||
LongT = Types(4, name = 'int64', sqlname = 'BIGINT', fp_type = DoubleT)
|
||||
ByteT = Types(1, name = 'int8', sqlname = 'TINYINT', long_type=LongT, fp_type=FloatT)
|
||||
ShortT = Types(2, name = 'int16', sqlname='SMALLINT', long_type=LongT, fp_type=FloatT)
|
||||
IntT = Types(3, name = 'int', cname = 'int', long_type=LongT, fp_type=FloatT)
|
||||
StrT = Types(200, name = 'str', cname = 'const char*', sqlname='VARCHAR', ctype_name = 'types::STRING')
|
||||
def _ty_make_dict(fn : str, *ty : Types):
|
||||
return {eval(fn):t for t in ty}
|
||||
|
||||
class String(Types):
|
||||
name = 'String'
|
||||
cname = 'const char*'
|
||||
ctype_name = "types::ASTR"
|
||||
def cast_from(self, ty, val, container = None):
|
||||
if type(ty) is Int:
|
||||
self.cxt.emit()
|
||||
int_types : Dict[str, Types] = _ty_make_dict('t.sqlname.lower()', LongT, ByteT, ShortT, IntT)
|
||||
fp_types : Dict[str, Types] = _ty_make_dict('t.sqlname.lower()', FloatT, DoubleT)
|
||||
builtin_types : Dict[str, Types] = {**_ty_make_dict('t.sqlname.lower()', AnyT, StrT), **int_types, **fp_types}
|
||||
|
||||
class Int(Types):
|
||||
name = "Int"
|
||||
cname = "int"
|
||||
ctype_name = "types::AINT"
|
||||
type_bylength : Dict[int, TypeCollection] = {}
|
||||
type_bylength[1] = TypeCollection(1, ByteT)
|
||||
type_bylength[2] = TypeCollection(2, ShortT)
|
||||
type_bylength[4] = TypeCollection(4, IntT, FloatT)
|
||||
type_bylength[8] = TypeCollection(8, LongT, DoubleT, collection=[AnyT])
|
||||
|
||||
class OperatorBase:
|
||||
def extending_type(ops:Types):
|
||||
return ops.long_type
|
||||
def fraction_type (ops:Types):
|
||||
return ops.fp_type
|
||||
def __init__(self, opname, n_ops, return_fx, * ,
|
||||
optypes = None, cname = None, sqlname = None,
|
||||
call = None):
|
||||
self.name = opname
|
||||
self.cname = defval(cname, opname)
|
||||
self.sqlname = defval(sqlname, opname.upper())
|
||||
self.n_ops = n_ops
|
||||
self.optypes = optypes
|
||||
self.return_type = defval(return_fx, lambda: self.optypes[0])
|
||||
self.call = defval(call, lambda _, c_code = False, *args:
|
||||
f'{self.cname if c_code else self.sqlname}({", ". join(args)})')
|
||||
def __call__(self, c_code = False, *args) -> str:
|
||||
return self.call(self, c_code, *args)
|
||||
|
||||
def get_return_type(self, inputs):
|
||||
return self.return_type(inputs)
|
||||
|
||||
class Float(Types):
|
||||
name = "Float"
|
||||
cname = "float"
|
||||
ctype_name = "types::AFLOAT"
|
||||
def __repr__(self) -> str:
|
||||
return self.name
|
||||
def __str__(self) -> str:
|
||||
return self.name
|
||||
|
||||
# TODO: Type checks, Type catagories, e.g.: value type, etc.
|
||||
|
||||
|
||||
# return type deduction
|
||||
def auto_extension(*args : Types) -> Types:
|
||||
final_type = AnyT
|
||||
is_fp = False
|
||||
for a in args:
|
||||
if not is_fp and a.is_fp:
|
||||
is_fp = True
|
||||
final_type = final_type.fp_type
|
||||
elif is_fp:
|
||||
a = a.fp_type
|
||||
final_type = a if a.priority > final_type.priority else final_type
|
||||
return final_type
|
||||
|
||||
def auto_extension_int(*args : Types) -> Types:
|
||||
final_type = AnyT
|
||||
for a in args:
|
||||
final_type = a if a.priority > final_type.priority else final_type
|
||||
return final_type
|
||||
def ty_clamp(fn, l:int = None, r:int = None):
|
||||
return lambda *args : fn(*args[l: r])
|
||||
def logical(*_ : Types) -> Types:
|
||||
return ByteT
|
||||
def int_return(*_ : Types) -> Types:
|
||||
return IntT
|
||||
def as_is (t: Types) -> Types:
|
||||
return t
|
||||
|
||||
def fp (fx):
|
||||
return lambda *args : fx(*args).fp_type
|
||||
def ext (fx):
|
||||
return lambda *args : fx(*args).long_type
|
||||
|
||||
|
||||
# operator call behavior
|
||||
def binary_op_behavior(op:OperatorBase, c_code, x, y):
|
||||
name = op.cname if c_code else op.sqlname
|
||||
return f'({x} {name} {y})'
|
||||
|
||||
def unary_op_behavior(op:OperatorBase, c_code, x):
|
||||
name = op.cname if c_code else op.sqlname
|
||||
return f'({x} {name})'
|
||||
|
||||
def fn_behavior(op:OperatorBase, c_code, *x):
|
||||
name = op.cname if c_code else op.sqlname
|
||||
return f'{name}({", ".join([f"{xx}" for xx in x])})'
|
||||
|
||||
def windowed_fn_behavor(op: OperatorBase, c_code, *x):
|
||||
if not c_code:
|
||||
return f'{op.sqlname}({", ".join([f"{xx}" for xx in x])})'
|
||||
else:
|
||||
name = op.cname if len(x) == 1 else op.cname[:-1] + 'w'
|
||||
return f'{name}({", ".join([f"{xx}" for xx in x])})'
|
||||
|
||||
# arithmetic
|
||||
opadd = OperatorBase('add', 2, auto_extension, cname = '+', sqlname = '+', call = binary_op_behavior)
|
||||
opdiv = OperatorBase('div', 2, fp(auto_extension), cname = '/', sqlname = '/', call = binary_op_behavior)
|
||||
opmul = OperatorBase('mul', 2, fp(auto_extension), cname = '*', sqlname = '*', call = binary_op_behavior)
|
||||
opsub = OperatorBase('sub', 2, auto_extension, cname = '-', sqlname = '-', call = binary_op_behavior)
|
||||
opmod = OperatorBase('mod', 2, auto_extension_int, cname = '%', sqlname = '%', call = binary_op_behavior)
|
||||
opneg = OperatorBase('neg', 1, as_is, cname = '-', sqlname = '-', call = unary_op_behavior)
|
||||
# logical
|
||||
opand = OperatorBase('and', 2, logical, cname = '&&', sqlname = ' AND ', call = binary_op_behavior)
|
||||
opor = OperatorBase('or', 2, logical, cname = '||', sqlname = ' OR ', call = binary_op_behavior)
|
||||
opxor = OperatorBase('xor', 2, logical, cname = '^', sqlname = ' XOR ', call = binary_op_behavior)
|
||||
opgt = OperatorBase('gt', 2, logical, cname = '>', sqlname = '>', call = binary_op_behavior)
|
||||
oplt = OperatorBase('lt', 2, logical, cname = '<', sqlname = '<', call = binary_op_behavior)
|
||||
opge = OperatorBase('gte', 2, logical, cname = '>=', sqlname = '>=', call = binary_op_behavior)
|
||||
oplte = OperatorBase('lte', 2, logical, cname = '<=', sqlname = '<=', call = binary_op_behavior)
|
||||
opneq = OperatorBase('neq', 2, logical, cname = '!=', sqlname = '!=', call = binary_op_behavior)
|
||||
opeq = OperatorBase('eq', 2, logical, cname = '==', sqlname = '=', call = binary_op_behavior)
|
||||
opnot = OperatorBase('not', 1, logical, cname = '!', sqlname = 'NOT', call = unary_op_behavior)
|
||||
# functional
|
||||
fnmax = OperatorBase('max', 1, as_is, cname = 'max', sqlname = 'MAX', call = fn_behavior)
|
||||
fnmin = OperatorBase('min', 1, as_is, cname = 'min', sqlname = 'MIN', call = fn_behavior)
|
||||
fnsum = OperatorBase('sum', 1, ext(auto_extension), cname = 'sum', sqlname = 'SUM', call = fn_behavior)
|
||||
fnavg = OperatorBase('avg', 1, fp(ext(auto_extension)), cname = 'avg', sqlname = 'AVG', call = fn_behavior)
|
||||
fnmaxs = OperatorBase('maxs', [1, 2], ty_clamp(as_is, -1), cname = 'maxs', sqlname = 'MAXS', call = windowed_fn_behavor)
|
||||
fnmins = OperatorBase('mins', [1, 2], ty_clamp(as_is, -1), cname = 'mins', sqlname = 'MINS', call = windowed_fn_behavor)
|
||||
fnsums = OperatorBase('sums', [1, 2], ext(ty_clamp(auto_extension, -1)), cname = 'sums', sqlname = 'SUMS', call = windowed_fn_behavor)
|
||||
fnavgs = OperatorBase('avgs', [1, 2], fp(ext(ty_clamp(auto_extension, -1))), cname = 'avgs', sqlname = 'AVGS', call = windowed_fn_behavor)
|
||||
fncnt = OperatorBase('count', 1, int_return, cname = 'count', sqlname = 'COUNT', call = fn_behavior)
|
||||
# special
|
||||
def is_null_call_behavior(op:OperatorBase, c_code : bool, x : str):
|
||||
if c_code :
|
||||
return f'{x} == nullval<decays<decltype({x})>>'
|
||||
else :
|
||||
return f'{x} IS NULL'
|
||||
spnull = OperatorBase('missing', 1, logical, cname = "", sqlname = "", call = is_null_call_behavior)
|
||||
|
||||
# cstdlib
|
||||
fnsqrt = OperatorBase('sqrt', 1, lambda *_ : DoubleT, cname = 'sqrt', sqlname = 'SQRT', call = fn_behavior)
|
||||
|
||||
# type collections
|
||||
def _op_make_dict(*items : OperatorBase):
|
||||
return { i.name: i for i in items}
|
||||
builtin_binary_arith = _op_make_dict(opadd, opdiv, opmul, opsub, opmod)
|
||||
builtin_binary_logical = _op_make_dict(opand, opor, opxor, opgt, oplt, opge, oplte, opneq, opeq)
|
||||
builtin_unary_logical = _op_make_dict(opnot)
|
||||
builtin_unary_arith = _op_make_dict(opneg)
|
||||
builtin_unary_special = _op_make_dict(spnull)
|
||||
builtin_cstdlib = _op_make_dict(fnsqrt)
|
||||
builtin_func = _op_make_dict(fnmax, fnmin, fnsum, fnavg, fnmaxs, fnmins, fnsums, fnavgs, fncnt)
|
||||
builtin_operators : dict[str, OperatorBase] = {**builtin_binary_arith, **builtin_binary_logical,
|
||||
**builtin_unary_arith, **builtin_unary_logical, **builtin_unary_special, **builtin_func, **builtin_cstdlib}
|
||||
|
||||
+33
-2
@@ -1,7 +1,8 @@
|
||||
import uuid
|
||||
|
||||
base62alp = '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
|
||||
|
||||
nums = '0123456789'
|
||||
reserved_monet = ['month']
|
||||
def base62uuid(crop=8):
|
||||
id = uuid.uuid4().int
|
||||
ret = ''
|
||||
@@ -12,6 +13,33 @@ def base62uuid(crop=8):
|
||||
|
||||
return ret[:crop] if len(ret) else '0'
|
||||
|
||||
def get_leagl_name(name, lower = True):
|
||||
if name is not None:
|
||||
if lower:
|
||||
name = name.lower()
|
||||
name = ''.join([n for n in name if n in base62alp or n == '_'])
|
||||
|
||||
if name is None or len(name) == 0 or set(name) == set('_'):
|
||||
name = base62uuid(8)
|
||||
if(name[0] in nums):
|
||||
name = '_' + name
|
||||
|
||||
return name
|
||||
|
||||
def check_leagl_name(name):
|
||||
all_underscores = True
|
||||
for c in name:
|
||||
if c not in base62alp and c != '_':
|
||||
return False
|
||||
if c != '_':
|
||||
all_underscores = False
|
||||
if all_underscores:
|
||||
return False
|
||||
if name[0] in nums:
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
def enlist(l):
|
||||
return l if type(l) is list else [l]
|
||||
|
||||
@@ -22,4 +50,7 @@ def has_other(a, b):
|
||||
for ai in a:
|
||||
if ai not in b:
|
||||
return True
|
||||
return False
|
||||
return False
|
||||
|
||||
def defval(val, default):
|
||||
return default if val is None else val
|
||||
Reference in New Issue
Block a user