bill
/
AQuery
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

bugfix for duckdb engine, multi-engine support, perf regression investigate

Browse Source
master
Bill 2 years ago
parent 7c5440c4fb
commit 27ae26db90
23 changed files with 2587 additions and 95 deletions
Show Stats Download Patch File Download Diff File

4
.gitmodules vendored
Unescape View File

@ -1,3 +1,3 @@
[submodule "paper"]
path = paper
[submodule "docs/paper"]
path = docs/paper
url = https://github.com/sunyinqi0508/AQueryPaper

22
benchmark/h2o/groupby.sql
Unescape View File

@ -0,0 +1,22 @@
SELECT id1, sum(v1) AS v1 FROM source GROUP BY id1; --0.036 | 0.017 | .274
SELECT id1, id2, sum(v1) AS v1 FROM source GROUP BY id1, id2; -- 0.063 | 0.013
SELECT id3, sum(v1) AS v1, avg(v3) AS v3 FROM source GROUP BY id3; -- 2.322 | 0.406 | 2.27
SELECT id4, avg(v1) AS v1, avg(v2) AS v2, avg(v3) AS v3 FROM source GROUP BY id4; -- 0.159 | 0.022
SELECT id6, sum(v1) AS v1, sum(v2) AS v2, sum(v3) AS v3 FROM source GROUP BY id6; -- 1.778 | 0.699 | 2.283
--faster median
--SELECT id4, id5, median(v3) AS median_v3, stddev(v3) AS sd_v3 FROM source GROUP BY id4, id5; -- x4
SELECT id3, max(v1) - min(v2) AS range_v1_v2 FROM source GROUP BY id3; -- 0.857 | 0.467 | 2.236
-- select top 2 from each grp
-- SELECT id6, subvec(v3,0,2) AS v3 FROM source GROUP BY id6 order by v3;
-- implement corr
--SELECT id2, id4, pow(corr(v1, v2), 2) AS r2 FROM source GROUP BY id2, id4;
SELECT id1, id2, id3, id4, id5, id6, sum(v3) AS v3, count(*) AS cnt FROM source GROUP BY id1, id2, id3, id4, id5, id6; -- 2.669 | 1.232 | 2.221(1.8)

6
benchmark/h2o/groupby_prep.sql
Unescape View File

@ -0,0 +1,6 @@
create table source(id1 int,id2 int,id3 int,id4 int,id5 int,id6 int,v1 int,v2 int,v3 float)
LOAD DATA INFILE "data/h2o/G1_1e7_1e1_0_0_n.csv"
INTO TABLE source
FIELDS TERMINATED BY ","

21
common/types.py
Unescape View File

@ -290,6 +290,16 @@ def pack_behavior(op: OperatorBase, c_code, *x):
else:
return f'decltype({x[0]})::pack({len(x)}, {", ".join([f"{xx}.s()" for xx in x])})'
def subvec_behavior(op: OperatorBase, c_code, *x):
if len(x) < 1 :
raise ValueError('At least 1 parameters in subvec')
if len(x) == 1:
return f'{x[0]}'
if not c_code:
return f'{op.sqlname}({", ".join([f"{xx}" for xx in x])})'
else:
return f'{x[0]}.subvec({x[1]}{f", {x[2]}" if len(x) == 2 else ""})'
# arithmetic
opadd = OperatorBase('add', 2, auto_extension, cname = '+', sqlname = '+', call = binary_op_behavior)
# monetdb wont extend int division to fp type
@ -322,10 +332,12 @@ fnlast = OperatorBase('last', 1, as_is, cname = 'last', sqlname = 'LAST', call =
fnfirst = OperatorBase('first', 1, as_is, cname = 'frist', sqlname = 'FRIST', call = fn_behavior)
#fnsum = OperatorBase('sum', 1, ext(auto_extension), cname = 'sum', sqlname = 'SUM', call = fn_behavior)
#fnavg = OperatorBase('avg', 1, fp(ext(auto_extension)), cname = 'avg', sqlname = 'AVG', call = fn_behavior)
fnmedian = OperatorBase('median', 1, as_is, cname = 'median', sqlname = 'MEDIAN', call = fn_behavior)
fnsum = OperatorBase('sum', 1, long_return, cname = 'sum', sqlname = 'SUM', call = fn_behavior)
fnavg = OperatorBase('avg', 1, lfp_return, cname = 'avg', sqlname = 'AVG', call = fn_behavior)
fnvar = OperatorBase('var', 1, lfp_return, cname = 'var', sqlname = 'VAR_POP', call = fn_behavior)
fnstd = OperatorBase('stddev', 1, lfp_return, cname = 'stddev', sqlname = 'STDDEV_POP', call = fn_behavior)
fnvar = OperatorBase('var', 1, lfp_return, cname = 'var', sqlname = 'VAR', call = fn_behavior)
fnstd = OperatorBase('stddev', 1, lfp_return, cname = 'stddev', sqlname = 'STDDEV', call = fn_behavior)
fncorr = OperatorBase('corr', 1, lfp_return, cname = 'corr', sqlname = 'CORR', call = fn_behavior)
fnmaxs = OperatorBase('maxs', [1, 2], ty_clamp(as_is, -1), cname = 'maxs', sqlname = 'MAXS', call = windowed_fn_behavor)
fnmins = OperatorBase('mins', [1, 2], ty_clamp(as_is, -1), cname = 'mins', sqlname = 'MINS', call = windowed_fn_behavor)
fnsums = OperatorBase('sums', [1, 2], ext(ty_clamp(auto_extension, -1)), cname = 'sums', sqlname = 'SUMS', call = windowed_fn_behavor)
@ -334,6 +346,7 @@ fnvars = OperatorBase('vars', [1, 2], fp(ext(ty_clamp(auto_extension, -1))), cna
fnstds = OperatorBase('stddevs', [1, 2], fp(ext(ty_clamp(auto_extension, -1))), cname = 'stddevs', sqlname = 'STDDEVS', call = windowed_fn_behavor)
fncnt = OperatorBase('count', 1, int_return, cname = 'count', sqlname = 'COUNT', call = count_behavior)
fnpack = OperatorBase('pack', -1, pack_return, cname = 'pack', sqlname = 'PACK', call = pack_behavior)
fnsubvec = OperatorBase('subvec', [1, 2, 3], as_is, cname = 'subvec', sqlname = 'SUBVEC', call = subvec_behavior)
# special
def is_null_call_behavior(op:OperatorBase, c_code : bool, x : str):
if c_code :
@ -366,8 +379,8 @@ builtin_unary_arith = _op_make_dict(opneg)
builtin_unary_special = _op_make_dict(spnull, opdistinct)
# functions
builtin_cstdlib = _op_make_dict(fnsqrt, fnlog, fnsin, fncos, fntan, fnpow)
builtin_aggfunc = _op_make_dict(fnmax, fnmin, fnsum, fnavg,
fnlast, fnfirst, fncnt, fnvar, fnstd)
builtin_aggfunc = _op_make_dict(fnmedian, fnsubvec, fnmax, fnmin, fnsum, fnavg,
fnlast, fnfirst, fncnt, fnvar, fnstd, fncorr)
builtin_vecfunc = _op_make_dict(fnmaxs,
fnmins, fndeltas, fnratios, fnsums, fnavgs,
fnpack, fntrunc, fnprev, fnnext, fnvars, fnstds)

16
common/utils.py
Unescape View File

@ -190,3 +190,19 @@ def get_storedproc(name : str):
def execute_procedure(proc):
pass
import enum
class Backend_Type(enum.Enum):
BACKEND_AQuery = 0
BACKEND_MonetDB = 1
BACKEND_MariaDB = 2
BACKEND_DuckDB = 3
BACKEND_SQLite = 4
BACKEND_TOTAL = 5
backend_strings = {
'aquery': Backend_Type.BACKEND_AQuery,
'monetdb': Backend_Type.BACKEND_MonetDB,
'mariadb': Backend_Type.BACKEND_MariaDB,
'duckdb': Backend_Type.BACKEND_DuckDB,
'sqlite': Backend_Type.BACKEND_SQLite,
}

1
data/h2o
Unescape View File

@ -0,0 +1 @@
/Users/bill/Downloads/db-benchmark-master/data

1
docs/paper

@ -0,0 +1 @@
Subproject commit c8d27fe131e952ca4a30b0de27ff58b28a9990ad

24
engine/ast.py
Unescape View File

@ -5,7 +5,8 @@ from typing import Dict, List, Optional, Set, Tuple, Union
from common.types import *
from common.utils import (base62alp, base62uuid, enlist,
get_innermost, get_legal_name)
get_innermost, get_legal_name,
Backend_Type, backend_strings)
from engine.storage import ColRef, Context, TableInfo
class ast_node:
@ -72,7 +73,10 @@ class projection(ast_node):
force_use_spgb : bool = False,
subq_type: SubqType = SubqType.NONE
):
self.force_use_spgb = force_use_spgb
self.force_use_spgb = (
force_use_spgb |
context.system_state.cfg.backend_type == Backend_Type.BACKEND_AQuery.value
)
self.subq_type = subq_type
super().__init__(parent, node, context)
@ -426,6 +430,7 @@ class projection(ast_node):
def finalize(self, node):
self.deal_with_into(node)
self.context.emitc(f'puts("done.");')
self.context.emitc('printf("done: %lld\\n", (chrono::high_resolution_clock::now() - timer).count());timer = chrono::high_resolution_clock::now();')
if self.parent is None:
self.context.sql_end()
@ -575,6 +580,7 @@ class groupby_c(ast_node):
g_contents = ', '.join(
[f'{c}[{scanner_itname}]' for c in g_contents_list]
)
self.context.emitc('printf("init_time: %lld\\n", (chrono::high_resolution_clock::now() - timer).count()); timer = chrono::high_resolution_clock::now();')
self.context.emitc(f'typedef record<{",".join(g_contents_decltype)}> {self.group_type};')
self.context.emitc(f'AQHashTable<{self.group_type}, '
f'transTypes<{self.group_type}, hasher>> {self.group} {{{self.total_sz}}};')
@ -602,7 +608,7 @@ class groupby_c(ast_node):
self.context.emitc(f'auto buf_{c} = static_cast<{typename.cname} *>(calloc({self.total_sz}, sizeof({typename.cname})));')
tovec_columns.add(c)
else:
self.context.emitc(f'{c}.reserve({self.group}.size());')
self.context.emitc(f'{c}.resize({self.group}.size());')
self.arr_len = 'arrlen_' + base62uuid(3)
self.arr_values = 'arrvals_' + base62uuid(3)
@ -621,7 +627,7 @@ class groupby_c(ast_node):
)
preproc_scanner.finalize()
self.context.emitc(f'GC::scratch_space = GC::gc_handle ? &(GC::gc_handle->scratch) : nullptr;')
self.context.emitc('GC::scratch_space = GC::gc_handle ? &(GC::gc_handle->scratch) : nullptr;')
# gscanner = scan(self, self.group, loop_style = 'for_each')
gscanner = scan(self, self.arr_len)
key_var = 'key_'+base62uuid(7)
@ -674,7 +680,7 @@ class groupby_c(ast_node):
materialize_builtin['_builtin_len'] = len_var
if '_builtin_ret' in ex.udf_called.builtin_used:
define_len_var()
gscanner.add(f'{ce[0]}.emplace_back({len_var});\n')
gscanner.add(f'{ce[0]}[{gscanner.it_var}] = {len_var};\n')
materialize_builtin['_builtin_ret'] = f'{ce[0]}.back()'
gscanner.add(f'{ex.eval(c_code = True, y=get_var_names, materialize_builtin = materialize_builtin)};\n')
continue
@ -684,12 +690,14 @@ class groupby_c(ast_node):
else:
gscanner.add(f'{ce[0]}[{gscanner.it_var}] = {get_var_names_ex(ex)};\n')
else:
gscanner.add(f'{ce[0]}.emplace_back({get_var_names_ex(ex)});\n')
gscanner.add(f'{ce[0]}[{gscanner.it_var}] = ({get_var_names_ex(ex)});\n')
gscanner.add(f'GC::scratch_space->release();')
self.context.emitc('printf("ht_construct: %lld\\n", (chrono::high_resolution_clock::now() - timer).count());timer = chrono::high_resolution_clock::now();')
gscanner.finalize()
self.context.emitc(f'GC::scratch_space = nullptr;')
self.context.emitc('printf("agg: %lld\\n", (chrono::high_resolution_clock::now() - timer).count());timer = chrono::high_resolution_clock::now();')
self.datasource.groupinfo = None
@ -698,7 +706,7 @@ class groupby(ast_node):
name = 'group by'
@staticmethod
def check_special(parent, node):
def check_special(parent : projection, node):
node = enlist(node)
for g in node:
if ('value' in g and
@ -1252,7 +1260,7 @@ class load(ast_node):
name="load"
first_order = name
def init(self, node):
from prompt import Backend_Type
from common.utils import Backend_Type
self.module = False
if node['load']['file_type'] == 'module':
self.produce = self.produce_module

3
engine/expr.py
Unescape View File

@ -3,6 +3,7 @@ from typing import Optional, Set
from common.types import *
from engine.ast import ast_node
from engine.storage import ColRef, Context
from common.utils import Backend_Type
# TODO: Decouple expr and upgrade architecture
# C_CODE : get ccode/sql code?
@ -180,6 +181,8 @@ class expr(ast_node):
and
self.root.parent.force_use_spgb
)
or
self.context.system_state.cfg.backend_type == Backend_Type.BACKEND_AQuery.value
):
special_func = [*special_func, *self.ext_aggfuncs]

5
engine/storage.py
Unescape View File

@ -212,7 +212,10 @@ class Context:
function_head = ('(Context* cxt) {\n' +
'\tusing namespace std;\n' +
'\tusing namespace types;\n' +
'\tauto server = static_cast<DataSource*>(cxt->curr_server);\n')
'\tauto server = static_cast<DataSource*>(cxt->curr_server);\n'
'\tauto timer = chrono::high_resolution_clock::now();\n'
)
udf_head = ('#pragma once\n'
'#include \"./server/libaquery.h\"\n'

2418
monetdb/msvc/gdk.h
View File

File diff suppressed because it is too large Load Diff

32
msc-plugin/server.vcxproj
Unescape View File

@ -192,9 +192,9 @@
<Link>
<SubSystem>Console</SubSystem>
<GenerateDebugInformation>true</GenerateDebugInformation>
<AdditionalDependencies>bat.lib;monetdbsql.lib;..\libaquery.lib;$(ProjectDir)\..\monetdb\msvc\monetdbe.lib;%(AdditionalDependencies)</AdditionalDependencies>
<AdditionalDependencies>bat.lib;monetdbsql.lib;duckdb.lib;..\libaquery.lib;$(ProjectDir)\..\monetdb\msvc\monetdbe.lib;%(AdditionalDependencies)</AdditionalDependencies>
<AdditionalOptions>/WHOLEARCHIVE:libaquery.lib %(AdditionalOptions)</AdditionalOptions>
<AdditionalLibraryDirectories>../monetdb/msvc</AdditionalLibraryDirectories>
<AdditionalLibraryDirectories>../monetdb/msvc;../deps</AdditionalLibraryDirectories>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
@ -217,10 +217,10 @@
<EnableCOMDATFolding>true</EnableCOMDATFolding>
<OptimizeReferences>true</OptimizeReferences>
<GenerateDebugInformation>false</GenerateDebugInformation>
<AdditionalDependencies>bat.lib;monetdbsql.lib;..\libaquery.lib;$(ProjectDir)\..\monetdb\msvc\monetdbe.lib;%(AdditionalDependencies)</AdditionalDependencies>
<AdditionalDependencies>bat.lib;monetdbsql.lib;duckdb.lib;..\libaquery.lib;$(ProjectDir)\..\monetdb\msvc\monetdbe.lib;%(AdditionalDependencies)</AdditionalDependencies>
<DataExecutionPrevention>false</DataExecutionPrevention>
<AdditionalOptions>/WHOLEARCHIVE:libaquery.lib %(AdditionalOptions)</AdditionalOptions>
<AdditionalLibraryDirectories>../monetdb/msvc</AdditionalLibraryDirectories>
<AdditionalLibraryDirectories>../monetdb/msvc;../deps</AdditionalLibraryDirectories>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='RelWDbgInf|Win32'">
@ -243,10 +243,10 @@
<EnableCOMDATFolding>true</EnableCOMDATFolding>
<OptimizeReferences>true</OptimizeReferences>
<GenerateDebugInformation>false</GenerateDebugInformation>
<AdditionalDependencies>bat.lib;monetdbsql.lib;..\libaquery.lib;$(ProjectDir)\..\monetdb\msvc\monetdbe.lib;%(AdditionalDependencies)</AdditionalDependencies>
<AdditionalDependencies>bat.lib;monetdbsql.lib;duckdb.lib;..\libaquery.lib;$(ProjectDir)\..\monetdb\msvc\monetdbe.lib;%(AdditionalDependencies)</AdditionalDependencies>
<DataExecutionPrevention>false</DataExecutionPrevention>
<AdditionalOptions>/WHOLEARCHIVE:libaquery.lib %(AdditionalOptions)</AdditionalOptions>
<AdditionalLibraryDirectories>../monetdb/msvc</AdditionalLibraryDirectories>
<AdditionalLibraryDirectories>../monetdb/msvc;../deps</AdditionalLibraryDirectories>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='sharedlib|Win32'">
@ -268,9 +268,9 @@
<EnableCOMDATFolding>true</EnableCOMDATFolding>
<OptimizeReferences>true</OptimizeReferences>
<GenerateDebugInformation>true</GenerateDebugInformation>
<AdditionalDependencies>bat.lib;monetdbsql.lib;$(ProjectDir)\..\monetdb\msvc\monetdbe.lib;%(AdditionalDependencies)</AdditionalDependencies>
<AdditionalDependencies>bat.lib;monetdbsql.lib;duckdb.lib;$(ProjectDir)\..\monetdb\msvc\monetdbe.lib;%(AdditionalDependencies)</AdditionalDependencies>
<AdditionalOptions>/WHOLEARCHIVE:libaquery.lib %(AdditionalOptions)</AdditionalOptions>
<AdditionalLibraryDirectories>../monetdb/msvc</AdditionalLibraryDirectories>
<AdditionalLibraryDirectories>../monetdb/msvc;../deps</AdditionalLibraryDirectories>
</Link>
<PostBuildEvent>
<Command>copy $(OutDir)$(TargetName)$(TargetExt) $(ProjectDir)\..\server.so /y</Command>
@ -292,9 +292,9 @@
<Link>
<SubSystem>Console</SubSystem>
<GenerateDebugInformation>true</GenerateDebugInformation>
<AdditionalDependencies>bat.lib;monetdbsql.lib;..\libaquery.lib;$(ProjectDir)\..\monetdb\msvc\monetdbe.lib;%(AdditionalDependencies)</AdditionalDependencies>
<AdditionalDependencies>bat.lib;monetdbsql.lib;duckdb.lib;..\libaquery.lib;$(ProjectDir)\..\monetdb\msvc\monetdbe.lib;%(AdditionalDependencies)</AdditionalDependencies>
<AdditionalOptions>/WHOLEARCHIVE:libaquery.lib %(AdditionalOptions)</AdditionalOptions>
<AdditionalLibraryDirectories>../monetdb/msvc</AdditionalLibraryDirectories>
<AdditionalLibraryDirectories>../monetdb/msvc;../deps</AdditionalLibraryDirectories>
</Link>
<Bscmake>
<PreserveSbr>true</PreserveSbr>
@ -320,10 +320,10 @@
<EnableCOMDATFolding>true</EnableCOMDATFolding>
<OptimizeReferences>true</OptimizeReferences>
<GenerateDebugInformation>false</GenerateDebugInformation>
<AdditionalDependencies>bat.lib;monetdbsql.lib;..\libaquery.lib;$(ProjectDir)\..\monetdb\msvc\monetdbe.lib;%(AdditionalDependencies)</AdditionalDependencies>
<AdditionalDependencies>bat.lib;monetdbsql.lib;duckdb.lib;..\libaquery.lib;$(ProjectDir)\..\monetdb\msvc\monetdbe.lib;%(AdditionalDependencies)</AdditionalDependencies>
<DataExecutionPrevention>false</DataExecutionPrevention>
<AdditionalOptions>/WHOLEARCHIVE:libaquery.lib %(AdditionalOptions)</AdditionalOptions>
<AdditionalLibraryDirectories>../monetdb/msvc</AdditionalLibraryDirectories>
<AdditionalLibraryDirectories>../monetdb/msvc;../deps</AdditionalLibraryDirectories>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='RelWDbgInf|x64'">
@ -346,10 +346,10 @@
<EnableCOMDATFolding>true</EnableCOMDATFolding>
<OptimizeReferences>true</OptimizeReferences>
<GenerateDebugInformation>false</GenerateDebugInformation>
<AdditionalDependencies>bat.lib;monetdbsql.lib;..\libaquery.lib;$(ProjectDir)\..\monetdb\msvc\monetdbe.lib;%(AdditionalDependencies)</AdditionalDependencies>
<AdditionalDependencies>bat.lib;monetdbsql.lib;duckdb.lib;..\libaquery.lib;$(ProjectDir)\..\monetdb\msvc\monetdbe.lib;%(AdditionalDependencies)</AdditionalDependencies>
<DataExecutionPrevention>false</DataExecutionPrevention>
<AdditionalOptions>/WHOLEARCHIVE:libaquery.lib %(AdditionalOptions)</AdditionalOptions>
<AdditionalLibraryDirectories>../monetdb/msvc</AdditionalLibraryDirectories>
<AdditionalLibraryDirectories>../monetdb/msvc;../deps</AdditionalLibraryDirectories>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='sharedlib|x64'">
@ -371,9 +371,9 @@
<EnableCOMDATFolding>true</EnableCOMDATFolding>
<OptimizeReferences>true</OptimizeReferences>
<GenerateDebugInformation>true</GenerateDebugInformation>
<AdditionalDependencies>bat.lib;monetdbsql.lib;$(ProjectDir)\..\monetdb\msvc\monetdbe.lib;%(AdditionalDependencies)</AdditionalDependencies>
<AdditionalDependencies>bat.lib;monetdbsql.lib;duckdb.lib;$(ProjectDir)\..\monetdb\msvc\monetdbe.lib;%(AdditionalDependencies)</AdditionalDependencies>
<AdditionalOptions>/WHOLEARCHIVE:libaquery.lib %(AdditionalOptions)</AdditionalOptions>
<AdditionalLibraryDirectories>../monetdb/msvc</AdditionalLibraryDirectories>
<AdditionalLibraryDirectories>../monetdb/msvc;../deps</AdditionalLibraryDirectories>
</Link>
<PostBuildEvent>
<Command>copy "$(OutDir)$(TargetName)$(TargetExt)" "$(ProjectDir)\..\server.so" /y</Command>

24
msc-plugin/testmain.vcxproj
Unescape View File

@ -192,8 +192,9 @@
<Link>
<SubSystem>Console</SubSystem>
<GenerateDebugInformation>true</GenerateDebugInformation>
<AdditionalDependencies>..\libaquery.lib;$(ProjectDir)\..\monetdb\msvc\monetdbe.lib;%(AdditionalDependencies)</AdditionalDependencies>
<AdditionalDependencies>bat.lib;monetdbsql.lib;duckdb.lib;..\libaquery.lib;$(ProjectDir)\..\monetdb\msvc\monetdbe.lib;%(AdditionalDependencies)</AdditionalDependencies>
<AdditionalOptions>/WHOLEARCHIVE:libaquery.lib %(AdditionalOptions)</AdditionalOptions>
<AdditionalLibraryDirectories>../deps/;../monetdb/msvc</AdditionalLibraryDirectories>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
@ -216,10 +217,11 @@
<EnableCOMDATFolding>true</EnableCOMDATFolding>
<OptimizeReferences>true</OptimizeReferences>
<GenerateDebugInformation>false</GenerateDebugInformation>
<AdditionalDependencies>..\libaquery.lib;$(ProjectDir)\..\monetdb\msvc\monetdbe.lib;%(AdditionalDependencies)</AdditionalDependencies>
<AdditionalDependencies>bat.lib;monetdbsql.lib;duckdb.lib;..\libaquery.lib;$(ProjectDir)\..\monetdb\msvc\monetdbe.lib;%(AdditionalDependencies)</AdditionalDependencies>
<DataExecutionPrevention>false</DataExecutionPrevention>
<AdditionalOptions>/WHOLEARCHIVE:libaquery.lib %(AdditionalOptions)</AdditionalOptions>
<Profile>true</Profile>
<AdditionalLibraryDirectories>../deps/;../monetdb/msvc</AdditionalLibraryDirectories>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='RelWDbgInf|Win32'">
@ -242,10 +244,11 @@
<EnableCOMDATFolding>true</EnableCOMDATFolding>
<OptimizeReferences>true</OptimizeReferences>
<GenerateDebugInformation>DebugFull</GenerateDebugInformation>
<AdditionalDependencies>..\libaquery.lib;$(ProjectDir)\..\monetdb\msvc\monetdbe.lib;%(AdditionalDependencies)</AdditionalDependencies>
<AdditionalDependencies>bat.lib;monetdbsql.lib;duckdb.lib;..\libaquery.lib;$(ProjectDir)\..\monetdb\msvc\monetdbe.lib;%(AdditionalDependencies)</AdditionalDependencies>
<DataExecutionPrevention>false</DataExecutionPrevention>
<AdditionalOptions>/WHOLEARCHIVE:libaquery.lib %(AdditionalOptions)</AdditionalOptions>
<AssemblyDebug>true</AssemblyDebug>
<AdditionalLibraryDirectories>../deps/;../monetdb/msvc</AdditionalLibraryDirectories>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='sharedlib|Win32'">
@ -267,8 +270,9 @@
<EnableCOMDATFolding>true</EnableCOMDATFolding>
<OptimizeReferences>true</OptimizeReferences>
<GenerateDebugInformation>true</GenerateDebugInformation>
<AdditionalDependencies>$(ProjectDir)\..\monetdb\msvc\monetdbe.lib;%(AdditionalDependencies)</AdditionalDependencies>
<AdditionalDependencies>bat.lib;monetdbsql.lib;duckdb.lib;$(ProjectDir)\..\monetdb\msvc\monetdbe.lib;%(AdditionalDependencies)</AdditionalDependencies>
<AdditionalOptions>/WHOLEARCHIVE:libaquery.lib %(AdditionalOptions)</AdditionalOptions>
<AdditionalLibraryDirectories>../deps/;../monetdb/msvc</AdditionalLibraryDirectories>
</Link>
<PostBuildEvent>
<Command>copy $(OutDir)$(TargetName)$(TargetExt) $(ProjectDir)\..\server.so /y</Command>
@ -290,8 +294,9 @@
<Link>
<SubSystem>Console</SubSystem>
<GenerateDebugInformation>true</GenerateDebugInformation>
<AdditionalDependencies>..\libaquery.lib;$(ProjectDir)\..\monetdb\msvc\monetdbe.lib;%(AdditionalDependencies)</AdditionalDependencies>
<AdditionalDependencies>bat.lib;monetdbsql.lib;duckdb.lib;..\libaquery.lib;$(ProjectDir)\..\monetdb\msvc\monetdbe.lib;%(AdditionalDependencies)</AdditionalDependencies>
<AdditionalOptions>/WHOLEARCHIVE:libaquery.lib %(AdditionalOptions)</AdditionalOptions>
<AdditionalLibraryDirectories>../deps/;../monetdb/msvc</AdditionalLibraryDirectories>
</Link>
<Bscmake>
<PreserveSbr>true</PreserveSbr>
@ -317,10 +322,11 @@
<EnableCOMDATFolding>true</EnableCOMDATFolding>
<OptimizeReferences>true</OptimizeReferences>
<GenerateDebugInformation>false</GenerateDebugInformation>
<AdditionalDependencies>..\libaquery.lib;$(ProjectDir)\..\monetdb\msvc\monetdbe.lib;%(AdditionalDependencies)</AdditionalDependencies>
<AdditionalDependencies>bat.lib;monetdbsql.lib;duckdb.lib;..\libaquery.lib;$(ProjectDir)\..\monetdb\msvc\monetdbe.lib;%(AdditionalDependencies)</AdditionalDependencies>
<DataExecutionPrevention>false</DataExecutionPrevention>
<AdditionalOptions>/WHOLEARCHIVE:libaquery.lib %(AdditionalOptions)</AdditionalOptions>
<Profile>true</Profile>
<AdditionalLibraryDirectories>../deps/;../monetdb/msvc</AdditionalLibraryDirectories>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='RelWDbgInf|x64'">
@ -343,10 +349,11 @@
<EnableCOMDATFolding>true</EnableCOMDATFolding>
<OptimizeReferences>true</OptimizeReferences>
<GenerateDebugInformation>DebugFull</GenerateDebugInformation>
<AdditionalDependencies>..\libaquery.lib;$(ProjectDir)\..\monetdb\msvc\monetdbe.lib;%(AdditionalDependencies)</AdditionalDependencies>
<AdditionalDependencies>bat.lib;monetdbsql.lib;duckdb.lib;..\libaquery.lib;$(ProjectDir)\..\monetdb\msvc\monetdbe.lib;%(AdditionalDependencies)</AdditionalDependencies>
<DataExecutionPrevention>false</DataExecutionPrevention>
<AdditionalOptions>/WHOLEARCHIVE:libaquery.lib %(AdditionalOptions)</AdditionalOptions>
<AssemblyDebug>true</AssemblyDebug>
<AdditionalLibraryDirectories>../deps/;../monetdb/msvc</AdditionalLibraryDirectories>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='sharedlib|x64'">
@ -368,8 +375,9 @@
<EnableCOMDATFolding>true</EnableCOMDATFolding>
<OptimizeReferences>true</OptimizeReferences>
<GenerateDebugInformation>true</GenerateDebugInformation>
<AdditionalDependencies>$(ProjectDir)\..\monetdb\msvc\monetdbe.lib;%(AdditionalDependencies)</AdditionalDependencies>
<AdditionalDependencies>bat.lib;monetdbsql.lib;duckdb.lib;$(ProjectDir)\..\monetdb\msvc\monetdbe.lib;%(AdditionalDependencies)</AdditionalDependencies>
<AdditionalOptions>/WHOLEARCHIVE:libaquery.lib %(AdditionalOptions)</AdditionalOptions>
<AdditionalLibraryDirectories>../deps/;../monetdb/msvc</AdditionalLibraryDirectories>
</Link>
<PostBuildEvent>
<Command>copy "$(OutDir)$(TargetName)$(TargetExt)" "$(ProjectDir)\..\server.so" /y</Command>

1
paper

@ -1 +0,0 @@
Subproject commit 6f49544835a35915e1561a2e6581ee9ac0759381

23
prompt.py
Unescape View File

@ -105,7 +105,7 @@ import common.ddl
import common.projection
import engine as xengine
from build import build_manager
from common.utils import add_dll_dir, base62uuid, nullstream, ws
from common.utils import add_dll_dir, base62uuid, nullstream, ws, Backend_Type, backend_strings
from enum import auto
## CLASSES BEGIN
@ -113,21 +113,6 @@ class RunType(enum.Enum):
Threaded = 0
IPC = 1
class Backend_Type(enum.Enum):
BACKEND_AQuery = 0
BACKEND_MonetDB = 1
BACKEND_MariaDB = 2
BACKEND_DuckDB = 3
BACKEND_SQLite = 4
BACKEND_TOTAL = 5
backend_strings = {
'aquery': Backend_Type.BACKEND_AQuery,
'monetdb': Backend_Type.BACKEND_MonetDB,
'mariadb': Backend_Type.BACKEND_MariaDB,
'duckdb': Backend_Type.BACKEND_DuckDB,
'sqlite': Backend_Type.BACKEND_SQLite,
}
class StoredProcedure(ctypes.Structure):
_fields_ = [
('cnt', ctypes.c_uint32),
@ -367,7 +352,7 @@ def init_threaded(state : PromptState):
if aquery_config.have_hge != 0:
from common.types import get_int128_support
get_int128_support()
state.th = threading.Thread(target=server_so['main'], args=(-1, ctypes.POINTER(ctypes.c_char_p)(state.cfg.c)), daemon=True)
state.th = threading.Thread(target=server_so['dllmain'], args=(-1, ctypes.POINTER(ctypes.c_char_p)(state.cfg.c)), daemon=True)
state.th.start()
def init_prompt() -> PromptState:
@ -695,12 +680,12 @@ def prompt(running = lambda:True, next = lambda:input('> '), state : Optional[Pr
state.force_compiled = True
cxt.force_compiled = True
continue
elif q.startswith('backend'):
elif q.startswith('engine'):
splits = q.split()
if len(splits) > 1 and splits[1] in backend_strings:
state.cfg.backend_type = backend_strings[splits[1]].value
else:
cxt.Error('Not a valid backend type.')
cxt.Error('Not a valid engine type.')
print('External Engine is set to', Backend_Type(state.cfg.backend_type).name)
continue
trimed = ws.sub(' ', og_q).split(' ')

5
server/DataSource_conn.h
Unescape View File

@ -1,5 +1,7 @@
#ifndef __DATASOURCE_CONN_H__
#define __DATASOURCE_CONN_H__
#include <limits>
#include <cstdint>
struct Context;
#ifndef __AQQueryResult__
@ -45,7 +47,8 @@ struct DataSource {
// virtual long long getFirstElement() = 0;
virtual void close() = 0;
virtual bool haserror() = 0;
// virtual void print_results(const char* sep = " ", const char* end = "\n");
virtual void print_results(const char* sep = " ", const char* end = "\n",
uint32_t limit = std::numeric_limits<uint32_t>::max()) {};
virtual ~DataSource() {};
};
// TODO: replace with super class

1
server/duckdb_conn.cpp
Unescape View File

@ -1,3 +1,4 @@
#include "pch_msc.hpp"
#include "duckdb_conn.h"
#include "../deps/duckdb.hpp"
#include "libaquery.h"

4
server/duckdb_conn.h
Unescape View File

@ -10,7 +10,9 @@ struct DuckdbServer : DataSource {
long long getFirstElement();
void close();
bool haserror();
void print_results(const char* sep = " ", const char* end = "\n");
void print_results(const char* sep = " ", const char* end = "\n",
uint32_t limit = std::numeric_limits<uint32_t>::max()) {}
~DuckdbServer();
};
#endif //__DUCKDB_CONN_H__

32
server/libaquery.cpp
Unescape View File

@ -260,28 +260,28 @@ inline const char* str(const bool& v) {
return v ? "true" : "false";
}
class A {
public:
std::chrono::high_resolution_clock::time_point tp;
A(){
tp = std::chrono::high_resolution_clock::now();
printf("A %llu created.\n", tp.time_since_epoch().count());
}
~A() {
printf("A %llu died after %lldns.\n", tp.time_since_epoch().count(),
(std::chrono::high_resolution_clock::now() - tp).count());
}
};
// class A {
// public:
// std::chrono::high_resolution_clock::time_point tp;
// A(){
// tp = std::chrono::high_resolution_clock::now();
// printf("A %llu created.\n", tp.time_since_epoch().count());
// }
// ~A() {
// printf("A %llu died after %lldns.\n", tp.time_since_epoch().count(),
// (std::chrono::high_resolution_clock::now() - tp).count());
// }
// };
Context::Context() {
current.memory_map = new std::unordered_map<void*, deallocator_t>;
#ifndef __AQ_USE_THREADEDGC__
this->gc = new GC();
#endif
GC::gc_handle->reg(new A(), 6553600, [](void* a){
puts("deleting");
delete ((A*)a);
});
// GC::gc_handle->reg(new A(), 6553600, [](void* a){
// puts("deleting");
// delete ((A*)a);
// });
init_session();
}

0
server/main.cpp
Unescape View File

2
server/monetdb_conn.cpp
Unescape View File

@ -150,7 +150,7 @@ bool MonetdbServer::haserror(){
}
void MonetdbServer::print_results(const char* sep, const char* end){
void MonetdbServer::print_results(const char* sep, const char* end, uint32_t) {
if (!haserror()){
auto _res = static_cast<monetdbe_result*> (res);

3
server/monetdb_conn.h
Unescape View File

@ -11,7 +11,8 @@ struct MonetdbServer : DataSource {
void close() override;
bool haserror() override;
static bool havehge();
void print_results(const char* sep = " ", const char* end = "\n");
void print_results(const char* sep = " ", const char* end = "\n",
uint32_t limit = std::numeric_limits<uint32_t>::max()) override;
friend void print_monetdb_results(void* _srv, const char* sep, const char* end, int limit);
~MonetdbServer() override;
};

34
server/server.cpp
Unescape View File

@ -174,7 +174,7 @@ void initialize_module(const char* module_name, void* module_handle, Context* cx
}
#pragma endregion
int dll_main(int argc, char** argv, Context* cxt){
int threaded_main(int argc, char** argv, Context* cxt){
aq_timer timer;
Config *cfg = reinterpret_cast<Config *>(argv[0]);
std::unordered_map<std::string, void*> user_module_map;
@ -218,7 +218,11 @@ int dll_main(int argc, char** argv, Context* cxt){
const auto& update_backend = [&cxt, &cfg](){
auto& curr_server = cxt->alt_server[cfg->backend_type];
auto backend_type = cfg->backend_type;
if (backend_type == Backend_Type::BACKEND_AQuery) {
backend_type = Backend_Type::BACKEND_MonetDB;
}
auto& curr_server = cxt->alt_server[backend_type];
if (curr_server == nullptr) {
curr_server = get_server[cfg->backend_type](cxt);
cxt->alt_server[cfg->backend_type] = curr_server;
@ -239,7 +243,8 @@ start:
void *handle = nullptr;
void *user_module_handle = nullptr;
if (cfg->backend_type == BACKEND_MonetDB||
cfg->backend_type == BACKEND_DuckDB
cfg->backend_type == BACKEND_DuckDB ||
cfg->backend_type == BACKEND_AQuery
) {
update_backend();
auto server = reinterpret_cast<DataSource*>(cxt->curr_server);
@ -559,11 +564,11 @@ start:
}
// puts(cfg->has_dll ? "true" : "false");
if (cfg->backend_type == BACKEND_AQuery) {
handle = dlopen("./dll.so", RTLD_NOW);
code_snippet c = reinterpret_cast<code_snippet>(dlsym(handle, "dllmain"));
c(cxt);
}
// if (cfg->backend_type == BACKEND_AQuery) {
// handle = dlopen("./dll.so", RTLD_NOW);
// code_snippet c = reinterpret_cast<code_snippet>(dlsym(handle, "dllmain"));
// c(cxt);
// }
if (handle &&
!procedure_replaying && !procedure_recording) {
printf("Destroy %p\n", handle);
@ -580,8 +585,8 @@ start:
return 0;
}
int launcher(int argc, char** argv){
#ifdef __AQ_BUILD_LAUNCHER__
int main(int argc, char** argv){
#ifdef _WIN32
constexpr char sep = '\\';
#else
@ -603,11 +608,9 @@ int launcher(int argc, char** argv){
str = std::string("cd ") + pwd + std::string("&& python3 ./prompt.py ") + str;
return system(str.c_str());
}
#if true || !defined(TESTMAIN) && !( defined(_MSC_VER) && defined(_DEBUG) )
extern "C" int __DLLEXPORT__ main(int argc, char** argv) {
#ifdef __AQ_BUILD_LAUNCHER__
return launcher(argc, argv);
#endif
extern "C" int __DLLEXPORT__ dllmain(int argc, char** argv) {
// puts("running");
Context* cxt = new Context();
_g_cxt = cxt;
@ -623,7 +626,7 @@ extern "C" int __DLLEXPORT__ main(int argc, char** argv) {
const char* shmname;
if (argc < 0)
return dll_main(argc, argv, cxt);
return threaded_main(argc, argv, cxt);
else
shmname = argv[1];
SharedMemory shm = SharedMemory(shmname);
@ -657,4 +660,3 @@ extern "C" int __DLLEXPORT__ main(int argc, char** argv) {
return 0;
}
#endif // MSCDBG

Write Preview
Loading…
Cancel
Save