bill/AQuery
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

update

Browse Source
This commit is contained in:
bill
2023-04-09 09:56:32 +08:00
parent 6f267f982d
commit dfb3ec2380
18 changed files with 94 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+1
View File
@@ -1,3 +1,4 @@
duckdb.dll
*.swp
tests/datagen_jose/histgen
tests/datagen_jose/tickgen
aquery_parser/__init__.py
+1 -1
View File
@@ -13,7 +13,7 @@ import json
from threading import Lock
from aquery_parser.parser import scrub
from aquery_parser.utils import ansi_string, simple_op, normal_op
from aquery_parser.utils import simple_op, normal_op
import aquery_parser.parser
parse_locker = Lock() # ENSURE ONLY ONE PARSING AT A TIME
common_parser = None
aquery_parser/parser.py
-1
View File
@@ -7,7 +7,6 @@
# Contact: Kyle Lahnakoski (kyle@lahnakoski.com)
# Bill Sun 2022 - 2023
from sre_parse import WHITESPACE
from mo_parsing.helpers import restOfLine
from mo_parsing.infix import delimited_list
benchmark/h2o/groupby.sql
+3 -2
View File
@@ -14,9 +14,10 @@ SELECT id6, sum(v1) AS v1, sum(v2) AS v2, sum(v3) AS v3 FROM source GROUP BY id6
SELECT id3, max(v1) - min(v2) AS range_v1_v2 FROM source GROUP BY id3; -- 0.857 | 0.467 | 2.236
-- select top 2 from each grp
-- SELECT id6, subvec(v3,0,2) AS v3 FROM source GROUP BY id6 order by v3;
SELECT id6, subvec(v3,0,2) AS v3 FROM source GROUP BY id6 order by v3;
-- implement corr
--SELECT id2, id4, pow(corr(v1, v2), 2) AS r2 FROM source GROUP BY id2, id4;
SELECT id2, id4, pow(corr(v1, v2), 2) AS r2 FROM source GROUP BY id2, id4;
-- NA | 0.240 | 0.6
SELECT id1, id2, id3, id4, id5, id6, sum(v3) AS v3, count(*) AS cnt FROM source GROUP BY id1, id2, id3, id4, id5, id6; -- 2.669 | 1.232 | 2.221(1.8)
build.py
+1 -1
View File
@@ -187,7 +187,7 @@ class build_manager:
def __init__(self) -> None:
self.method = 'make'
self.cxx = ''
self.OptimizationLv = '0' # [O0, O1, O2, O3, Ofast]
self.OptimizationLv = '4' # [O0, O1, O2, O3, Ofast]
self.Platform = 'amd64'
self.PCH = os.environ['PCH'] if 'PCH' in os.environ else 1
self.StaticLib = 1
common/types.py
+2 -2
View File
@@ -298,7 +298,7 @@ def subvec_behavior(op: OperatorBase, c_code, *x):
if not c_code:
return f'{op.sqlname}({", ".join([f"{xx}" for xx in x])})'
else:
return f'{x[0]}.subvec({x[1]}{f", {x[2]}" if len(x) == 2 else ""})'
return f'{x[0]}.subvec({x[1]}{f", {x[2]}" if len(x) == 3 else ""})'
# arithmetic
opadd = OperatorBase('add', 2, auto_extension, cname = '+', sqlname = '+', call = binary_op_behavior)
@@ -346,7 +346,7 @@ fnvars = OperatorBase('vars', [1, 2], fp(ext(ty_clamp(auto_extension, -1))), cna
fnstds = OperatorBase('stddevs', [1, 2], fp(ext(ty_clamp(auto_extension, -1))), cname = 'stddevs', sqlname = 'STDDEVS', call = windowed_fn_behavor)
fncnt = OperatorBase('count', 1, int_return, cname = 'count', sqlname = 'COUNT', call = count_behavior)
fnpack = OperatorBase('pack', -1, pack_return, cname = 'pack', sqlname = 'PACK', call = pack_behavior)
fnsubvec = OperatorBase('subvec', [1, 2, 3], as_is, cname = 'subvec', sqlname = 'SUBVEC', call = subvec_behavior)
fnsubvec = OperatorBase('subvec', [1, 2, 3], ty_clamp(as_is, 0, 1), cname = 'subvec', sqlname = 'SUBVEC', call = subvec_behavior)
# special
def is_null_call_behavior(op:OperatorBase, c_code : bool, x : str):
if c_code :
engine/ast.py
+3 -1
View File
@@ -591,7 +591,9 @@ class groupby_c(ast_node):
def consume(self, _):
self.scanner.finalize()
self.context.emitc('printf("ht_construct: %lld\\n", (chrono::high_resolution_clock::now() - timer).count()); timer = chrono::high_resolution_clock::now();')
self.context.emitc(f'auto {self.vecs} = {self.group}.ht_postproc({self.total_sz});')
self.context.emitc('printf("ht_postproc: %lld\\n", (chrono::high_resolution_clock::now() - timer).count()); timer = chrono::high_resolution_clock::now();')
# def deal_with_assumptions(self, assumption:assumption, out:TableInfo):
# gscanner = scan(self, self.group)
# val_var = 'val_'+base62uuid(7)
@@ -694,7 +696,7 @@ class groupby_c(ast_node):
gscanner.add(f'{ce[0]}[{gscanner.it_var}] = ({get_var_names_ex(ex)});\n')
gscanner.add(f'GC::scratch_space->release();')
self.context.emitc('printf("ht_construct: %lld\\n", (chrono::high_resolution_clock::now() - timer).count());timer = chrono::high_resolution_clock::now();')
self.context.emitc('printf("ht_initfrom: %lld\\n", (chrono::high_resolution_clock::now() - timer).count());timer = chrono::high_resolution_clock::now();')
gscanner.finalize()
self.context.emitc(f'GC::scratch_space = nullptr;')
engine/expr.py
+1 -1
View File
@@ -171,7 +171,7 @@ class expr(ast_node):
special_func = [*self.context.udf_map.keys(), *self.context.module_map.keys(),
"maxs", "mins", "avgs", "sums", "deltas", "last", "first",
"stddevs", "vars", "ratios", "pack", "truncate"]
"stddevs", "vars", "ratios", "pack", "truncate", "subvec"]
if (
self.context.special_gb
msc-plugin/libaquery.vcxproj
+1
View File
@@ -289,6 +289,7 @@
<AdditionalIncludeDirectories>$(ProjectDir)\..\monetdb\msvc</AdditionalIncludeDirectories>
<PrecompiledHeaderOutputFile>$(SolutionDir)..\msc-plugin\pch_msc.pch</PrecompiledHeaderOutputFile>
<ProgramDataBaseFileName>$(IntDir)vc$(PlatformToolsetVersion).pdb</ProgramDataBaseFileName>
<FloatingPointModel>Fast</FloatingPointModel>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
msc-plugin/msc-plugin.vcxproj
+10 -8
View File
@@ -134,7 +134,7 @@
<ClCompile>
<WarningLevel>Level3</WarningLevel>
<SDLCheck>true</SDLCheck>
<PreprocessorDefinitions>QUERY_DECLSPEC=__declspec(dllimport);THREADING;_CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<PreprocessorDefinitions>QUERY_DECLSPEC=__declspec(dllexport);THREADING;_CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<ConformanceMode>true</ConformanceMode>
<LanguageStandard>stdcpplatest</LanguageStandard>
<ControlFlowGuard>Guard</ControlFlowGuard>
@@ -166,7 +166,7 @@
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<SDLCheck>false</SDLCheck>
<PreprocessorDefinitions>QUERY_DECLSPEC=__declspec(dllimport);THREADING;_CRT_SECURE_NO_WARNINGS;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<PreprocessorDefinitions>QUERY_DECLSPEC=__declspec(dllexport);THREADING;_CRT_SECURE_NO_WARNINGS;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<ConformanceMode>true</ConformanceMode>
<LanguageStandard>stdcpplatest</LanguageStandard>
<AdditionalIncludeDirectories>$(ProjectDir)\..\monetdb\msvc</AdditionalIncludeDirectories>
@@ -209,7 +209,7 @@
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<SDLCheck>false</SDLCheck>
<PreprocessorDefinitions>QUERY_DECLSPEC=__declspec(dllimport);THREADING;_CRT_SECURE_NO_WARNINGS;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<PreprocessorDefinitions>QUERY_DECLSPEC=__declspec(dllexport);THREADING;_CRT_SECURE_NO_WARNINGS;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<ConformanceMode>true</ConformanceMode>
<LanguageStandard>stdcpplatest</LanguageStandard>
<AdditionalIncludeDirectories>$(ProjectDir)\..\monetdb\msvc</AdditionalIncludeDirectories>
@@ -250,7 +250,7 @@
<ClCompile>
<WarningLevel>Level3</WarningLevel>
<SDLCheck>true</SDLCheck>
<PreprocessorDefinitions>QUERY_DECLSPEC=__declspec(dllimport);THREADING;_ALLOW_RTCc_IN_STL;_CRT_SECURE_NO_WARNINGS;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<PreprocessorDefinitions>QUERY_DECLSPEC=__declspec(dllexport);THREADING;_ALLOW_RTCc_IN_STL;_CRT_SECURE_NO_WARNINGS;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<ConformanceMode>true</ConformanceMode>
<LanguageStandard>stdcpplatest</LanguageStandard>
<ControlFlowGuard>false</ControlFlowGuard>
@@ -300,12 +300,12 @@
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<SDLCheck>false</SDLCheck>
<PreprocessorDefinitions>QUERY_DECLSPEC=__declspec(dllimport);THREADING;_CRT_SECURE_NO_WARNINGS;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<PreprocessorDefinitions>QUERY_DECLSPEC=__declspec(dllexport);THREADING;_CRT_SECURE_NO_WARNINGS;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<ConformanceMode>true</ConformanceMode>
<LanguageStandard>stdcpplatest</LanguageStandard>
<AdditionalIncludeDirectories>$(ProjectDir)\..\monetdb\msvc</AdditionalIncludeDirectories>
<LanguageStandard_C>stdc17</LanguageStandard_C>
<PrecompiledHeader>Use</PrecompiledHeader>
<PrecompiledHeader>NotUsing</PrecompiledHeader>
<PrecompiledHeaderFile>pch_msc.hpp</PrecompiledHeaderFile>
<DebugInformationFormat>None</DebugInformationFormat>
<Optimization>Full</Optimization>
@@ -325,7 +325,9 @@
</PrecompiledHeaderOutputFile>
<EnableModules>false</EnableModules>
<ProgramDataBaseFileName>$(IntDir)vc$(PlatformToolsetVersion).pdb</ProgramDataBaseFileName>
<ForcedIncludeFiles>pch_msc.hpp</ForcedIncludeFiles>
<ForcedIncludeFiles>
</ForcedIncludeFiles>
<FloatingPointModel>Fast</FloatingPointModel>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
@@ -343,7 +345,7 @@
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<SDLCheck>false</SDLCheck>
<PreprocessorDefinitions>QUERY_DECLSPEC=__declspec(dllimport);THREADING;_CRT_SECURE_NO_WARNINGS;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<PreprocessorDefinitions>QUERY_DECLSPEC=__declspec(dllexport);THREADING;_CRT_SECURE_NO_WARNINGS;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<ConformanceMode>true</ConformanceMode>
<LanguageStandard>stdcpplatest</LanguageStandard>
<AdditionalIncludeDirectories>$(ProjectDir)\..\monetdb\msvc</AdditionalIncludeDirectories>
msc-plugin/server.vcxproj
+13
View File
@@ -314,6 +314,18 @@
<LanguageStandard_C>stdc17</LanguageStandard_C>
<AdditionalIncludeDirectories>$(ProjectDir)\..\monetdb\msvc</AdditionalIncludeDirectories>
<DebugInformationFormat>None</DebugInformationFormat>
<Optimization>Full</Optimization>
<InlineFunctionExpansion>AnySuitable</InlineFunctionExpansion>
<FavorSizeOrSpeed>Speed</FavorSizeOrSpeed>
<OmitFramePointers>true</OmitFramePointers>
<EnableFiberSafeOptimizations>true</EnableFiberSafeOptimizations>
<StringPooling>true</StringPooling>
<ExceptionHandling>false</ExceptionHandling>
<BufferSecurityCheck>false</BufferSecurityCheck>
<ControlFlowGuard>false</ControlFlowGuard>
<EnableEnhancedInstructionSet>AdvancedVectorExtensions2</EnableEnhancedInstructionSet>
<FloatingPointModel>Fast</FloatingPointModel>
<FloatingPointExceptions>false</FloatingPointExceptions>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
@@ -324,6 +336,7 @@
<DataExecutionPrevention>false</DataExecutionPrevention>
<AdditionalOptions>/WHOLEARCHIVE:libaquery.lib %(AdditionalOptions)</AdditionalOptions>
<AdditionalLibraryDirectories>../monetdb/msvc;../deps</AdditionalLibraryDirectories>
<AssemblyDebug>false</AssemblyDebug>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='RelWDbgInf|x64'">
msc-plugin/testmain.vcxproj
+13
View File
@@ -316,6 +316,18 @@
<LanguageStandard_C>stdc17</LanguageStandard_C>
<AdditionalIncludeDirectories>$(ProjectDir)\..\monetdb\msvc</AdditionalIncludeDirectories>
<DebugInformationFormat>None</DebugInformationFormat>
<Optimization>Full</Optimization>
<InlineFunctionExpansion>AnySuitable</InlineFunctionExpansion>
<FavorSizeOrSpeed>Speed</FavorSizeOrSpeed>
<OmitFramePointers>true</OmitFramePointers>
<EnableFiberSafeOptimizations>true</EnableFiberSafeOptimizations>
<ExceptionHandling>false</ExceptionHandling>
<BufferSecurityCheck>false</BufferSecurityCheck>
<ControlFlowGuard>false</ControlFlowGuard>
<EnableEnhancedInstructionSet>AdvancedVectorExtensions2</EnableEnhancedInstructionSet>
<FloatingPointModel>Fast</FloatingPointModel>
<FloatingPointExceptions>false</FloatingPointExceptions>
<StringPooling>true</StringPooling>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
@@ -327,6 +339,7 @@
<AdditionalOptions>/WHOLEARCHIVE:libaquery.lib %(AdditionalOptions)</AdditionalOptions>
<Profile>true</Profile>
<AdditionalLibraryDirectories>../deps/;../monetdb/msvc</AdditionalLibraryDirectories>
<AssemblyDebug>false</AssemblyDebug>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='RelWDbgInf|x64'">
msvs-py/msvs-py.pyproj
-1
View File
@@ -49,7 +49,6 @@
<Compile Include="..\engine\orderby.py" />
<Compile Include="..\engine\projection.py" />
<Compile Include="..\engine\scan.py" />
<Compile Include="..\engine\types.py" />
<Compile Include="..\reconstruct\storage.py" />
<Compile Include="..\reconstruct\expr.py" />
<Compile Include="..\reconstruct\__init__.py" />
server/aggregations.h
+6 -3
View File
@@ -380,14 +380,17 @@ auto corr(const VT<T>& x, const VT2<T2>&y) {
sxy += x[i] * y[i];
sy2 += y[i] * y[i];
}
return (sxy - FPType(sx*sy))
return (len*sxy - FPType(sx*sy))
/
(len * sqrt(
(sx2 - FPType(sx*sx)/len) * (sy2 - FPType(sy*sy)/len)
(sqrt(
(len*sx2 - FPType(sx*sx)) * (len*sy2 - FPType(sy*sy))
)
);
}
void pow(auto x, auto y, auto& z) {
z = pow(x, y);
}
template<class T, template<typename ...> class VT>
inline types::GetFPType<types::GetLongType<decays<T>>> stddev(const VT<T>& arr) {
server/gc.h
+2 -1
View File
@@ -85,7 +85,8 @@ public:
start_deamon();
GC::gc_handle = this;
this->scratch.init(1);
this->scratch.init(65536);
GC::scratch_space = &this->scratch;
} // 256 MB
~GC(){
server/libaquery.h
+17
View File
@@ -300,4 +300,21 @@ inline _This_Type* AQ_DupObject(_This_Type* __val) {
void print_monetdb_results(void* _srv, const char* sep, const char* end, uint32_t limit);
StoredProcedure get_procedure(Context* cxt, const char* name);
#define AQTIMER(x) auto __timer##x = std::chrono::high_resolution_clock::now();
#define __AQTIMERLAP__IMPL(x, y) \
printf(#y": %llu\n", (unsigned long long)( \
std::chrono::high_resolution_clock::now() - __timer##x \
).count()); \
__timer##x = std::chrono::high_resolution_clock::now();
#define __AQTIMERLAP_STRIP(X, Y, IMPL, ...) IMPL
#define AQTIMERLAP(...) \
__AQTIMERLAP_STRIP(__VA_ARGS__, \
__AQTIMERLAP__IMPL(__VA_ARGS__), \
__AQTIMERLAP__IMPL(, __VA_ARGS__), \
__AQTIMERLAP__IMPL(,) \
)
#endif
server/table.h
+11 -11
View File
@@ -149,18 +149,18 @@ public:
return *this;
}
ColView<_Ty> operator [](vector_type<uint32_t>& idxs) const {
return ColView<_Ty>(*this, std::move(idxs));
// ColView<_Ty> operator [](vector_type<uint32_t>& idxs) const {
// return ColView<_Ty>(*this, idxs);
// }
// ColView<_Ty> operator [](const vector_type<uint32_t>& idxs) const {
// return ColView<_Ty>(*this, idxs);
// }
vector_type<_Ty> operator[](vector_type<uint32_t>& idxs) const {
vector_type<_Ty> ret(idxs.size);
for (uint32_t i = 0; i < idxs.size; ++i)
ret.container[i] = this->container[idxs[i]];
return ret;
}
ColView<_Ty> operator [](const vector_type<uint32_t>& idxs) const {
return ColView<_Ty>(*this, idxs);
}
//vector_type<_Ty> operator[](vector_type<uint32_t>& idxs) const {
// vector_type<_Ty> ret(idxs.size);
// for (uint32_t i = 0; i < idxs.size; ++i)
// ret.container[i] = this->container[idxs[i]];
// return ret;
//}
vector_type<_Ty> operator [](const std::vector<bool>& idxs) const {
vector_type<_Ty> ret (this->size);
uint32_t i = 0;
server/vector_type.hpp
+9 -3
View File
@@ -27,13 +27,19 @@ template <typename _Ty>
class vector_type : public vector_base<_Ty>{
public:
typedef vector_type<_Ty> Decayed_t;
void inline _ref(vector_type<_Ty>& vt) {
// make a reference of vt
this->size = vt.size;
this->capacity = 0;
this->container = vt.container;
}
void inline _copy(const vector_type<_Ty>& vt) {
// quick init while using malloc
//if (capacity > 0) free(container);
this->size = vt.size;
this->capacity = vt.capacity;
if (capacity) {
//puts("copy");
puts("copy");
this->container = (_Ty*)malloc(size * sizeof(_Ty));
memcpy(container, vt.container, sizeof(_Ty) * size);
}
@@ -63,7 +69,7 @@ public:
container = (_Ty*)GC::scratch_space->alloc(size * sizeof(_Ty));
this->capacity = 0;
}
else{
else {
container = (_Ty*)malloc(size * sizeof(_Ty));
}
// TODO: calloc for objects.
@@ -84,7 +90,7 @@ public:
_copy(vt);
}
constexpr vector_type(vector_type<_Ty>& vt) noexcept : capacity(0) {
_move(std::move(vt));
_ref(vt);
}
constexpr vector_type(vector_type<_Ty>&& vt) noexcept : capacity(0) {
_move(std::move(vt));