WIP: rework exec-engine structure, GC, fast print

dev
Bill 2 years ago
parent be7fb9f523
commit fa1f9822bc

@ -8,15 +8,15 @@ ifeq ($(AQ_DEBUG), 1)
LINKFLAGS =
else
OPTFLAGS = -O3 -DNDEBUG -fno-stack-protector
LINKFLAGS = -flto
LINKFLAGS = -flto -s
endif
SHAREDFLAGS = -shared
FPIC = -fPIC
COMPILER = $(shell $(CXX) --version | grep -q clang && echo clang|| echo gcc)
LIBTOOL = ar rcs
USELIB_FLAG = -Wl,--whole-archive,libaquery.a -Wl,-no-whole-archive
LIBAQ_SRC = server/server.cpp server/monetdb_conn.cpp server/io.cpp
LIBAQ_OBJ = server.o monetdb_conn.o io.o
LIBAQ_SRC = server/monetdb_conn.cpp server/libaquery.cpp
LIBAQ_OBJ = monetdb_conn.o libaquery.o
SEMANTIC_INTERPOSITION = -fno-semantic-interposition
RANLIB = ranlib
@ -118,19 +118,21 @@ info:
pch:
$(CXX) -x c++-header server/pch.hpp $(FPIC) $(CXXFLAGS)
libaquery.a:
$(CXX) -c $(FPIC) $(PCHFLAGS) $(LIBAQ_SRC) $(MonetDB_LIB) $(OS_SUPPORT) $(CXXFLAGS) &&\
$(CXX) -c $(FPIC) $(PCHFLAGS) $(LIBAQ_SRC) $(OS_SUPPORT) $(CXXFLAGS) &&\
$(LIBTOOL) libaquery.a $(LIBAQ_OBJ) &&\
$(RANLIB) libaquery.a
warmup:
$(CXX) $(SHAREDFLAGS) msc-plugin/dummy.cpp libaquery.a -o dll.so
server.bin:
$(CXX) $(LIBAQ_SRC) $(BINARYFLAGS) $(OS_SUPPORT) -o server.bin
launcher:
$(CXX) -D__AQ_BUILD_LAUNCHER__ $(LIBAQ_SRC) $(OS_SUPPORT) $(BINARYFLAGS) -o aq
server.so:
# $(CXX) -z muldefs server/server.cpp server/monetdb_conn.cpp -fPIC -shared $(OS_SUPPORT) monetdb/msvc/monetdbe.dll --std=c++1z -O3 -march=native -o server.so -I./monetdb/msvc
$(CXX) $(SHAREDFLAGS) $(PCHFLAGS) $(LIBAQ_SRC) $(OS_SUPPORT) -o server.so
$(CXX) $(SHAREDFLAGS) $(PCHFLAGS) $(LIBAQ_SRC) server/server.cpp server/dragonbox/dragonbox_to_chars.cpp $(OS_SUPPORT) -o server.so
server_uselib:
$(CXX) $(SHAREDFLAGS) $(USELIB_FLAG),libaquery.a -o server.so
$(CXX) $(SHAREDFLAGS) server/server.cpp libaquery.a server/dragonbox/dragonbox_to_chars.cpp -o server.so
snippet:
$(CXX) $(SHAREDFLAGS) $(PCHFLAGS) out.cpp $(LIBAQ_SRC) -o dll.so

@ -231,7 +231,7 @@ DROP TABLE my_table IF EXISTS
- `next(col), prev(col)`: moving column back and forth by 1, e.g. `next(col)[i] = col[i+1]`.
- `first(col), last(col)`: first and last value of a column, i.e. `first(col)= col[0]`, `last(col) = col[n-1]`.
- `sqrt(x), trunc(x), and other builtin math functions`: value-wise math operations. `sqrt(x)[i] = sqrt(x[i])`
- `pack(cols, ...)`: pack multiple columns into a single column.
- `pack(cols, ...)`: pack multiple columns with exact same type into a single column.
# Architecture
![Architecture](./docs/arch-hybrid.svg)
@ -287,3 +287,24 @@ DROP TABLE my_table IF EXISTS
- [ ] Bug: Order By after Group By
- [ ] Functionality: Having clause, With clause
- [ ] Decouple expr.py
# Credit:
- [mo-sql-parsing](https://github.com/klahnakoski/mo-sql-parsing) <br>
Author: Kyle Lahnakoski <br>
License (Mozilla Public License 2.0): https://github.com/klahnakoski/mo-sql-parsing/blob/dev/LICENSE
- [Fast C++ CSV pParser](https://github.com/ben-strasser/fast-cpp-csv-parser) <br>
Author: Ben Strasser <br>
License (BSD 3-Clause License): https://github.com/ben-strasser/fast-cpp-csv-parser/blob/master/LICENSE
- [Dragonbox](https://github.com/jk-jeon/dragonbox)<br>
Author: Junekey Jeon
License (Boost, Apache2-LLVM): <br>https://github.com/jk-jeon/dragonbox/blob/master/LICENSE-Boost <br>
https://github.com/jk-jeon/dragonbox/blob/master/LICENSE-Apache2-LLVM
- [itoa](https://github.com/jeaiii/itoa) <br>
Author: James Edward Anhalt III <br>
License (MIT): https://github.com/jeaiii/itoa/blob/main/LICENSE
- [MobetDB] (https://www.monetdb.org) <br>
License (Mozilla Public License): https://github.com/MonetDB/MonetDB/blob/master/license.txt

@ -11,6 +11,7 @@ cygroot = 'c:/msys64/usr/bin'
msbuildroot = ''
os_platform = 'unknown'
build_driver = 'Auto'
compilation_output = True
def init_config():
global __config_initialized__, os_platform, msbuildroot, build_driver

@ -73,7 +73,7 @@ class checksums:
class build_manager:
sourcefiles = [
'build.py', 'Makefile',
'server/server.cpp', 'server/io.cpp',
'server/server.cpp', 'server/libaquery.cpp',
'server/monetdb_conn.cpp', 'server/threading.cpp',
'server/winhelper.cpp'
]
@ -94,6 +94,9 @@ class build_manager:
return False
def build(self, stdout = sys.stdout, stderr = sys.stderr):
ret = True
if not aquery_config.compilation_output:
stdout = nullstream
stderr = nullstream
for c in self.build_cmd:
if c:
try: # only last success matters
@ -102,6 +105,8 @@ class build_manager:
ret = False
pass
return ret
def warmup(self):
return True
class MakefileDriver(DriverBase):
def __init__(self, mgr : 'build_manager') -> None:
@ -113,7 +118,7 @@ class build_manager:
mgr.cxx = os.environ['CXX']
if 'AQ_DEBUG' not in os.environ:
os.environ['AQ_DEBUG'] = '0' if mgr.OptimizationLv else '1'
def libaquery_a(self):
self.build_cmd = [['rm', 'libaquery.a'],['make', 'libaquery.a']]
return self.build()
@ -168,6 +173,10 @@ class build_manager:
self.build_cmd = [[aquery_config.msbuildroot, loc, self.opt, self.platform]]
return self.build()
def warmup(self):
self.build_cmd = [['make', 'warmup']]
return self.build()
#class PythonDriver(DriverBase):
# def __init__(self, mgr : 'build_manager') -> None:
# super().__init__(mgr)
@ -223,6 +232,9 @@ class build_manager:
current.calc(self.cxx, libaquery_a)
with open('.cached', 'wb') as cache_sig:
cache_sig.write(pickle.dumps(current))
self.driver.warmup()
else:
if aquery_config.os_platform == 'mac':
os.system('./arch-check.sh')

@ -238,7 +238,7 @@
<ItemGroup>
<ClCompile Include="..\server\server.cpp" />
<ClCompile Include="..\server\winhelper.cpp" />
<ClCompile Include="..\server\io.cpp" />
<ClCompile Include="..\server\libaquery.cpp" />
<ClCompile Include="..\server\monetdb_conn.cpp" />
</ItemGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />

@ -156,6 +156,7 @@ class Context:
self.queries = []
self.module_init_loc = 0
self.special_gb = False
self.has_dll = False
def __init__(self):
self.tables_byname = dict()
@ -169,7 +170,6 @@ class Context:
self.udf_agg_map = dict()
self.use_columnstore = False
self.print = print
self.has_dll = False
self.dialect = 'MonetDB'
self.is_msvc = False
self.have_hge = False

@ -1,6 +1,6 @@
debug:
g++ -g3 -O0 server/server.cpp server/io.cpp -o a.out -Wall -Wextra -Wpedantic -lpthread
g++ -g3 -O0 server/server.cpp server/libaquery.cpp -o a.out -Wall -Wextra -Wpedantic -lpthread
test:
g++ --std=c++1z -g3 -O0 server.cpp io.cpp -o a.out -Wall -Wextra -Wpedantic -lpthread
g++ --std=c++1z -g3 -O0 server.cpp libaquery.cpp -o a.out -Wall -Wextra -Wpedantic -lpthread

File diff suppressed because it is too large Load Diff

@ -0,0 +1,519 @@
// Copyright 2020-2022 Junekey Jeon
//
// The contents of this file may be used under the terms of
// the Apache License v2.0 with LLVM Exceptions.
//
// (See accompanying file LICENSE-Apache or copy at
// https://llvm.org/foundation/relicensing/LICENSE.txt)
//
// Alternatively, the contents of this file may be used under the terms of
// the Boost Software License, Version 1.0.
// (See accompanying file LICENSE-Boost or copy at
// https://www.boost.org/LICENSE_1_0.txt)
//
// Unless required by applicable law or agreed to in writing, this software
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.
#include "dragonbox_to_chars.h"
#if defined(__GNUC__) || defined(__clang__)
#define JKJ_FORCEINLINE inline __attribute__((always_inline))
#elif defined(_MSC_VER)
#define JKJ_FORCEINLINE __forceinline
#else
#define JKJ_FORCEINLINE inline
#endif
namespace jkj::dragonbox {
namespace to_chars_detail {
// These "//"'s are to prevent clang-format to ruin this nice alignment.
// Thanks to reddit user u/mcmcc:
// https://www.reddit.com/r/cpp/comments/so3wx9/dragonbox_110_is_released_a_fast_floattostring/hw8z26r/?context=3
static constexpr char radix_100_table[] = {
'0', '0', '0', '1', '0', '2', '0', '3', '0', '4', //
'0', '5', '0', '6', '0', '7', '0', '8', '0', '9', //
'1', '0', '1', '1', '1', '2', '1', '3', '1', '4', //
'1', '5', '1', '6', '1', '7', '1', '8', '1', '9', //
'2', '0', '2', '1', '2', '2', '2', '3', '2', '4', //
'2', '5', '2', '6', '2', '7', '2', '8', '2', '9', //
'3', '0', '3', '1', '3', '2', '3', '3', '3', '4', //
'3', '5', '3', '6', '3', '7', '3', '8', '3', '9', //
'4', '0', '4', '1', '4', '2', '4', '3', '4', '4', //
'4', '5', '4', '6', '4', '7', '4', '8', '4', '9', //
'5', '0', '5', '1', '5', '2', '5', '3', '5', '4', //
'5', '5', '5', '6', '5', '7', '5', '8', '5', '9', //
'6', '0', '6', '1', '6', '2', '6', '3', '6', '4', //
'6', '5', '6', '6', '6', '7', '6', '8', '6', '9', //
'7', '0', '7', '1', '7', '2', '7', '3', '7', '4', //
'7', '5', '7', '6', '7', '7', '7', '8', '7', '9', //
'8', '0', '8', '1', '8', '2', '8', '3', '8', '4', //
'8', '5', '8', '6', '8', '7', '8', '8', '8', '9', //
'9', '0', '9', '1', '9', '2', '9', '3', '9', '4', //
'9', '5', '9', '6', '9', '7', '9', '8', '9', '9' //
};
static constexpr char radix_100_head_table[] = {
'0', '.', '1', '.', '2', '.', '3', '.', '4', '.', //
'5', '.', '6', '.', '7', '.', '8', '.', '9', '.', //
'1', '.', '1', '.', '1', '.', '1', '.', '1', '.', //
'1', '.', '1', '.', '1', '.', '1', '.', '1', '.', //
'2', '.', '2', '.', '2', '.', '2', '.', '2', '.', //
'2', '.', '2', '.', '2', '.', '2', '.', '2', '.', //
'3', '.', '3', '.', '3', '.', '3', '.', '3', '.', //
'3', '.', '3', '.', '3', '.', '3', '.', '3', '.', //
'4', '.', '4', '.', '4', '.', '4', '.', '4', '.', //
'4', '.', '4', '.', '4', '.', '4', '.', '4', '.', //
'5', '.', '5', '.', '5', '.', '5', '.', '5', '.', //
'5', '.', '5', '.', '5', '.', '5', '.', '5', '.', //
'6', '.', '6', '.', '6', '.', '6', '.', '6', '.', //
'6', '.', '6', '.', '6', '.', '6', '.', '6', '.', //
'7', '.', '7', '.', '7', '.', '7', '.', '7', '.', //
'7', '.', '7', '.', '7', '.', '7', '.', '7', '.', //
'8', '.', '8', '.', '8', '.', '8', '.', '8', '.', //
'8', '.', '8', '.', '8', '.', '8', '.', '8', '.', //
'9', '.', '9', '.', '9', '.', '9', '.', '9', '.', //
'9', '.', '9', '.', '9', '.', '9', '.', '9', '.' //
};
// These digit generation routines are inspired by James Anhalt's itoa algorithm:
// https://github.com/jeaiii/itoa
// The main idea is for given n, find y such that floor(10^k * y / 2^32) = n holds,
// where k is an appropriate integer depending on the length of n.
// For example, if n = 1234567, we set k = 6. In this case, we have
// floor(y / 2^32) = 1,
// floor(10^2 * ((10^0 * y) mod 2^32) / 2^32) = 23,
// floor(10^2 * ((10^2 * y) mod 2^32) / 2^32) = 45, and
// floor(10^2 * ((10^4 * y) mod 2^32) / 2^32) = 67.
// See https://jk-jeon.github.io/posts/2022/02/jeaiii-algorithm/ for more explanation.
JKJ_FORCEINLINE static void print_9_digits(std::uint32_t s32, int& exponent,
char*& buffer) noexcept {
// -- IEEE-754 binary32
// Since we do not cut trailing zeros in advance, s32 must be of 6~9 digits
// unless the original input was subnormal.
// In particular, when it is of 9 digits it shouldn't have any trailing zeros.
// -- IEEE-754 binary64
// In this case, s32 must be of 7~9 digits unless the input is subnormal,
// and it shouldn't have any trailing zeros if it is of 9 digits.
if (s32 >= 1'0000'0000) {
// 9 digits.
// 1441151882 = ceil(2^57 / 1'0000'0000) + 1
auto prod = s32 * std::uint64_t(1441151882);
prod >>= 25;
std::memcpy(buffer, radix_100_head_table + std::uint32_t(prod >> 32) * 2, 2);
prod = std::uint32_t(prod) * std::uint64_t(100);
std::memcpy(buffer + 2, radix_100_table + std::uint32_t(prod >> 32) * 2, 2);
prod = std::uint32_t(prod) * std::uint64_t(100);
std::memcpy(buffer + 4, radix_100_table + std::uint32_t(prod >> 32) * 2, 2);
prod = std::uint32_t(prod) * std::uint64_t(100);
std::memcpy(buffer + 6, radix_100_table + std::uint32_t(prod >> 32) * 2, 2);
prod = std::uint32_t(prod) * std::uint64_t(100);
std::memcpy(buffer + 8, radix_100_table + std::uint32_t(prod >> 32) * 2, 2);
exponent += 8;
buffer += 10;
}
else if (s32 >= 100'0000) {
// 7 or 8 digits.
// 281474978 = ceil(2^48 / 100'0000) + 1
auto prod = s32 * std::uint64_t(281474978);
prod >>= 16;
auto two_digits = std::uint32_t(prod >> 32);
// If s32 is of 8 digits, increase the exponent by 7.
// Otherwise, increase it by 6.
exponent += (6 + unsigned(two_digits >= 10));
// Write the first digit and the decimal point.
std::memcpy(buffer, radix_100_head_table + two_digits * 2, 2);
// This third character may be overwritten later but we don't care.
buffer[2] = radix_100_table[two_digits * 2 + 1];
// Remaining 6 digits are all zero?
if (std::uint32_t(prod) <= std::uint32_t((std::uint64_t(1) << 32) / 100'0000)) {
// The number of characters actually written is:
// 1, if only the first digit is nonzero, which means that either s32 is of 7
// digits or it is of 8 digits but the second digit is zero, or
// 3, otherwise.
// Note that buffer[2] is never zero if s32 is of 7 digits, because the input is
// never zero.
buffer += (1 + (unsigned(two_digits >= 10) & unsigned(buffer[2] > '0')) * 2);
}
else {
// At least one of the remaining 6 digits are nonzero.
// After this adjustment, now the first destination becomes buffer + 2.
buffer += unsigned(two_digits >= 10);
// Obtain the next two digits.
prod = std::uint32_t(prod) * std::uint64_t(100);
two_digits = std::uint32_t(prod >> 32);
std::memcpy(buffer + 2, radix_100_table + two_digits * 2, 2);
// Remaining 4 digits are all zero?
if (std::uint32_t(prod) <= std::uint32_t((std::uint64_t(1) << 32) / 1'0000)) {
buffer += (3 + unsigned(buffer[3] > '0'));
}
else {
// At least one of the remaining 4 digits are nonzero.
// Obtain the next two digits.
prod = std::uint32_t(prod) * std::uint64_t(100);
two_digits = std::uint32_t(prod >> 32);
std::memcpy(buffer + 4, radix_100_table + two_digits * 2, 2);
// Remaining 2 digits are all zero?
if (std::uint32_t(prod) <= std::uint32_t((std::uint64_t(1) << 32) / 100)) {
buffer += (5 + unsigned(buffer[5] > '0'));
}
else {
// Obtain the last two digits.
prod = std::uint32_t(prod) * std::uint64_t(100);
two_digits = std::uint32_t(prod >> 32);
std::memcpy(buffer + 6, radix_100_table + two_digits * 2, 2);
buffer += (7 + unsigned(buffer[7] > '0'));
}
}
}
}
else if (s32 >= 1'0000) {
// 5 or 6 digits.
// 429497 = ceil(2^32 / 1'0000)
auto prod = s32 * std::uint64_t(429497);
auto two_digits = std::uint32_t(prod >> 32);
// If s32 is of 6 digits, increase the exponent by 5.
// Otherwise, increase it by 4.
exponent += (4 + unsigned(two_digits >= 10));
// Write the first digit and the decimal point.
std::memcpy(buffer, radix_100_head_table + two_digits * 2, 2);
// This third character may be overwritten later but we don't care.
buffer[2] = radix_100_table[two_digits * 2 + 1];
// Remaining 4 digits are all zero?
if (std::uint32_t(prod) <= std::uint32_t((std::uint64_t(1) << 32) / 1'0000)) {
// The number of characters actually written is 1 or 3, similarly to the case of
// 7 or 8 digits.
buffer += (1 + (unsigned(two_digits >= 10) & unsigned(buffer[2] > '0')) * 2);
}
else {
// At least one of the remaining 4 digits are nonzero.
// After this adjustment, now the first destination becomes buffer + 2.
buffer += unsigned(two_digits >= 10);
// Obtain the next two digits.
prod = std::uint32_t(prod) * std::uint64_t(100);
two_digits = std::uint32_t(prod >> 32);
std::memcpy(buffer + 2, radix_100_table + two_digits * 2, 2);
// Remaining 2 digits are all zero?
if (std::uint32_t(prod) <= std::uint32_t((std::uint64_t(1) << 32) / 100)) {
buffer += (3 + unsigned(buffer[3] > '0'));
}
else {
// Obtain the last two digits.
prod = std::uint32_t(prod) * std::uint64_t(100);
two_digits = std::uint32_t(prod >> 32);
std::memcpy(buffer + 4, radix_100_table + two_digits * 2, 2);
buffer += (5 + unsigned(buffer[5] > '0'));
}
}
}
else if (s32 >= 100) {
// 3 or 4 digits.
// 42949673 = ceil(2^32 / 100)
auto prod = s32 * std::uint64_t(42949673);
auto two_digits = std::uint32_t(prod >> 32);
// If s32 is of 4 digits, increase the exponent by 3.
// Otherwise, increase it by 2.
exponent += (2 + int(two_digits >= 10));
// Write the first digit and the decimal point.
std::memcpy(buffer, radix_100_head_table + two_digits * 2, 2);
// This third character may be overwritten later but we don't care.
buffer[2] = radix_100_table[two_digits * 2 + 1];
// Remaining 2 digits are all zero?
if (std::uint32_t(prod) <= std::uint32_t((std::uint64_t(1) << 32) / 100)) {
// The number of characters actually written is 1 or 3, similarly to the case of
// 7 or 8 digits.
buffer += (1 + (unsigned(two_digits >= 10) & unsigned(buffer[2] > '0')) * 2);
}
else {
// At least one of the remaining 2 digits are nonzero.
// After this adjustment, now the first destination becomes buffer + 2.
buffer += unsigned(two_digits >= 10);
// Obtain the last two digits.
prod = std::uint32_t(prod) * std::uint64_t(100);
two_digits = std::uint32_t(prod >> 32);
std::memcpy(buffer + 2, radix_100_table + two_digits * 2, 2);
buffer += (3 + unsigned(buffer[3] > '0'));
}
}
else {
// 1 or 2 digits.
// If s32 is of 2 digits, increase the exponent by 1.
exponent += int(s32 >= 10);
// Write the first digit and the decimal point.
std::memcpy(buffer, radix_100_head_table + s32 * 2, 2);
// This third character may be overwritten later but we don't care.
buffer[2] = radix_100_table[s32 * 2 + 1];
// The number of characters actually written is 1 or 3, similarly to the case of
// 7 or 8 digits.
buffer += (1 + (unsigned(s32 >= 10) & unsigned(buffer[2] > '0')) * 2);
}
}
template <>
char* to_chars<float, default_float_traits<float>>(std::uint32_t s32, int exponent,
char* buffer) noexcept {
// Print significand.
print_9_digits(s32, exponent, buffer);
// Print exponent and return
if (exponent < 0) {
std::memcpy(buffer, "E-", 2);
buffer += 2;
exponent = -exponent;
}
else if (exponent > 0) {
buffer[0] = 'E';
buffer += 1;
}
else {
return buffer;
}
if (exponent >= 10) {
std::memcpy(buffer, &radix_100_table[exponent * 2], 2);
buffer += 2;
}
else {
buffer[0] = char('0' + exponent);
buffer += 1;
}
return buffer;
}
template <>
char* to_chars<double, default_float_traits<double>>(std::uint64_t const significand,
int exponent, char* buffer) noexcept {
// Print significand by decomposing it into a 9-digit block and a 8-digit block.
std::uint32_t first_block, second_block;
bool no_second_block;
if (significand >= 1'0000'0000) {
first_block = std::uint32_t(significand / 1'0000'0000);
second_block = std::uint32_t(significand) - first_block * 1'0000'0000;
exponent += 8;
no_second_block = (second_block == 0);
}
else {
first_block = std::uint32_t(significand);
no_second_block = true;
}
if (no_second_block) {
print_9_digits(first_block, exponent, buffer);
}
else {
// We proceed similarly to print_9_digits(), but since we do not need to remove
// trailing zeros, the procedure is a bit simpler.
if (first_block >= 1'0000'0000) {
// The input is of 17 digits, thus there should be no trailing zero at all.
// The first block is of 9 digits.
// 1441151882 = ceil(2^57 / 1'0000'0000) + 1
auto prod = first_block * std::uint64_t(1441151882);
prod >>= 25;
std::memcpy(buffer, radix_100_head_table + std::uint32_t(prod >> 32) * 2, 2);
prod = std::uint32_t(prod) * std::uint64_t(100);
std::memcpy(buffer + 2, radix_100_table + std::uint32_t(prod >> 32) * 2, 2);
prod = std::uint32_t(prod) * std::uint64_t(100);
std::memcpy(buffer + 4, radix_100_table + std::uint32_t(prod >> 32) * 2, 2);
prod = std::uint32_t(prod) * std::uint64_t(100);
std::memcpy(buffer + 6, radix_100_table + std::uint32_t(prod >> 32) * 2, 2);
prod = std::uint32_t(prod) * std::uint64_t(100);
std::memcpy(buffer + 8, radix_100_table + std::uint32_t(prod >> 32) * 2, 2);
// The second block is of 8 digits.
// 281474978 = ceil(2^48 / 100'0000) + 1
prod = second_block * std::uint64_t(281474978);
prod >>= 16;
prod += 1;
std::memcpy(buffer + 10, radix_100_table + std::uint32_t(prod >> 32) * 2, 2);
prod = std::uint32_t(prod) * std::uint64_t(100);
std::memcpy(buffer + 12, radix_100_table + std::uint32_t(prod >> 32) * 2, 2);
prod = std::uint32_t(prod) * std::uint64_t(100);
std::memcpy(buffer + 14, radix_100_table + std::uint32_t(prod >> 32) * 2, 2);
prod = std::uint32_t(prod) * std::uint64_t(100);
std::memcpy(buffer + 16, radix_100_table + std::uint32_t(prod >> 32) * 2, 2);
exponent += 8;
buffer += 18;
}
else {
if (first_block >= 100'0000) {
// 7 or 8 digits.
// 281474978 = ceil(2^48 / 100'0000) + 1
auto prod = first_block * std::uint64_t(281474978);
prod >>= 16;
auto two_digits = std::uint32_t(prod >> 32);
std::memcpy(buffer, radix_100_head_table + two_digits * 2, 2);
buffer[2] = radix_100_table[two_digits * 2 + 1];
exponent += (6 + unsigned(two_digits >= 10));
buffer += unsigned(two_digits >= 10);
// Print remaining 6 digits.
prod = std::uint32_t(prod) * std::uint64_t(100);
std::memcpy(buffer + 2, radix_100_table + std::uint32_t(prod >> 32) * 2, 2);
prod = std::uint32_t(prod) * std::uint64_t(100);
std::memcpy(buffer + 4, radix_100_table + std::uint32_t(prod >> 32) * 2, 2);
prod = std::uint32_t(prod) * std::uint64_t(100);
std::memcpy(buffer + 6, radix_100_table + std::uint32_t(prod >> 32) * 2, 2);
buffer += 8;
}
else if (first_block >= 1'0000) {
// 5 or 6 digits.
// 429497 = ceil(2^32 / 1'0000)
auto prod = first_block * std::uint64_t(429497);
auto two_digits = std::uint32_t(prod >> 32);
std::memcpy(buffer, radix_100_head_table + two_digits * 2, 2);
buffer[2] = radix_100_table[two_digits * 2 + 1];
exponent += (4 + unsigned(two_digits >= 10));
buffer += unsigned(two_digits >= 10);
// Print remaining 4 digits.
prod = std::uint32_t(prod) * std::uint64_t(100);
std::memcpy(buffer + 2, radix_100_table + std::uint32_t(prod >> 32) * 2, 2);
prod = std::uint32_t(prod) * std::uint64_t(100);
std::memcpy(buffer + 4, radix_100_table + std::uint32_t(prod >> 32) * 2, 2);
buffer += 6;
}
else if (first_block >= 100) {
// 3 or 4 digits.
// 42949673 = ceil(2^32 / 100)
auto prod = first_block * std::uint64_t(42949673);
auto two_digits = std::uint32_t(prod >> 32);
std::memcpy(buffer, radix_100_head_table + two_digits * 2, 2);
buffer[2] = radix_100_table[two_digits * 2 + 1];
exponent += (2 + unsigned(two_digits >= 10));
buffer += unsigned(two_digits >= 10);
// Print remaining 2 digits.
prod = std::uint32_t(prod) * std::uint64_t(100);
std::memcpy(buffer + 2, radix_100_table + std::uint32_t(prod >> 32) * 2, 2);
buffer += 4;
}
else {
// 1 or 2 digits.
std::memcpy(buffer, radix_100_head_table + first_block * 2, 2);
buffer[2] = radix_100_table[first_block * 2 + 1];
exponent += unsigned(first_block >= 10);
buffer += (2 + unsigned(first_block >= 10));
}
// Next, print the second block.
// The second block is of 8 digits, but we may have trailing zeros.
// 281474978 = ceil(2^48 / 100'0000) + 1
auto prod = second_block * std::uint64_t(281474978);
prod >>= 16;
prod += 1;
auto two_digits = std::uint32_t(prod >> 32);
std::memcpy(buffer, radix_100_table + two_digits * 2, 2);
// Remaining 6 digits are all zero?
if (std::uint32_t(prod) <= std::uint32_t((std::uint64_t(1) << 32) / 100'0000)) {
buffer += (1 + unsigned(buffer[1] > '0'));
}
else {
// Obtain the next two digits.
prod = std::uint32_t(prod) * std::uint64_t(100);
two_digits = std::uint32_t(prod >> 32);
std::memcpy(buffer + 2, radix_100_table + two_digits * 2, 2);
// Remaining 4 digits are all zero?
if (std::uint32_t(prod) <=
std::uint32_t((std::uint64_t(1) << 32) / 1'0000)) {
buffer += (3 + unsigned(buffer[3] > '0'));
}
else {
// Obtain the next two digits.
prod = std::uint32_t(prod) * std::uint64_t(100);
two_digits = std::uint32_t(prod >> 32);
std::memcpy(buffer + 4, radix_100_table + two_digits * 2, 2);
// Remaining 2 digits are all zero?
if (std::uint32_t(prod) <=
std::uint32_t((std::uint64_t(1) << 32) / 100)) {
buffer += (5 + unsigned(buffer[5] > '0'));
}
else {
// Obtain the last two digits.
prod = std::uint32_t(prod) * std::uint64_t(100);
two_digits = std::uint32_t(prod >> 32);
std::memcpy(buffer + 6, radix_100_table + two_digits * 2, 2);
buffer += (7 + unsigned(buffer[7] > '0'));
}
}
}
}
}
// Print exponent and return
if (exponent < 0) {
std::memcpy(buffer, "E-", 2);
buffer += 2;
exponent = -exponent;
}
else if (exponent > 0) {
buffer[0] = 'E';
buffer += 1;
}
else {
return buffer;
}
if (exponent >= 100) {
// d1 = exponent / 10; d2 = exponent % 10;
// 6554 = ceil(2^16 / 10)
auto prod = std::uint32_t(exponent) * std::uint32_t(6554);
auto d1 = prod >> 16;
prod = std::uint16_t(prod) * std::uint32_t(5); // * 10
auto d2 = prod >> 15; // >> 16
std::memcpy(buffer, &radix_100_table[d1 * 2], 2);
buffer[2] = char('0' + d2);
buffer += 3;
}
else if (exponent >= 10) {
std::memcpy(buffer, &radix_100_table[exponent * 2], 2);
buffer += 2;
}
else {
buffer[0] = char('0' + exponent);
buffer += 1;
}
return buffer;
}
}
}

@ -0,0 +1,108 @@
// Copyright 2020-2022 Junekey Jeon
//
// The contents of this file may be used under the terms of
// the Apache License v2.0 with LLVM Exceptions.
//
// (See accompanying file LICENSE-Apache or copy at
// https://llvm.org/foundation/relicensing/LICENSE.txt)
//
// Alternatively, the contents of this file may be used under the terms of
// the Boost Software License, Version 1.0.
// (See accompanying file LICENSE-Boost or copy at
// https://www.boost.org/LICENSE_1_0.txt)
//
// Unless required by applicable law or agreed to in writing, this software
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.
#ifndef JKJ_HEADER_DRAGONBOX_TO_CHARS
#define JKJ_HEADER_DRAGONBOX_TO_CHARS
#include "dragonbox.h"
namespace jkj::dragonbox {
namespace to_chars_detail {
template <class Float, class FloatTraits>
extern char* to_chars(typename FloatTraits::carrier_uint significand, int exponent,
char* buffer) noexcept;
// Avoid needless ABI overhead incurred by tag dispatch.
template <class PolicyHolder, class Float, class FloatTraits>
char* to_chars_n_impl(float_bits<Float, FloatTraits> br, char* buffer) noexcept {
auto const exponent_bits = br.extract_exponent_bits();
auto const s = br.remove_exponent_bits(exponent_bits);
if (br.is_finite(exponent_bits)) {
if (s.is_negative()) {
*buffer = '-';
++buffer;
}
if (br.is_nonzero()) {
auto result = to_decimal<Float, FloatTraits>(
s, exponent_bits, policy::sign::ignore, policy::trailing_zero::ignore,
typename PolicyHolder::decimal_to_binary_rounding_policy{},
typename PolicyHolder::binary_to_decimal_rounding_policy{},
typename PolicyHolder::cache_policy{});
return to_chars_detail::to_chars<Float, FloatTraits>(result.significand,
result.exponent, buffer);
}
else {
std::memcpy(buffer, "0E0", 3);
return buffer + 3;
}
}
else {
if (s.has_all_zero_significand_bits()) {
if (s.is_negative()) {
*buffer = '-';
++buffer;
}
std::memcpy(buffer, "Infinity", 8);
return buffer + 8;
}
else {
std::memcpy(buffer, "NaN", 3);
return buffer + 3;
}
}
}
}
// Returns the next-to-end position
template <class Float, class FloatTraits = default_float_traits<Float>, class... Policies>
char* to_chars_n(Float x, char* buffer, Policies... policies) noexcept {
using namespace jkj::dragonbox::detail::policy_impl;
using policy_holder = decltype(make_policy_holder(
base_default_pair_list<base_default_pair<decimal_to_binary_rounding::base,
decimal_to_binary_rounding::nearest_to_even>,
base_default_pair<binary_to_decimal_rounding::base,
binary_to_decimal_rounding::to_even>,
base_default_pair<cache::base, cache::full>>{},
policies...));
return to_chars_detail::to_chars_n_impl<policy_holder>(float_bits<Float, FloatTraits>(x),
buffer);
}
// Null-terminate and bypass the return value of fp_to_chars_n
template <class Float, class FloatTraits = default_float_traits<Float>, class... Policies>
char* to_chars(Float x, char* buffer, Policies... policies) noexcept {
auto ptr = to_chars_n<Float, FloatTraits>(x, buffer, policies...);
*ptr = '\0';
return ptr;
}
// Maximum required buffer size (excluding null-terminator)
template <class FloatFormat>
inline constexpr std::size_t max_output_string_length =
std::is_same_v<FloatFormat, ieee754_binary32>
?
// sign(1) + significand(9) + decimal_point(1) + exp_marker(1) + exp_sign(1) + exp(2)
(1 + 9 + 1 + 1 + 1 + 2)
:
// format == ieee754_format::binary64
// sign(1) + significand(17) + decimal_point(1) + exp_marker(1) + exp_sign(1) + exp(3)
(1 + 17 + 1 + 1 + 1 + 3);
}
#endif

@ -0,0 +1,59 @@
#ifndef __AQ_USE_THREADEDGC__
#include <atomic>
class GC {
private:;
size_t max_size, max_slots,
interval, forced_clean,
forceclean_timer = 0;
bool running, alive;
// ptr, dealloc, ref, sz
void *q, *q_back;
void* handle;
std::atomic<uint32_t> slot_pos;
std::atomic<uint32_t> alive_cnt;
std::atomic<uint64_t> current_size;
volatile bool lock;
// maybe use volatile std::thread::id instead
protected:
void acquire_lock();
void release_lock();
void gc();
void daemon();
void start_deamon();
void terminate_daemon();
public:
void reg(void* v, uint32_t sz = 1,
void(*f)(void*) = free
);
GC(
uint32_t max_size = 0xfffffff, uint32_t max_slots = 4096,
uint32_t interval = 10000, uint32_t forced_clean = 1000000 //one seconds
) : max_size(max_size), max_slots(max_slots),
interval(interval), forced_clean(forced_clean){
start_deamon();
GC::gc = this;
} // 256 MB
~GC(){
terminate_daemon();
}
static GC* gc;
constexpr static void(*_free) (void*) = free;
};
#else
class GC {
public:
GC(uint32_t) = default;
void reg(
void* v, uint32_t = 0,
void(*f)(void*) = free
) const { f(v); }
static GC* gc;
constexpr static void(*_free) (void*) = free;
}
#endif

@ -0,0 +1,116 @@
// Copyright (c) 2022 James Edward Anhalt III - https://github.com/jeaiii/itoa
using u32 = decltype(0xffffffff);
using u64 = decltype(0xffffffffffffffff);
static_assert(u32(-1) > 0, "u32 must be unsigned");
static_assert(u32(0xffffffff) + u32(1) == u32(0), "u32 must be 32 bits");
static_assert(u64(-1) > 0, "u64 must be unsigned");
static_assert(u64(0xffffffffffffffff) + u32(1) == u32(0), "u64 must be 64 bits");
constexpr auto digits_00_99 =
"00010203040506070809" "10111213141516171819" "20212223242526272829" "30313233343536373839" "40414243444546474849"
"50515253545556575859" "60616263646566676869" "70717273747576777879" "80818283848586878889" "90919293949596979899";
struct pair { char t, o; };
#define JEAIII_W(I, U) *(pair*)&b[I] = *(pair*)&digits_00_99[(U) * 2]
#define JEAIII_A(I, N) t = (u64(1) << (32 + N / 5 * N * 53 / 16)) / u32(1e##N) + 1 + N / 6 - N / 8, t *= u, t >>= N / 5 * N * 53 / 16, t += N / 6 * 4, JEAIII_W(I, t >> 32)
#define JEAIII_S(I) b[I] = char(u64(10) * u32(t) >> 32) + '0'
#define JEAIII_D(I) t = u64(100) * u32(t), JEAIII_W(I, t >> 32)
#define JEAIII_C0(I) b[I] = char(u) + '0'
#define JEAIII_C1(I) JEAIII_W(I, u)
#define JEAIII_C2(I) JEAIII_A(I, 1), JEAIII_S(I + 2)
#define JEAIII_C3(I) JEAIII_A(I, 2), JEAIII_D(I + 2)
#define JEAIII_C4(I) JEAIII_A(I, 3), JEAIII_D(I + 2), JEAIII_S(I + 4)
#define JEAIII_C5(I) JEAIII_A(I, 4), JEAIII_D(I + 2), JEAIII_D(I + 4)
#define JEAIII_C6(I) JEAIII_A(I, 5), JEAIII_D(I + 2), JEAIII_D(I + 4), JEAIII_S(I + 6)
#define JEAIII_C7(I) JEAIII_A(I, 6), JEAIII_D(I + 2), JEAIII_D(I + 4), JEAIII_D(I + 6)
#define JEAIII_C8(I) JEAIII_A(I, 7), JEAIII_D(I + 2), JEAIII_D(I + 4), JEAIII_D(I + 6), JEAIII_S(I + 8)
#define JEAIII_C9(I) JEAIII_A(I, 8), JEAIII_D(I + 2), JEAIII_D(I + 4), JEAIII_D(I + 6), JEAIII_D(I + 8)
#define JEAIII_L(N, A, B) u < u32(1e##N) ? A : B
#define JEAIII_L09(F) JEAIII_L(2, JEAIII_L(1, F(0), F(1)), JEAIII_L(6, JEAIII_L(4, JEAIII_L(3, F(2), F(3)), JEAIII_L(5, F(4), F(5))), JEAIII_L(8, JEAIII_L(7, F(6), F(7)), JEAIII_L(9, F(8), F(9)))))
#define JEAIII_L03(F) JEAIII_L(2, JEAIII_L(1, F(0), F(1)), JEAIII_L(3, F(2), F(3)))
#define JEAIII_K(N) (JEAIII_C##N(0), b + N + 1)
#define JEAIII_KX(N) (JEAIII_C##N(0), u = x, JEAIII_C7(N + 1), b + N + 9)
#define JEAIII_KYX(N) (JEAIII_C##N(0), u = y, JEAIII_C7(N + 1), u = x, JEAIII_C7(N + 9), b + N + 17)
template<bool B, class T, class F> struct _cond { using type = F; };
template<class T, class F> struct _cond<true, T, F> { using type = T; };
template<bool B, class T, class F> using cond = typename _cond<B, T, F>::type;
template<class T> inline char* to_text_from_integer(char* b, T i)
{
u64 t = u64(i);
if (i < T(0))
t = u64(0) - t, b[0] = '-', ++b;
u32 u = cond<T(1) != T(2), cond<sizeof(T) != 1, cond<sizeof(T) != sizeof(short), u32, unsigned short>, unsigned char>, bool>(t);
// if our input type fits in 32bits, or its value does, ctreat as 32bit (the line above ensures the compiler can still know the range limits of the input type)
// and optimize out cases for small integer types (if only c++ had a builtin way to get the unsigned type from a signed type)
if (sizeof(i) <= sizeof(u) || u == t)
return JEAIII_L09(JEAIII_K);
u32 x = t % 100000000u;
u = u32(t /= 100000000u);
// t / 10^8 (fits in 32 bit), t % 10^8 -> ~17.5 digits
if (u == t)
return JEAIII_L09(JEAIII_KX);
// t / 10^16 (1-4 digits), t / 10^8 % 10^8, t % 10^8
u32 y = t % 100000000u;
u = u32(t / 100000000u);
return JEAIII_L03(JEAIII_KYX);
}
inline char* to_text(char text[], signed char i) { return to_text_from_integer(text, i); }
inline char* to_text(char text[], unsigned char i) { return to_text_from_integer(text, i); }
inline char* to_text(char text[], short i) { return to_text_from_integer(text, i); }
inline char* to_text(char text[], unsigned short i) { return to_text_from_integer(text, i); }
inline char* to_text(char text[], int i) { return to_text_from_integer(text, i); }
inline char* to_text(char text[], unsigned int i) { return to_text_from_integer(text, i); }
inline char* to_text(char text[], long i) { return to_text_from_integer(text, i); }
inline char* to_text(char text[], unsigned long i) { return to_text_from_integer(text, i); }
inline char* to_text(char text[], long long i) { return to_text_from_integer(text, i); }
inline char* to_text(char text[], unsigned long long i) { return to_text_from_integer(text, i); }
// Copyright (c) 2022 Bill Sun
constexpr static __uint128_t _10_19 = 10000000000000000000ull,
_10_37 = _10_19*_10_19 / 10;
template<class T>
char* jeaiii_i128(char* buf, T v){
if (v < 0){
*(buf++) = '0';
v = -v;
}
if (v > _10_37){
uint8_t vv = uint8_t(v/_10_37);
// vv <<= 1;
// if (vv < 20)
// *buf ++ = digits_00_99[vv + 1];
// else{
// *buf++ = digits_00_99[vv ];
// *buf++ = digits_00_99[vv + 1];
// }
*(buf++) = vv%10 + '0';
vv/=10;
if (vv) {
*buf = *(buf-1);
*(buf++-1) = vv + '0';
}
}
if (v > _10_19)
buf = to_text(buf, uint64_t((v/_10_19) % _10_19));
buf = to_text(buf, uint64_t(v % _10_19));
return buf;
}

@ -1,20 +1,20 @@
#include "pch_msc.hpp"
#include "io.h"
#include "table.h"
#include <limits>
#include <chrono>
#include <ctime>
#include "utils.h"
#include "libaquery.h"
#include <random>
char* gbuf = nullptr;
void setgbuf(char* buf) {
static char* b = 0;
if (buf == 0)
static char* b = nullptr;
if (buf == nullptr)
gbuf = b;
else {
gbuf = buf;
@ -63,6 +63,7 @@ T getInt(const char*& buf){
}
return ret;
}
template<class T>
char* intToString(T val, char* buf){
@ -275,6 +276,43 @@ inline const char* str(const bool& v) {
return v ? "true" : "false";
}
Context::Context() {
current.memory_map = new std::unordered_map<void*, deallocator_t>;
init_session();
}
Context::~Context() {
auto memmap = (std::unordered_map<void*, deallocator_t>*) this->current.memory_map;
delete memmap;
}
void Context::init_session(){
if (log_level == LOG_INFO){
memset(&(this->current.stats), 0, sizeof(Session::Statistic));
}
auto memmap = (std::unordered_map<void*, deallocator_t>*) this->current.memory_map;
memmap->clear();
}
void Context::end_session(){
auto memmap = (std::unordered_map<void*, deallocator_t>*) this->current.memory_map;
for (auto& mem : *memmap) {
mem.second(mem.first);
}
memmap->clear();
}
void* Context::get_module_function(const char* fname){
auto fmap = static_cast<std::unordered_map<std::string, void*>*>
(this->module_function_maps);
// printf("%p\n", fmap->find("mydiv")->second);
// for (const auto& [key, value] : *fmap){
// printf("%s %p\n", key.c_str(), value);
// }
auto ret = fmap->find(fname);
return ret == fmap->end() ? nullptr : ret->second;
}
// template<typename _Ty>
// inline void vector_type<_Ty>::out(uint32_t n, const char* sep) const
// {
@ -288,3 +326,123 @@ inline const char* str(const bool& v) {
// }
// std::cout << ')';
// }
#include "gc.h"
#include <vector_type>
#include <utility>
#include <thread>
#include <chrono>
#ifndef __AQ_USE_THREADEDGC__
struct gcmemory_t{
void* memory;
void (*deallocator)(void*);
};
using memoryqueue_t = gcmemory_t*;
void GC::acquire_lock() {
auto this_tid = std::this_thread::get_id();
while(lock != this_tid)
{
while(lock != this_tid && lock != std::thread::id()) {
std::this_thread::sleep_for(std::chrono::milliseconds(0));
}
lock = this_tid;
}
}
void GC::release_lock(){
lock = std::thread::id();
}
void GC::gc()
{
auto& _q = static_cast<memoryqueue_t*>(q);
auto& _q_back = static_cast<memoryqueue_t*>(q_back);
if (_q->size == 0)
return;
auto t = _q;
lock = true;
while(alive_cnt > 0);
_q = q_back;
uint32_t _slot = slot_pos;
slot_pos = 0;
current_size = 0;
lock = false;
_q_back = t;
for(uint32_t i = 0; i < _slot; ++i){
if (_q_back[i]->memory != nullptr && _q_back[i]->deallocator != nullptr)
_q_back[i]->deallocator(_q_back[i]->memory);
}
memset(_q_back, 0, sizeof(gcmemory_t) * _slot);
running = false;
}
void GC::daemon() {
using namespace std::chrono;
while (alive) {
if (running) {
if (current_size > max_size ||
forceclean_timer > forced_clean)
{
gc();
forceclean_timer = 0;
}
std::this_thread::sleep_for(microseconds(interval));
forceclean_timer += interval;
}
else {
std::this_thread::sleep_for(10ms);
forceclean_timer += 10000;
}
}
}
void GC::start_deamon() {
q = new gcmemory_t[max_slots << 1];
q_back = new memoryqueue_t[max_slots << 1];
lock = false;
slot_pos = 0;
current_size = 0;
alive_cnt = 0;
alive = true;
handle = new std::thread(&GC::daemon, this);
}
void GC::terminate_daemon() {
running = false;
alive = false;
decltype(auto) _handle = static_cast<std::thread*>(handle);
delete[] static_cast<memoryqueue_t>(q);
delete[] static_cast<memoryqueue_t>(q_back);
using namespace std::chrono;
std::this_thread::sleep_for(microseconds(1000 + std::max(static_cast<size_t>(10000), interval)));
if (_handle->joinable()) {
_handle->join();
}
delete _handle;
}
void GC::reg(void* v, uint32_t sz, void(*f)(void*)) { //~ 40ns expected v. free ~ 75ns
if (v == nullptr || f == nullptr)
return;
if (sz < threshould){
f(v);
return;
}
auto _q = static_cast<memoryqueue_t>q;
while(lock);
++alive_cnt;
current_size += sz;
auto _slot = (slot_pos += 1);
q[_slot] = {v, f};
--alive_cnt;
running = true;
}
#endif
static GC* GC::gc = nullptr;

@ -1,6 +1,12 @@
#ifndef _AQUERY_H
#define _AQUERY_H
#ifdef __INTELLISENSE__
#define __AQUERY_ITC_USE_SEMPH__
#define THREADING
#define __AQ_THREADED_GC__
#endif
#include "table.h"
#include <unordered_map>
#include <chrono>
@ -55,7 +61,10 @@ struct Context{
#ifdef THREADING
void* thread_pool;
#endif
printf_type print = printf;
#ifdef __AQ_THREADED_GC__
void* gc;
#endif
printf_type print = &printf;
Context();
virtual ~Context();
template <class ...Types>

@ -6,6 +6,7 @@
#include "monetdb_conn.h"
#include "monetdbe.h"
#include "table.h"
#undef static_assert
const char* monetdbe_type_str[] = {
@ -121,6 +122,8 @@ bool Server::haserror(){
return false;
}
}
void Server::print_results(const char* sep, const char* end){
if (!haserror()){
@ -138,6 +141,7 @@ void Server::print_results(const char* sep, const char* end){
std::string(types::printf_str[types::monetdbe_type_aqtypes[cols[i]->type]])
+ (i < ncols - 1 ? sep : "");
puts(printf_string[i].c_str());
puts(monetdbe_type_str[cols[i]->type]);
col_data[i] = static_cast<char *>(cols[i]->data);
szs [i] = monetdbe_type_szs[cols[i]->type];
header_string = header_string + cols[i]->name + sep + '|' + sep;
@ -179,7 +183,7 @@ void* Server::getCol(int col_idx){
auto _ret_col = static_cast<monetdbe_column*>(this->ret_col);
cnt = _ret_col->count;
printf("Dbg: Getting col %s, type: %s\n",
_ret_col->name, monetdbe_type_str[_ret_col->type]);
_ret_col->name, monetdbe_type_str[_ret_col->type]);
return _ret_col->data;
}
else{
@ -198,10 +202,10 @@ Server::~Server(){
bool Server::havehge() {
#if defined(_MONETDBE_LIB_) and defined(HAVE_HGE)
puts("true");
// puts("true");
return HAVE_HGE;
#else
puts("false");
// puts("false");
return false;
#endif
}

@ -24,6 +24,7 @@ struct Server{
static bool havehge();
void test(const char*);
void print_results(const char* sep = " ", const char* end = "\n");
friend void print_monetdb_results(Server* srv, const char* sep, const char* end, int limit);
~Server();
};

@ -1,6 +1,5 @@
#include "pch_msc.hpp"
#include "../csv.h"
#include <iostream>
#include <string>
#include <chrono>
@ -10,28 +9,35 @@
#ifdef THREADING
#include "threading.h"
#endif
#ifdef _WIN32
#include "winhelper.h"
#else
#include <dlfcn.h>
#include <fcntl.h>
#include <sys/mman.h>
// fast numeric to string conversion
#include "jeaiii_to_text.h"
#include "dragonbox/dragonbox_to_chars.h"
struct SharedMemory
{
std::atomic<bool> a;
int hFileMap;
void* pData;
SharedMemory(const char* fname) {
explicit SharedMemory(const char* fname) {
hFileMap = open(fname, O_RDWR, 0);
if (hFileMap != -1)
pData = mmap(NULL, 8, PROT_READ | PROT_WRITE, MAP_SHARED, hFileMap, 0);
pData = mmap(nullptr, 8, PROT_READ | PROT_WRITE, MAP_SHARED, hFileMap, 0);
else
pData = 0;
pData = nullptr;
}
void FreeMemoryMap() {
void FreeMemoryMap() const {
// automatically unmapped in posix
}
};
#ifndef __USE_STD_SEMAPHORE__
#ifdef __APPLE__
#include <dispatch/dispatch.h>
@ -74,9 +80,10 @@ public:
};
#endif
#endif
#endif
#ifdef __USE_STD_SEMAPHORE__
#define __AQUERY_ITC_USE_SEMPH__
#include <semaphore>
class A_Semaphore {
private:
@ -94,6 +101,7 @@ public:
~A_Semaphore() { }
};
#endif
#ifdef __AQUERY_ITC_USE_SEMPH__
A_Semaphore prompt{ true }, engine{ false };
#define PROMPT_ACQUIRE() prompt.acquire()
@ -107,11 +115,9 @@ A_Semaphore prompt{ true }, engine{ false };
#define ENGINE_RELEASE()
#endif
#include "aggregations.h"
typedef int (*code_snippet)(void*);
typedef void (*module_init_fn)(Context*);
int test_main();
int n_recv = 0;
char** n_recvd = nullptr;
@ -119,6 +125,7 @@ char** n_recvd = nullptr;
__AQEXPORT__(void) wait_engine(){
PROMPT_ACQUIRE();
}
__AQEXPORT__(void) wake_engine(){
ENGINE_RELEASE();
}
@ -152,42 +159,6 @@ __AQEXPORT__(bool) have_hge(){
#endif
}
Context::Context() {
current.memory_map = new std::unordered_map<void*, deallocator_t>;
init_session();
}
Context::~Context() {
auto memmap = (std::unordered_map<void*, deallocator_t>*) this->current.memory_map;
delete memmap;
}
void Context::init_session(){
if (log_level == LOG_INFO){
memset(&(this->current.stats), 0, sizeof(Session::Statistic));
}
auto memmap = (std::unordered_map<void*, deallocator_t>*) this->current.memory_map;
memmap->clear();
}
void Context::end_session(){
auto memmap = (std::unordered_map<void*, deallocator_t>*) this->current.memory_map;
for (auto& mem : *memmap) {
mem.second(mem.first);
}
memmap->clear();
}
void* Context::get_module_function(const char* fname){
auto fmap = static_cast<std::unordered_map<std::string, void*>*>
(this->module_function_maps);
// printf("%p\n", fmap->find("mydiv")->second);
// for (const auto& [key, value] : *fmap){
// printf("%s %p\n", key.c_str(), value);
// }
auto ret = fmap->find(fname);
return ret == fmap->end() ? nullptr : ret->second;
}
void initialize_module(const char* module_name, void* module_handle, Context* cxt){
auto _init_module = reinterpret_cast<module_init_fn>(dlsym(module_handle, "init_session"));
@ -253,7 +224,7 @@ int dll_main(int argc, char** argv, Context* cxt){
timer.reset();
server->exec(n_recvd[i] + 1);
cfg->stats.monet_time += timer.elapsed();
printf("Exec Q%d: %s", i, n_recvd[i]);
// printf("Exec Q%d: %s", i, n_recvd[i]);
}
break;
case 'P': // Postprocessing procedure
@ -313,7 +284,7 @@ int dll_main(int argc, char** argv, Context* cxt){
dlclose(handle);
handle = nullptr;
}
printf("%ld, %ld", cfg->stats.monet_time, cfg->stats.postproc_time);
printf("%lld, %lld", cfg->stats.monet_time, cfg->stats.postproc_time);
cxt->end_session();
n_recv = 0;
}
@ -370,20 +341,21 @@ extern "C" int __DLLEXPORT__ main(int argc, char** argv) {
#ifdef __AQ_BUILD_LAUNCHER__
return launcher(argc, argv);
#endif
puts("running");
// puts("running");
Context* cxt = new Context();
cxt->log("%d %s\n", argc, argv[1]);
// cxt->log("%d %s\n", argc, argv[1]);
#ifdef THREADING
auto tp = new ThreadPool();
cxt->thread_pool = tp;
#endif
#ifdef __AQ_THREADED_GC__
cxt->gc_thread = new std::thread(gc_thread, cxt);
#endif
const char* shmname;
if (argc < 0)
return dll_main(argc, argv, cxt);
else if (argc <= 1)
return test_main();
else
shmname = argv[1];
SharedMemory shm = SharedMemory(shmname);
@ -417,56 +389,3 @@ extern "C" int __DLLEXPORT__ main(int argc, char** argv) {
return 0;
}
#include "utils.h"
#include "table_ext_monetdb.hpp"
int test_main()
{
Context* cxt = new Context();
if (cxt->alt_server == 0)
cxt->alt_server = new Server(cxt);
Server* server = reinterpret_cast<Server*>(cxt->alt_server);
const char* qs[]= {
"QCREATE TABLE trade(stocksymbol INT, time INT, quantity INT, price INT);",
"QCOPY OFFSET 2 INTO trade FROM 'w:/gg/AQuery++/data/trade_numerical.csv' ON SERVER USING DELIMITERS ',';",
"QSELECT stocksymbol, (SUM((quantity * price)) / SUM(quantity)) AS weighted_average FROM trade GROUP BY stocksymbol ;",
"Pdll_5lYrMY",
"QSELECT stocksymbol, price FROM trade ORDER BY time ;",
"Pdll_4Sg6Ri",
"QSELECT stocksymbol, quantity, price FROM trade ORDER BY time ;",
"Pdll_5h4kL2",
"QSELECT stocksymbol, price FROM trade ORDER BY time ;",
"Pdll_7tEWCO",
"QSELECT query_c.weighted_moving_averages, query_c.stocksymbol FROM query_c;",
"Pdll_7FCPnF"
};
n_recv = sizeof(qs)/(sizeof (char*));
n_recvd = const_cast<char**>(qs);
void* handle = 0;
handle = dlopen("./dll.so", RTLD_LAZY);
cxt->init_session();
for (int i = 0; i < n_recv; ++i)
{
//printf("%s, %d\n", n_recvd[i], n_recvd[i][0] == 'Q');
switch (n_recvd[i][0]) {
case 'Q': // SQL query for monetdbe
{
server->exec(n_recvd[i] + 1);
printf("Exec Q%d: %s\n", i, n_recvd[i]);
}
break;
case 'P': // Postprocessing procedure
if (handle && !server->haserror()) {
code_snippet c = reinterpret_cast<code_snippet>(dlsym(handle, n_recvd[i] + 1));
c(cxt);
}
break;
}
}
n_recv = 0;
//static_assert(std::is_same_v<decltype(fill_integer_array<5, 1>()), std::integer_sequence<bool, 1,1,1,1,1>>, "");
return 0;
}

@ -1,5 +1,9 @@
#pragma once
#include <ctime>
#include <type_traits>
#include<string>
#if ((defined(_MSVC_LANG) && _MSVC_LANG >= 201703L) || __cplusplus >= 201703L)
constexpr static bool cpp_17 = true;
#else
@ -10,5 +14,19 @@ inline const char* str(const T& v) {
return "";
}
#include<string>
template <class T>
constexpr char* aq_itoa(T t, char* buf){
if constexpr (std::is_signed<T>::value){
if (t < 0){
*buf++ = '-';
t = -t;
}
}
while(t > 0){
*buf++ = t%10 + '0';
t /= 10;
}
return buf;
}
extern std::string base62uuid(int l = 6);

Loading…
Cancel
Save