super fast export(console/wip:file)

dev
Bill 2 years ago
parent c1b1b26d1a
commit 95d94468a9

@ -15,8 +15,8 @@ FPIC = -fPIC
COMPILER = $(shell $(CXX) --version | grep -q clang && echo clang|| echo gcc) COMPILER = $(shell $(CXX) --version | grep -q clang && echo clang|| echo gcc)
LIBTOOL = ar rcs LIBTOOL = ar rcs
USELIB_FLAG = -Wl,--whole-archive,libaquery.a -Wl,-no-whole-archive USELIB_FLAG = -Wl,--whole-archive,libaquery.a -Wl,-no-whole-archive
LIBAQ_SRC = server/monetdb_conn.cpp server/libaquery.cpp server/dragonbox/dragonbox_to_chars.cpp LIBAQ_SRC = server/monetdb_conn.cpp server/libaquery.cpp
LIBAQ_OBJ = monetdb_conn.o libaquery.o dragonbox_to_chars.o LIBAQ_OBJ = monetdb_conn.o libaquery.o
SEMANTIC_INTERPOSITION = -fno-semantic-interposition SEMANTIC_INTERPOSITION = -fno-semantic-interposition
RANLIB = ranlib RANLIB = ranlib

@ -306,5 +306,5 @@ DROP TABLE my_table IF EXISTS
Author: James Edward Anhalt III <br> Author: James Edward Anhalt III <br>
License (MIT): https://github.com/jeaiii/itoa/blob/main/LICENSE License (MIT): https://github.com/jeaiii/itoa/blob/main/LICENSE
- [MobetDB] (https://www.monetdb.org) <br> - [MonetDB](https://www.monetdb.org) <br>
License (Mozilla Public License): https://github.com/MonetDB/MonetDB/blob/master/license.txt License (Mozilla Public License): https://github.com/MonetDB/MonetDB/blob/master/license.txt

@ -47,8 +47,8 @@ namespace jkj::dragonbox {
result.exponent, buffer); result.exponent, buffer);
} }
else { else {
std::memcpy(buffer, "0E0", 3); *buffer = '0';
return buffer + 3; return buffer + 1;
} }
} }
else { else {

@ -15,6 +15,7 @@
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY // is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. // KIND, either express or implied.
#pragma once
#include "dragonbox_to_chars.h" #include "dragonbox_to_chars.h"
@ -517,3 +518,4 @@ namespace jkj::dragonbox {
} }
} }
} }

@ -1,4 +1,4 @@
#pragma once
// Copyright (c) 2022 James Edward Anhalt III - https://github.com/jeaiii/itoa // Copyright (c) 2022 James Edward Anhalt III - https://github.com/jeaiii/itoa
using u32 = decltype(0xffffffff); using u32 = decltype(0xffffffff);
using u64 = decltype(0xffffffffffffffff); using u64 = decltype(0xffffffffffffffff);
@ -81,14 +81,18 @@ inline char* to_text(char text[], long long i) { return to_text_from_integer(tex
inline char* to_text(char text[], unsigned long long i) { return to_text_from_integer(text, i); } inline char* to_text(char text[], unsigned long long i) { return to_text_from_integer(text, i); }
// Copyright (c) 2022 Bill Sun // Copyright (c) 2022 Bill Sun
//#if defined(SIZEOF___INT128) || (defined(SIZEOF___INT128_T) && defined(SIZEOF___UINT128_T))
constexpr static __uint128_t _10_19 = 10000000000000000000ull, constexpr static __uint128_t _10_19 = 10000000000000000000ull,
_10_37 = _10_19*_10_19 / 10; _10_37 = _10_19*_10_19 / 10;
template<class T> template<class T>
char* jeaiii_i128(char* buf, T v){ char* jeaiii_i128(char* buf, T v){
if (v < 0){ if constexpr (std::is_signed_v<T>) {
*(buf++) = '0'; if (v < 0){
v = -v; *(buf++) = '0';
v = -v;
}
} }
if (v > _10_37){ if (v > _10_37){
uint8_t vv = uint8_t(v/_10_37); uint8_t vv = uint8_t(v/_10_37);
@ -96,8 +100,8 @@ char* jeaiii_i128(char* buf, T v){
// if (vv < 20) // if (vv < 20)
// *buf ++ = digits_00_99[vv + 1]; // *buf ++ = digits_00_99[vv + 1];
// else{ // else{
// *buf++ = digits_00_99[vv ]; // memcpy(buf, digits_00_99 + vv, 2);
// *buf++ = digits_00_99[vv + 1]; // buf += 2;
// } // }
*(buf++) = vv%10 + '0'; *(buf++) = vv%10 + '0';
@ -113,4 +117,5 @@ char* jeaiii_i128(char* buf, T v){
buf = to_text(buf, uint64_t(v % _10_19)); buf = to_text(buf, uint64_t(v % _10_19));
return buf; return buf;
} }
// #endif

@ -446,10 +446,76 @@ void GC::reg(void* v, uint32_t sz, void(*f)(void*)) { //~ 40ns expected v. free
GC* GC::gc_handle = nullptr; GC* GC::gc_handle = nullptr;
#include "dragonbox/dragonbox_to_chars.h" #include "dragonbox/dragonbox_to_chars.hpp"
void test(){
char buf[32];
double d = 123456789.123456789; template<>
auto ret = jkj::dragonbox::to_chars(d, buf); char*
printf("%s\n", buf); aq_to_chars<float>(void* value, char* buffer) {
return jkj::dragonbox::to_chars_n(*static_cast<float*>(value), buffer);
}
template<>
char*
aq_to_chars<double>(void* value, char* buffer) {
return jkj::dragonbox::to_chars_n(*static_cast<double*>(value), buffer);
}
template<>
inline char*
aq_to_chars<bool>(void* value, char* buffer) {
if (*static_cast<bool*>(value)){
memcpy(buffer, "true", 4);
return buffer + 4;
}
else{
memcpy(buffer, "false", 5);
return buffer + 5;
}
}
template<>
inline char*
aq_to_chars<char*>(void* value, char* buffer) {
const auto src = *static_cast<char**>(value);
const auto len = strlen(src);
memcpy(buffer, src, len);
return buffer + len;
} }
template<>
inline char*
aq_to_chars<types::date_t>(void* value, char* buffer) {
const auto& src = *static_cast<types::date_t*>(value);
buffer = to_text(buffer, src.year);
*buffer++ = '-';
buffer = to_text(buffer, src.month);
*buffer++ = '-';
buffer = to_text(buffer, src.day);
return buffer;
}
template<>
inline char*
aq_to_chars<types::time_t>(void* value, char* buffer) {
const auto& src = *static_cast<types::time_t*>(value);
buffer = to_text(buffer, src.hours);
*buffer++ = ':';
buffer = to_text(buffer, src.minutes);
*buffer++ = ':';
buffer = to_text(buffer, src.seconds);
*buffer++ = ':';
buffer = to_text(buffer, src.ms);
return buffer;
}
template<>
inline char*
aq_to_chars<types::timestamp_t>(void* value, char* buffer) {
auto& src = *static_cast<types::timestamp_t*>(value);
buffer = aq_to_chars<types::date_t>(static_cast<void*>(&src.date), buffer);
*buffer++ = ' ';
buffer = aq_to_chars<types::date_t>(static_cast<void*>(&src.time), buffer);
return buffer;
}

@ -117,6 +117,40 @@ struct Context{
#define __AQEXPORT__(_Ty) extern "C" _Ty __DLLEXPORT__ #define __AQEXPORT__(_Ty) extern "C" _Ty __DLLEXPORT__
typedef void (*deallocator_t) (void*); typedef void (*deallocator_t) (void*);
#include <type_traits>
#include "jeaiii_to_text.h"
template<class T>
inline std::enable_if_t<std::is_integral_v<T>, char *>
aq_to_chars(void* value, char* buffer) {
return to_text(buffer, *static_cast<T*>(value));
}
template<class T>
inline std::enable_if_t<!std::is_integral_v<T>, char *>
aq_to_chars(void* value, char* buffer) {
return buffer;
}
#ifdef __SIZEOF_INT128__
template<>
inline char*
aq_to_chars<__int128_t>(void* value, char* buffer) {
return jeaiii_i128<__int128_t>(buffer, *static_cast<__int128_t*>(value));
}
template<>
inline char*
aq_to_chars<__uint128_t>(void* value, char* buffer) {
return jeaiii_i128<__uint128_t>(buffer, *static_cast<__uint128_t*>(value));
}
#endif #endif
void test(); template<> char* aq_to_chars<float>(void* , char*);
template<> char* aq_to_chars<double>(void* , char*);
template<> char* aq_to_chars<char*>(void* , char*);
template<> char* aq_to_chars<types::date_t>(void* , char*);
template<> char* aq_to_chars<types::time_t>(void* , char*);
template<> char* aq_to_chars<types::timestamp_t>(void* , char*);
#endif

@ -9,7 +9,7 @@
#undef static_assert #undef static_assert
const char* monetdbe_type_str[] = { constexpr const char* monetdbe_type_str[] = {
"monetdbe_bool", "monetdbe_int8_t", "monetdbe_int16_t", "monetdbe_int32_t", "monetdbe_int64_t", "monetdbe_bool", "monetdbe_int8_t", "monetdbe_int16_t", "monetdbe_int32_t", "monetdbe_int64_t",
#ifdef HAVE_HGE #ifdef HAVE_HGE
"monetdbe_int128_t", "monetdbe_int128_t",
@ -22,7 +22,7 @@ const char* monetdbe_type_str[] = {
"monetdbe_type_unknown" "monetdbe_type_unknown"
} ; } ;
const unsigned char monetdbe_type_szs[] = { inline constexpr static unsigned char monetdbe_type_szs[] = {
sizeof(monetdbe_column_bool::null_value), sizeof(monetdbe_column_int8_t::null_value), sizeof(monetdbe_column_bool::null_value), sizeof(monetdbe_column_int8_t::null_value),
sizeof(monetdbe_column_int16_t::null_value), sizeof(monetdbe_column_int32_t::null_value), sizeof(monetdbe_column_int16_t::null_value), sizeof(monetdbe_column_int32_t::null_value),
sizeof(monetdbe_column_int64_t::null_value), sizeof(monetdbe_column_int64_t::null_value),
@ -37,8 +37,9 @@ const unsigned char monetdbe_type_szs[] = {
// should be last: // should be last:
1 1
}; };
namespace types{ namespace types{
const Type_t monetdbe_type_aqtypes[] = { constexpr const Type_t monetdbe_type_aqtypes[] = {
ABOOL, AINT8, AINT16, AINT32, AINT64, ABOOL, AINT8, AINT16, AINT32, AINT64,
#ifdef HAVE_HGE #ifdef HAVE_HGE
AINT128, AINT128,
@ -50,6 +51,7 @@ namespace types{
}; };
} }
Server::Server(Context* cxt){ Server::Server(Context* cxt){
if (cxt){ if (cxt){
connect(cxt); connect(cxt);

@ -161,6 +161,89 @@ __AQEXPORT__(bool) have_hge(){
#endif #endif
} }
using prt_fn_t = char* (*)(void*, char*);
constexpr prt_fn_t monetdbe_prtfns[] = {
aq_to_chars<bool>, aq_to_chars<int8_t>, aq_to_chars<int16_t>, aq_to_chars<int32_t>,
aq_to_chars<int64_t>,
#if __SIZEOF_INT128__
aq_to_chars<__int128_t>,
#endif
aq_to_chars<size_t>, aq_to_chars<float>, aq_to_chars<double>,
aq_to_chars<char*>, aq_to_chars<nullptr_t>,
aq_to_chars<types::date_t>, aq_to_chars<types::time_t>, aq_to_chars<types::timestamp_t>,
// should be last:
aq_to_chars<nullptr_t>
};
#include "monetdbe.h"
inline constexpr static unsigned char monetdbe_type_szs[] = {
sizeof(monetdbe_column_bool::null_value), sizeof(monetdbe_column_int8_t::null_value),
sizeof(monetdbe_column_int16_t::null_value), sizeof(monetdbe_column_int32_t::null_value),
sizeof(monetdbe_column_int64_t::null_value),
#ifdef __SIZEOF_INT128__
sizeof(monetdbe_column_int128_t::null_value),
#endif
sizeof(monetdbe_column_size_t::null_value), sizeof(monetdbe_column_float::null_value),
sizeof(monetdbe_column_double::null_value),
sizeof(monetdbe_column_str::null_value), sizeof(monetdbe_column_blob::null_value),
sizeof(monetdbe_data_date), sizeof(monetdbe_data_time), sizeof(monetdbe_data_timestamp),
// should be last:
1
};
constexpr uint32_t output_buffer_size = 65536;
void print_monetdb_results(Server* srv, const char* sep = " ", const char* end = "\n",
uint32_t limit = std::numeric_limits<uint32_t>::max()) {
if (!srv->haserror() && limit){
char buffer[output_buffer_size];
auto _res = static_cast<monetdbe_result*> (srv->res);
const auto& ncols = _res->ncols;
monetdbe_column** cols = static_cast<monetdbe_column**>(malloc(sizeof(monetdbe_column*) * ncols));
prt_fn_t *prtfns = (prt_fn_t*) alloca(sizeof(prt_fn_t) * ncols);
char** col_data = static_cast<char**> (alloca(sizeof(char*) * ncols));
uint8_t* szs = static_cast<uint8_t*>(alloca(ncols));
std::string header_string = "";
const char* err_msg = nullptr;
for(uint32_t i = 0; i < ncols; ++i){
err_msg = monetdbe_result_fetch(_res, &cols[i], i);
col_data[i] = static_cast<char *>(cols[i]->data);
prtfns[i] = monetdbe_prtfns[cols[i]->type];
szs [i] = monetdbe_type_szs[cols[i]->type];
header_string = header_string + cols[i]->name + sep + '|' + sep;
}
const size_t l_sep = strlen(sep);
const size_t l_end = strlen(end);
if (header_string.size() - l_sep - 1>= 0)
header_string.resize(header_string.size() - l_sep - 1);
header_string += end + std::string(header_string.size(), '=') + end;
fputs(header_string.c_str(), stdout);
char* _buffer = buffer;
for(uint64_t i = 0; i < srv->cnt; ++i){
for(uint32_t j = 0; j < ncols; ++j){
//copy the field to buf
_buffer = prtfns[j](col_data[j], _buffer);
if (j != ncols - 1){
memcpy(_buffer, sep, l_sep);
_buffer += l_sep;
}
col_data[j] += szs[j];
}
memcpy(_buffer, end, l_end);
_buffer += l_end;
if(output_buffer_size - (_buffer - buffer) <= 1024){
fwrite(buffer, 1, _buffer - buffer, stdout);
_buffer = buffer;
}
}
if (_buffer != buffer)
fwrite(buffer, 1, _buffer - buffer, stdout);
free(cols);
}
}
void initialize_module(const char* module_name, void* module_handle, Context* cxt){ void initialize_module(const char* module_name, void* module_handle, Context* cxt){
auto _init_module = reinterpret_cast<module_init_fn>(dlsym(module_handle, "init_session")); auto _init_module = reinterpret_cast<module_init_fn>(dlsym(module_handle, "init_session"));
@ -173,7 +256,6 @@ void initialize_module(const char* module_name, void* module_handle, Context* cx
} }
int dll_main(int argc, char** argv, Context* cxt){ int dll_main(int argc, char** argv, Context* cxt){
test();
aq_timer timer; aq_timer timer;
Config *cfg = reinterpret_cast<Config *>(argv[0]); Config *cfg = reinterpret_cast<Config *>(argv[0]);
std::unordered_map<std::string, void*> user_module_map; std::unordered_map<std::string, void*> user_module_map;
@ -266,7 +348,7 @@ int dll_main(int argc, char** argv, Context* cxt){
{ {
if(!server->haserror()){ if(!server->haserror()){
timer.reset(); timer.reset();
server->print_results(); print_monetdb_results(server);
cfg->stats.postproc_time += timer.elapsed(); cfg->stats.postproc_time += timer.elapsed();
} }
} }

Loading…
Cancel
Save