From 95d94468a9a3dfe6bfa9130b09909b5c0a0bba7e Mon Sep 17 00:00:00 2001 From: Bill Date: Sat, 12 Nov 2022 01:06:48 +0800 Subject: [PATCH] super fast export(console/wip:file) --- Makefile | 4 +- README.md | 2 +- server/dragonbox/dragonbox_to_chars.h | 4 +- ...ox_to_chars.cpp => dragonbox_to_chars.hpp} | 2 + server/jeaiii_to_text.h | 19 ++-- server/libaquery.cpp | 78 +++++++++++++++-- server/libaquery.h | 36 +++++++- server/monetdb_conn.cpp | 8 +- server/server.cpp | 86 ++++++++++++++++++- 9 files changed, 215 insertions(+), 24 deletions(-) rename server/dragonbox/{dragonbox_to_chars.cpp => dragonbox_to_chars.hpp} (99%) diff --git a/Makefile b/Makefile index a3aa69d..cd0d8e5 100644 --- a/Makefile +++ b/Makefile @@ -15,8 +15,8 @@ FPIC = -fPIC COMPILER = $(shell $(CXX) --version | grep -q clang && echo clang|| echo gcc) LIBTOOL = ar rcs USELIB_FLAG = -Wl,--whole-archive,libaquery.a -Wl,-no-whole-archive -LIBAQ_SRC = server/monetdb_conn.cpp server/libaquery.cpp server/dragonbox/dragonbox_to_chars.cpp -LIBAQ_OBJ = monetdb_conn.o libaquery.o dragonbox_to_chars.o +LIBAQ_SRC = server/monetdb_conn.cpp server/libaquery.cpp +LIBAQ_OBJ = monetdb_conn.o libaquery.o SEMANTIC_INTERPOSITION = -fno-semantic-interposition RANLIB = ranlib diff --git a/README.md b/README.md index 2e6f0f2..a0ec936 100644 --- a/README.md +++ b/README.md @@ -306,5 +306,5 @@ DROP TABLE my_table IF EXISTS Author: James Edward Anhalt III
License (MIT): https://github.com/jeaiii/itoa/blob/main/LICENSE -- [MobetDB] (https://www.monetdb.org)
+- [MonetDB](https://www.monetdb.org)
License (Mozilla Public License): https://github.com/MonetDB/MonetDB/blob/master/license.txt diff --git a/server/dragonbox/dragonbox_to_chars.h b/server/dragonbox/dragonbox_to_chars.h index e22a2b4..ca5384f 100644 --- a/server/dragonbox/dragonbox_to_chars.h +++ b/server/dragonbox/dragonbox_to_chars.h @@ -47,8 +47,8 @@ namespace jkj::dragonbox { result.exponent, buffer); } else { - std::memcpy(buffer, "0E0", 3); - return buffer + 3; + *buffer = '0'; + return buffer + 1; } } else { diff --git a/server/dragonbox/dragonbox_to_chars.cpp b/server/dragonbox/dragonbox_to_chars.hpp similarity index 99% rename from server/dragonbox/dragonbox_to_chars.cpp rename to server/dragonbox/dragonbox_to_chars.hpp index 75b82b6..7199d74 100644 --- a/server/dragonbox/dragonbox_to_chars.cpp +++ b/server/dragonbox/dragonbox_to_chars.hpp @@ -15,6 +15,7 @@ // is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY // KIND, either express or implied. +#pragma once #include "dragonbox_to_chars.h" @@ -517,3 +518,4 @@ namespace jkj::dragonbox { } } } + diff --git a/server/jeaiii_to_text.h b/server/jeaiii_to_text.h index a4f1a53..63fd9ee 100644 --- a/server/jeaiii_to_text.h +++ b/server/jeaiii_to_text.h @@ -1,4 +1,4 @@ - +#pragma once // Copyright (c) 2022 James Edward Anhalt III - https://github.com/jeaiii/itoa using u32 = decltype(0xffffffff); using u64 = decltype(0xffffffffffffffff); @@ -81,14 +81,18 @@ inline char* to_text(char text[], long long i) { return to_text_from_integer(tex inline char* to_text(char text[], unsigned long long i) { return to_text_from_integer(text, i); } // Copyright (c) 2022 Bill Sun + +//#if defined(SIZEOF___INT128) || (defined(SIZEOF___INT128_T) && defined(SIZEOF___UINT128_T)) constexpr static __uint128_t _10_19 = 10000000000000000000ull, _10_37 = _10_19*_10_19 / 10; template char* jeaiii_i128(char* buf, T v){ - if (v < 0){ - *(buf++) = '0'; - v = -v; + if constexpr (std::is_signed_v) { + if (v < 0){ + *(buf++) = '0'; + v = -v; + } } if (v > _10_37){ uint8_t vv = uint8_t(v/_10_37); @@ -96,8 +100,8 @@ char* jeaiii_i128(char* buf, T v){ // if (vv < 20) // *buf ++ = digits_00_99[vv + 1]; // else{ - // *buf++ = digits_00_99[vv ]; - // *buf++ = digits_00_99[vv + 1]; + // memcpy(buf, digits_00_99 + vv, 2); + // buf += 2; // } *(buf++) = vv%10 + '0'; @@ -113,4 +117,5 @@ char* jeaiii_i128(char* buf, T v){ buf = to_text(buf, uint64_t(v % _10_19)); return buf; -} \ No newline at end of file +} +// #endif diff --git a/server/libaquery.cpp b/server/libaquery.cpp index af8fedc..320852a 100644 --- a/server/libaquery.cpp +++ b/server/libaquery.cpp @@ -446,10 +446,76 @@ void GC::reg(void* v, uint32_t sz, void(*f)(void*)) { //~ 40ns expected v. free GC* GC::gc_handle = nullptr; -#include "dragonbox/dragonbox_to_chars.h" -void test(){ - char buf[32]; - double d = 123456789.123456789; - auto ret = jkj::dragonbox::to_chars(d, buf); - printf("%s\n", buf); +#include "dragonbox/dragonbox_to_chars.hpp" + + +template<> +char* +aq_to_chars(void* value, char* buffer) { + return jkj::dragonbox::to_chars_n(*static_cast(value), buffer); +} +template<> +char* +aq_to_chars(void* value, char* buffer) { + return jkj::dragonbox::to_chars_n(*static_cast(value), buffer); +} + +template<> +inline char* +aq_to_chars(void* value, char* buffer) { + if (*static_cast(value)){ + memcpy(buffer, "true", 4); + return buffer + 4; + } + else{ + memcpy(buffer, "false", 5); + return buffer + 5; + } +} + +template<> +inline char* +aq_to_chars(void* value, char* buffer) { + const auto src = *static_cast(value); + const auto len = strlen(src); + memcpy(buffer, src, len); + return buffer + len; } + +template<> +inline char* +aq_to_chars(void* value, char* buffer) { + const auto& src = *static_cast(value); + buffer = to_text(buffer, src.year); + *buffer++ = '-'; + buffer = to_text(buffer, src.month); + *buffer++ = '-'; + buffer = to_text(buffer, src.day); + return buffer; +} + +template<> +inline char* +aq_to_chars(void* value, char* buffer) { + const auto& src = *static_cast(value); + buffer = to_text(buffer, src.hours); + *buffer++ = ':'; + buffer = to_text(buffer, src.minutes); + *buffer++ = ':'; + buffer = to_text(buffer, src.seconds); + *buffer++ = ':'; + buffer = to_text(buffer, src.ms); + return buffer; +} + +template<> +inline char* +aq_to_chars(void* value, char* buffer) { + auto& src = *static_cast(value); + buffer = aq_to_chars(static_cast(&src.date), buffer); + *buffer++ = ' '; + buffer = aq_to_chars(static_cast(&src.time), buffer); + return buffer; +} + + diff --git a/server/libaquery.h b/server/libaquery.h index e737396..cc0b5a9 100644 --- a/server/libaquery.h +++ b/server/libaquery.h @@ -117,6 +117,40 @@ struct Context{ #define __AQEXPORT__(_Ty) extern "C" _Ty __DLLEXPORT__ typedef void (*deallocator_t) (void*); + +#include +#include "jeaiii_to_text.h" + +template +inline std::enable_if_t, char *> +aq_to_chars(void* value, char* buffer) { + return to_text(buffer, *static_cast(value)); +} + +template +inline std::enable_if_t, char *> +aq_to_chars(void* value, char* buffer) { + return buffer; +} + +#ifdef __SIZEOF_INT128__ +template<> +inline char* +aq_to_chars<__int128_t>(void* value, char* buffer) { + return jeaiii_i128<__int128_t>(buffer, *static_cast<__int128_t*>(value)); +} + +template<> +inline char* +aq_to_chars<__uint128_t>(void* value, char* buffer) { + return jeaiii_i128<__uint128_t>(buffer, *static_cast<__uint128_t*>(value)); +} #endif -void test(); \ No newline at end of file +template<> char* aq_to_chars(void* , char*); +template<> char* aq_to_chars(void* , char*); +template<> char* aq_to_chars(void* , char*); +template<> char* aq_to_chars(void* , char*); +template<> char* aq_to_chars(void* , char*); +template<> char* aq_to_chars(void* , char*); +#endif diff --git a/server/monetdb_conn.cpp b/server/monetdb_conn.cpp index 644c1c4..b3fbd1e 100644 --- a/server/monetdb_conn.cpp +++ b/server/monetdb_conn.cpp @@ -9,7 +9,7 @@ #undef static_assert -const char* monetdbe_type_str[] = { +constexpr const char* monetdbe_type_str[] = { "monetdbe_bool", "monetdbe_int8_t", "monetdbe_int16_t", "monetdbe_int32_t", "monetdbe_int64_t", #ifdef HAVE_HGE "monetdbe_int128_t", @@ -22,7 +22,7 @@ const char* monetdbe_type_str[] = { "monetdbe_type_unknown" } ; -const unsigned char monetdbe_type_szs[] = { +inline constexpr static unsigned char monetdbe_type_szs[] = { sizeof(monetdbe_column_bool::null_value), sizeof(monetdbe_column_int8_t::null_value), sizeof(monetdbe_column_int16_t::null_value), sizeof(monetdbe_column_int32_t::null_value), sizeof(monetdbe_column_int64_t::null_value), @@ -37,8 +37,9 @@ const unsigned char monetdbe_type_szs[] = { // should be last: 1 }; + namespace types{ - const Type_t monetdbe_type_aqtypes[] = { + constexpr const Type_t monetdbe_type_aqtypes[] = { ABOOL, AINT8, AINT16, AINT32, AINT64, #ifdef HAVE_HGE AINT128, @@ -50,6 +51,7 @@ namespace types{ }; } + Server::Server(Context* cxt){ if (cxt){ connect(cxt); diff --git a/server/server.cpp b/server/server.cpp index eaea30e..446baf9 100644 --- a/server/server.cpp +++ b/server/server.cpp @@ -161,6 +161,89 @@ __AQEXPORT__(bool) have_hge(){ #endif } +using prt_fn_t = char* (*)(void*, char*); + + +constexpr prt_fn_t monetdbe_prtfns[] = { + aq_to_chars, aq_to_chars, aq_to_chars, aq_to_chars, + aq_to_chars, +#if __SIZEOF_INT128__ + aq_to_chars<__int128_t>, +#endif + aq_to_chars, aq_to_chars, aq_to_chars, + aq_to_chars, aq_to_chars, + aq_to_chars, aq_to_chars, aq_to_chars, + + // should be last: + aq_to_chars +}; + +#include "monetdbe.h" +inline constexpr static unsigned char monetdbe_type_szs[] = { + sizeof(monetdbe_column_bool::null_value), sizeof(monetdbe_column_int8_t::null_value), + sizeof(monetdbe_column_int16_t::null_value), sizeof(monetdbe_column_int32_t::null_value), + sizeof(monetdbe_column_int64_t::null_value), +#ifdef __SIZEOF_INT128__ + sizeof(monetdbe_column_int128_t::null_value), +#endif + sizeof(monetdbe_column_size_t::null_value), sizeof(monetdbe_column_float::null_value), + sizeof(monetdbe_column_double::null_value), + sizeof(monetdbe_column_str::null_value), sizeof(monetdbe_column_blob::null_value), + sizeof(monetdbe_data_date), sizeof(monetdbe_data_time), sizeof(monetdbe_data_timestamp), + + // should be last: + 1 +}; +constexpr uint32_t output_buffer_size = 65536; +void print_monetdb_results(Server* srv, const char* sep = " ", const char* end = "\n", + uint32_t limit = std::numeric_limits::max()) { + if (!srv->haserror() && limit){ + char buffer[output_buffer_size]; + auto _res = static_cast (srv->res); + const auto& ncols = _res->ncols; + monetdbe_column** cols = static_cast(malloc(sizeof(monetdbe_column*) * ncols)); + prt_fn_t *prtfns = (prt_fn_t*) alloca(sizeof(prt_fn_t) * ncols); + char** col_data = static_cast (alloca(sizeof(char*) * ncols)); + uint8_t* szs = static_cast(alloca(ncols)); + std::string header_string = ""; + const char* err_msg = nullptr; + for(uint32_t i = 0; i < ncols; ++i){ + err_msg = monetdbe_result_fetch(_res, &cols[i], i); + col_data[i] = static_cast(cols[i]->data); + prtfns[i] = monetdbe_prtfns[cols[i]->type]; + szs [i] = monetdbe_type_szs[cols[i]->type]; + header_string = header_string + cols[i]->name + sep + '|' + sep; + } + const size_t l_sep = strlen(sep); + const size_t l_end = strlen(end); + if (header_string.size() - l_sep - 1>= 0) + header_string.resize(header_string.size() - l_sep - 1); + header_string += end + std::string(header_string.size(), '=') + end; + fputs(header_string.c_str(), stdout); + char* _buffer = buffer; + for(uint64_t i = 0; i < srv->cnt; ++i){ + for(uint32_t j = 0; j < ncols; ++j){ + //copy the field to buf + _buffer = prtfns[j](col_data[j], _buffer); + if (j != ncols - 1){ + memcpy(_buffer, sep, l_sep); + _buffer += l_sep; + } + col_data[j] += szs[j]; + } + memcpy(_buffer, end, l_end); + _buffer += l_end; + if(output_buffer_size - (_buffer - buffer) <= 1024){ + fwrite(buffer, 1, _buffer - buffer, stdout); + _buffer = buffer; + } + } + if (_buffer != buffer) + fwrite(buffer, 1, _buffer - buffer, stdout); + free(cols); + } +} + void initialize_module(const char* module_name, void* module_handle, Context* cxt){ auto _init_module = reinterpret_cast(dlsym(module_handle, "init_session")); @@ -173,7 +256,6 @@ void initialize_module(const char* module_name, void* module_handle, Context* cx } int dll_main(int argc, char** argv, Context* cxt){ - test(); aq_timer timer; Config *cfg = reinterpret_cast(argv[0]); std::unordered_map user_module_map; @@ -266,7 +348,7 @@ int dll_main(int argc, char** argv, Context* cxt){ { if(!server->haserror()){ timer.reset(); - server->print_results(); + print_monetdb_results(server); cfg->stats.postproc_time += timer.elapsed(); } }