bill/AQuery
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

super fast export(console/wip:file)

Browse Source
This commit is contained in:
bill
2022-11-12 01:06:48 +08:00
parent c1b1b26d1a
commit 95d94468a9
9 changed files with 215 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Makefile
+2 -2
View File
@@ -15,8 +15,8 @@ FPIC = -fPIC
COMPILER = $(shell $(CXX) --version | grep -q clang && echo clang|| echo gcc)
LIBTOOL = ar rcs
USELIB_FLAG = -Wl,--whole-archive,libaquery.a -Wl,-no-whole-archive
LIBAQ_SRC = server/monetdb_conn.cpp server/libaquery.cpp server/dragonbox/dragonbox_to_chars.cpp
LIBAQ_OBJ = monetdb_conn.o libaquery.o dragonbox_to_chars.o
LIBAQ_SRC = server/monetdb_conn.cpp server/libaquery.cpp
LIBAQ_OBJ = monetdb_conn.o libaquery.o
SEMANTIC_INTERPOSITION = -fno-semantic-interposition
RANLIB = ranlib
README.md
+1 -1
View File
@@ -306,5 +306,5 @@ DROP TABLE my_table IF EXISTS
Author: James Edward Anhalt III <br>
License (MIT): https://github.com/jeaiii/itoa/blob/main/LICENSE
- [MobetDB] (https://www.monetdb.org) <br>
- [MonetDB](https://www.monetdb.org) <br>
License (Mozilla Public License): https://github.com/MonetDB/MonetDB/blob/master/license.txt
server/dragonbox/dragonbox_to_chars.h
+2 -2
View File
@@ -47,8 +47,8 @@ namespace jkj::dragonbox {
result.exponent, buffer);
}
else {
std::memcpy(buffer, "0E0", 3);
return buffer + 3;
*buffer = '0';
return buffer + 1;
}
}
else {
server/dragonbox/dragonbox_to_chars.cpp → server/dragonbox/dragonbox_to_chars.hpp
+2
View File
@@ -15,6 +15,7 @@
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.
#pragma once
#include "dragonbox_to_chars.h"
@@ -517,3 +518,4 @@ namespace jkj::dragonbox {
}
}
}
server/jeaiii_to_text.h
+11 -6
View File
@@ -1,4 +1,4 @@
#pragma once
// Copyright (c) 2022 James Edward Anhalt III - https://github.com/jeaiii/itoa
using u32 = decltype(0xffffffff);
using u64 = decltype(0xffffffffffffffff);
@@ -81,14 +81,18 @@ inline char* to_text(char text[], long long i) { return to_text_from_integer(tex
inline char* to_text(char text[], unsigned long long i) { return to_text_from_integer(text, i); }
// Copyright (c) 2022 Bill Sun
//#if defined(SIZEOF___INT128) || (defined(SIZEOF___INT128_T) && defined(SIZEOF___UINT128_T))
constexpr static __uint128_t _10_19 = 10000000000000000000ull,
_10_37 = _10_19*_10_19 / 10;
template<class T>
char* jeaiii_i128(char* buf, T v){
if (v < 0){
*(buf++) = '0';
v = -v;
if constexpr (std::is_signed_v<T>) {
if (v < 0){
*(buf++) = '0';
v = -v;
}
}
if (v > _10_37){
uint8_t vv = uint8_t(v/_10_37);
@@ -96,8 +100,8 @@ char* jeaiii_i128(char* buf, T v){
// if (vv < 20)
// *buf ++ = digits_00_99[vv + 1];
// else{
// *buf++ = digits_00_99[vv ];
// *buf++ = digits_00_99[vv + 1];
// memcpy(buf, digits_00_99 + vv, 2);
// buf += 2;
// }
*(buf++) = vv%10 + '0';
@@ -114,3 +118,4 @@ char* jeaiii_i128(char* buf, T v){
buf = to_text(buf, uint64_t(v % _10_19));
return buf;
}
// #endif
server/libaquery.cpp
+72 -6
View File
@@ -446,10 +446,76 @@ void GC::reg(void* v, uint32_t sz, void(*f)(void*)) { //~ 40ns expected v. free
GC* GC::gc_handle = nullptr;
#include "dragonbox/dragonbox_to_chars.h"
void test(){
char buf[32];
double d = 123456789.123456789;
auto ret = jkj::dragonbox::to_chars(d, buf);
printf("%s\n", buf);
#include "dragonbox/dragonbox_to_chars.hpp"
template<>
char*
aq_to_chars<float>(void* value, char* buffer) {
return jkj::dragonbox::to_chars_n(*static_cast<float*>(value), buffer);
}
template<>
char*
aq_to_chars<double>(void* value, char* buffer) {
return jkj::dragonbox::to_chars_n(*static_cast<double*>(value), buffer);
}
template<>
inline char*
aq_to_chars<bool>(void* value, char* buffer) {
if (*static_cast<bool*>(value)){
memcpy(buffer, "true", 4);
return buffer + 4;
}
else{
memcpy(buffer, "false", 5);
return buffer + 5;
}
}
template<>
inline char*
aq_to_chars<char*>(void* value, char* buffer) {
const auto src = *static_cast<char**>(value);
const auto len = strlen(src);
memcpy(buffer, src, len);
return buffer + len;
}
template<>
inline char*
aq_to_chars<types::date_t>(void* value, char* buffer) {
const auto& src = *static_cast<types::date_t*>(value);
buffer = to_text(buffer, src.year);
*buffer++ = '-';
buffer = to_text(buffer, src.month);
*buffer++ = '-';
buffer = to_text(buffer, src.day);
return buffer;
}
template<>
inline char*
aq_to_chars<types::time_t>(void* value, char* buffer) {
const auto& src = *static_cast<types::time_t*>(value);
buffer = to_text(buffer, src.hours);
*buffer++ = ':';
buffer = to_text(buffer, src.minutes);
*buffer++ = ':';
buffer = to_text(buffer, src.seconds);
*buffer++ = ':';
buffer = to_text(buffer, src.ms);
return buffer;
}
template<>
inline char*
aq_to_chars<types::timestamp_t>(void* value, char* buffer) {
auto& src = *static_cast<types::timestamp_t*>(value);
buffer = aq_to_chars<types::date_t>(static_cast<void*>(&src.date), buffer);
*buffer++ = ' ';
buffer = aq_to_chars<types::date_t>(static_cast<void*>(&src.time), buffer);
return buffer;
}
server/libaquery.h
+35 -1
View File
@@ -117,6 +117,40 @@ struct Context{
#define __AQEXPORT__(_Ty) extern "C" _Ty __DLLEXPORT__
typedef void (*deallocator_t) (void*);
#include <type_traits>
#include "jeaiii_to_text.h"
template<class T>
inline std::enable_if_t<std::is_integral_v<T>, char *>
aq_to_chars(void* value, char* buffer) {
return to_text(buffer, *static_cast<T*>(value));
}
template<class T>
inline std::enable_if_t<!std::is_integral_v<T>, char *>
aq_to_chars(void* value, char* buffer) {
return buffer;
}
#ifdef __SIZEOF_INT128__
template<>
inline char*
aq_to_chars<__int128_t>(void* value, char* buffer) {
return jeaiii_i128<__int128_t>(buffer, *static_cast<__int128_t*>(value));
}
template<>
inline char*
aq_to_chars<__uint128_t>(void* value, char* buffer) {
return jeaiii_i128<__uint128_t>(buffer, *static_cast<__uint128_t*>(value));
}
#endif
void test();
template<> char* aq_to_chars<float>(void* , char*);
template<> char* aq_to_chars<double>(void* , char*);
template<> char* aq_to_chars<char*>(void* , char*);
template<> char* aq_to_chars<types::date_t>(void* , char*);
template<> char* aq_to_chars<types::time_t>(void* , char*);
template<> char* aq_to_chars<types::timestamp_t>(void* , char*);
#endif
server/monetdb_conn.cpp
+5 -3
View File
@@ -9,7 +9,7 @@
#undef static_assert
const char* monetdbe_type_str[] = {
constexpr const char* monetdbe_type_str[] = {
"monetdbe_bool", "monetdbe_int8_t", "monetdbe_int16_t", "monetdbe_int32_t", "monetdbe_int64_t",
#ifdef HAVE_HGE
"monetdbe_int128_t",
@@ -22,7 +22,7 @@ const char* monetdbe_type_str[] = {
"monetdbe_type_unknown"
} ;
const unsigned char monetdbe_type_szs[] = {
inline constexpr static unsigned char monetdbe_type_szs[] = {
sizeof(monetdbe_column_bool::null_value), sizeof(monetdbe_column_int8_t::null_value),
sizeof(monetdbe_column_int16_t::null_value), sizeof(monetdbe_column_int32_t::null_value),
sizeof(monetdbe_column_int64_t::null_value),
@@ -37,8 +37,9 @@ const unsigned char monetdbe_type_szs[] = {
// should be last:
1
};
namespace types{
const Type_t monetdbe_type_aqtypes[] = {
constexpr const Type_t monetdbe_type_aqtypes[] = {
ABOOL, AINT8, AINT16, AINT32, AINT64,
#ifdef HAVE_HGE
AINT128,
@@ -50,6 +51,7 @@ namespace types{
};
}
Server::Server(Context* cxt){
if (cxt){
connect(cxt);
server/server.cpp
+84 -2
View File
@@ -161,6 +161,89 @@ __AQEXPORT__(bool) have_hge(){
#endif
}
using prt_fn_t = char* (*)(void*, char*);
constexpr prt_fn_t monetdbe_prtfns[] = {
aq_to_chars<bool>, aq_to_chars<int8_t>, aq_to_chars<int16_t>, aq_to_chars<int32_t>,
aq_to_chars<int64_t>,
#if __SIZEOF_INT128__
aq_to_chars<__int128_t>,
#endif
aq_to_chars<size_t>, aq_to_chars<float>, aq_to_chars<double>,
aq_to_chars<char*>, aq_to_chars<nullptr_t>,
aq_to_chars<types::date_t>, aq_to_chars<types::time_t>, aq_to_chars<types::timestamp_t>,
// should be last:
aq_to_chars<nullptr_t>
};
#include "monetdbe.h"
inline constexpr static unsigned char monetdbe_type_szs[] = {
sizeof(monetdbe_column_bool::null_value), sizeof(monetdbe_column_int8_t::null_value),
sizeof(monetdbe_column_int16_t::null_value), sizeof(monetdbe_column_int32_t::null_value),
sizeof(monetdbe_column_int64_t::null_value),
#ifdef __SIZEOF_INT128__
sizeof(monetdbe_column_int128_t::null_value),
#endif
sizeof(monetdbe_column_size_t::null_value), sizeof(monetdbe_column_float::null_value),
sizeof(monetdbe_column_double::null_value),
sizeof(monetdbe_column_str::null_value), sizeof(monetdbe_column_blob::null_value),
sizeof(monetdbe_data_date), sizeof(monetdbe_data_time), sizeof(monetdbe_data_timestamp),
// should be last:
1
};
constexpr uint32_t output_buffer_size = 65536;
void print_monetdb_results(Server* srv, const char* sep = " ", const char* end = "\n",
uint32_t limit = std::numeric_limits<uint32_t>::max()) {
if (!srv->haserror() && limit){
char buffer[output_buffer_size];
auto _res = static_cast<monetdbe_result*> (srv->res);
const auto& ncols = _res->ncols;
monetdbe_column** cols = static_cast<monetdbe_column**>(malloc(sizeof(monetdbe_column*) * ncols));
prt_fn_t *prtfns = (prt_fn_t*) alloca(sizeof(prt_fn_t) * ncols);
char** col_data = static_cast<char**> (alloca(sizeof(char*) * ncols));
uint8_t* szs = static_cast<uint8_t*>(alloca(ncols));
std::string header_string = "";
const char* err_msg = nullptr;
for(uint32_t i = 0; i < ncols; ++i){
err_msg = monetdbe_result_fetch(_res, &cols[i], i);
col_data[i] = static_cast<char *>(cols[i]->data);
prtfns[i] = monetdbe_prtfns[cols[i]->type];
szs [i] = monetdbe_type_szs[cols[i]->type];
header_string = header_string + cols[i]->name + sep + '|' + sep;
}
const size_t l_sep = strlen(sep);
const size_t l_end = strlen(end);
if (header_string.size() - l_sep - 1>= 0)
header_string.resize(header_string.size() - l_sep - 1);
header_string += end + std::string(header_string.size(), '=') + end;
fputs(header_string.c_str(), stdout);
char* _buffer = buffer;
for(uint64_t i = 0; i < srv->cnt; ++i){
for(uint32_t j = 0; j < ncols; ++j){
//copy the field to buf
_buffer = prtfns[j](col_data[j], _buffer);
if (j != ncols - 1){
memcpy(_buffer, sep, l_sep);
_buffer += l_sep;
}
col_data[j] += szs[j];
}
memcpy(_buffer, end, l_end);
_buffer += l_end;
if(output_buffer_size - (_buffer - buffer) <= 1024){
fwrite(buffer, 1, _buffer - buffer, stdout);
_buffer = buffer;
}
}
if (_buffer != buffer)
fwrite(buffer, 1, _buffer - buffer, stdout);
free(cols);
}
}
void initialize_module(const char* module_name, void* module_handle, Context* cxt){
auto _init_module = reinterpret_cast<module_init_fn>(dlsym(module_handle, "init_session"));
@@ -173,7 +256,6 @@ void initialize_module(const char* module_name, void* module_handle, Context* cx
}
int dll_main(int argc, char** argv, Context* cxt){
test();
aq_timer timer;
Config *cfg = reinterpret_cast<Config *>(argv[0]);
std::unordered_map<std::string, void*> user_module_map;
@@ -266,7 +348,7 @@ int dll_main(int argc, char** argv, Context* cxt){
{
if(!server->haserror()){
timer.reset();
server->print_results();
print_monetdb_results(server);
cfg->stats.postproc_time += timer.elapsed();
}
}