From a464029d1464106d52aeba07c678e951e9985305 Mon Sep 17 00:00:00 2001 From: Bill Date: Thu, 8 Sep 2022 05:06:56 +0800 Subject: [PATCH] Added select into --- Makefile | 2 + README.md | 1 + aquery_config.py | 6 ++- aquery_parser/sql_parser.py | 1 + prompt.py | 3 +- reconstruct/ast.py | 46 ++++++++++++++++++--- reconstruct/storage.py | 10 +++-- server/io.h | 21 ++++++---- server/monetdb_conn.cpp | 6 ++- server/monetdb_conn.h | 6 +++ server/server.cpp | 51 ++++++++++++++++------- server/table.cpp | 10 +++-- server/table.h | 27 ++++++++++-- server/table_ext_monetdb.hpp | 79 ++++++++++++++++++++++++++++++++++++ server/types.h | 15 ++++--- server/vector_type.hpp | 8 ++-- 16 files changed, 241 insertions(+), 51 deletions(-) create mode 100644 server/table_ext_monetdb.hpp diff --git a/Makefile b/Makefile index cdb55dc..ee54d9d 100644 --- a/Makefile +++ b/Makefile @@ -15,6 +15,8 @@ ifeq ($(OS),Windows_NT) OS_SUPPORT += server/winhelper.cpp MonetDB_LIB += -Imonetdb/msvc msc-plugin/monetdbe.dll else + UNAME_S = $(shell uname -s) + UNAME_M = $(shell uname -m) NULL_DEVICE = /dev/null MonetDB_LIB += -I/usr/local/include/monetdb -I/usr/include/monetdb -lmonetdbe endif diff --git a/README.md b/README.md index 1faa59f..d42a5f5 100644 --- a/README.md +++ b/README.md @@ -48,6 +48,7 @@ AQuery++ Database is a cross-platform, In-Memory Column-Store Database that inco - [x] Bug fixes: type deduction misaligned in Hybrid Engine - [ ] Investigation: Using postproc only for q1 in Hybrid Engine (make is_special always on) - [ ] Limitation: putting ColRefs back to monetdb. +- [ ] Limitation: String operations and Date/Time data type. - [ ] C++ Meta-Programming: Eliminate template recursions as much as possible. # Installation diff --git a/aquery_config.py b/aquery_config.py index 6f86395..e3d9f19 100644 --- a/aquery_config.py +++ b/aquery_config.py @@ -1,9 +1,9 @@ # put environment specific configuration here ## GLOBAL CONFIGURATION FLAGS -version_string = '0.4.1a' +version_string = '0.4.2a' add_path_to_ldpath = True -rebuild_backend = True +rebuild_backend = False run_backend = True have_hge = False os_platform = 'unkown' @@ -15,6 +15,8 @@ class Build_Config: def __init__(self) -> None: self.OptimizationLv = '4' # [O0, O1, O2, O3, Ofast] self.Platform = 'x64' + self.PCH = '' + self.StaticLib = '' self.fLTO = True self.fmarchnative = True diff --git a/aquery_parser/sql_parser.py b/aquery_parser/sql_parser.py index a531e7f..c07aea3 100644 --- a/aquery_parser/sql_parser.py +++ b/aquery_parser/sql_parser.py @@ -401,6 +401,7 @@ def parser(literal_string, ident, sqlserver=False): unordered_sql = Group( values | selection + + Optional(INTO + table_source("into")) + Optional( (FROM + delimited_list(table_source) + ZeroOrMore(join))("from") + Optional(WHERE + expr("where")) diff --git a/prompt.py b/prompt.py index 5209cf4..0e536ac 100644 --- a/prompt.py +++ b/prompt.py @@ -1,3 +1,4 @@ +from multiprocessing.sharedctypes import Value import aquery_config help_message = '''\ ====================================================== @@ -472,7 +473,7 @@ def prompt(running = lambda:True, next = input, state = None): except SystemExit: print("\nBye.") raise - except BaseException as e: + except ValueError as e: import code, traceback raise_exception = True sh = code.InteractiveConsole({**globals(), **locals()}) diff --git a/reconstruct/ast.py b/reconstruct/ast.py index 63f4aa2..1a76a92 100644 --- a/reconstruct/ast.py +++ b/reconstruct/ast.py @@ -91,7 +91,12 @@ class projection(ast_node): def consume(self, node): # deal with projections - self.out_table = TableInfo('out_'+base62uuid(4), [], self.context) + if 'into' not in node: + out_table_name = 'out_'+base62uuid(6) + else: + out_table_name = node['into'] + + self.out_table : TableInfo = TableInfo(out_table_name, [], self.context) cols = [] self.col_ext : Set[ColRef]= set() col_exprs : List[Tuple[str, Types]] = [] @@ -139,6 +144,8 @@ class projection(ast_node): # TODO: Type deduction in Python cols.append(ColRef(this_type, self.out_table, None, disp_name, i, compound=compound)) + self.out_table.add_cols(cols, new = False) + if 'groupby' in node: self.group_node = groupby(self, node['groupby']) else: @@ -200,7 +207,7 @@ class projection(ast_node): self.context.emitc(f'auto {vname} = ColRef<{typenames[idx].cname}>({length_name}, server->getCol({idx}));') vid2cname[idx] = vname # Create table into context - self.outtable_name = 'out_' + base62uuid(6) + self.outtable_name = self.out_table.table_name out_typenames = [None] * len(proj_map) for key, val in proj_map.items(): @@ -240,9 +247,11 @@ class projection(ast_node): gb_cexprs.append((col_name, val[2])) self.group_node.finalize(gb_cexprs, gb_vartable) else: - for key, val in proj_map.items(): + for i, (key, val) in enumerate(proj_map.items()): if type(val[1]) is int: - self.context.emitc(f'{self.outtable_name}->get_col<{key}>().initfrom({vid2cname[val[1]]});') + self.context.emitc( + f'{self.outtable_name}->get_col<{key}>().initfrom({vid2cname[val[1]]}, "{cols[i].name}");' + ) else: # for funcs evaluate f_i(x, ...) self.context.emitc(f'{self.outtable_name}->get_col<{key}>() = {val[1]};') @@ -251,13 +260,40 @@ class projection(ast_node): if self.outfile: self.outfile.finalize() + + if 'into' in node: + self.context.emitc(select_into(self, node['into']).ccode) + self.context.emitc(f'puts("done.");') if self.parent is None: self.context.sql_end() self.context.postproc_end(self.postproc_fname) - + + +class select_into(ast_node): + def init(self, node): + if type(self.parent) is projection: + if self.context.has_dll: + # has postproc put back to monetdb + self.produce = self.produce_cpp + else: + self.produce = self.produce_sql + else: + raise ValueError('parent must be projection') + def produce_cpp(self, node): + assert(type(self.parent) is projection) + if not hasattr(self.parent, 'out_table'): + raise Exception('No out_table found.') + else: + self.context.headers.add('"./server/table_ext_monetdb.hpp"') + self.ccode = f'{self.parent.out_table.table_name}->monetdb_append_table(cxt->alt_server, \"{node}\");' + + def produce_sql(self, node): + self.sql = f' INTO {node}' + + class orderby(ast_node): name = 'order by' def produce(self, node): diff --git a/reconstruct/storage.py b/reconstruct/storage.py index d387855..8aa6aed 100644 --- a/reconstruct/storage.py +++ b/reconstruct/storage.py @@ -1,5 +1,7 @@ from engine.types import * from engine.utils import enlist +from typing import List, Dict, Set + class ColRef: def __init__(self, _ty, cobj, table:'TableInfo', name, id, compound = False, _ty_args = None): self.type : Types = AnyT @@ -30,10 +32,10 @@ class ColRef: class TableInfo: def __init__(self, table_name, cols, cxt:'Context'): # statics - self.table_name = table_name - self.alias = set([table_name]) - self.columns_byname = dict() # column_name, type - self.columns = [] + self.table_name : str= table_name + self.alias : Set[str] = set([table_name]) + self.columns_byname : Dict[str, ColRef] = dict() # column_name, type + self.columns : List[ColRef] = [] self.cxt = cxt # keep track of temp vars self.rec = None diff --git a/server/io.h b/server/io.h index fc87a06..56019d5 100644 --- a/server/io.h +++ b/server/io.h @@ -15,6 +15,16 @@ std::string generate_printf_string(const char* sep = " ", const char* end = "\n" return str; } +template +inline decltype(auto) print_hook(const T& v){ + return v; +} + +template<> +inline decltype(auto) print_hook(const bool& v) { + return v? "true" : "false"; +} + #ifdef __SIZEOF_INT128__ constexpr struct __int128__struct{ uint64_t low, high; @@ -53,24 +63,19 @@ extern char* gbuf; void setgbuf(char* buf = 0); -template -inline decltype(auto) printi128(const T& v){ - return v; -} - template<> -inline decltype(auto) printi128<__int128_t>(const __int128_t& v) { +inline decltype(auto) print_hook<__int128_t>(const __int128_t& v) { *(gbuf+=40) = 0; return get_int128str(v, gbuf++); } template<> -inline decltype(auto) printi128<__uint128_t>(const __uint128_t& v) { +inline decltype(auto) print_hook<__uint128_t>(const __uint128_t& v) { *(gbuf+=40) = 0; return get_uint128str(v, gbuf++); } #else -#define printi128(x) x + #define setgbuf() #endif diff --git a/server/monetdb_conn.cpp b/server/monetdb_conn.cpp index 61428d8..bd1fb81 100644 --- a/server/monetdb_conn.cpp +++ b/server/monetdb_conn.cpp @@ -2,6 +2,7 @@ #include #include "monetdb_conn.h" #include "monetdbe.h" +#include "table.h" #undef static_assert const char* monetdbe_type_str[] = { @@ -16,6 +17,7 @@ const char* monetdbe_type_str[] = { // should be last: "monetdbe_type_unknown" } ; + const unsigned char monetdbe_type_szs[] = { sizeof(monetdbe_column_bool::null_value), sizeof(monetdbe_column_int8_t::null_value), sizeof(monetdbe_column_int16_t::null_value), sizeof(monetdbe_column_int32_t::null_value), @@ -31,11 +33,13 @@ const unsigned char monetdbe_type_szs[] = { // should be last: 1 }; + + + Server::Server(Context* cxt){ if (cxt){ connect(cxt); } - } void Server::connect(Context *cxt){ diff --git a/server/monetdb_conn.h b/server/monetdb_conn.h index 5afae0e..467cb2c 100644 --- a/server/monetdb_conn.h +++ b/server/monetdb_conn.h @@ -25,4 +25,10 @@ struct Server{ ~Server(); }; +struct monetdbe_table_data{ + const char* table_name; + const char* create_table_sql; + void* cols; +}; + #endif diff --git a/server/server.cpp b/server/server.cpp index fcbdbad..193eb9e 100644 --- a/server/server.cpp +++ b/server/server.cpp @@ -81,9 +81,9 @@ void* Context::get_module_function(const char* fname){ auto fmap = static_cast*> (this->module_function_maps); //printf("%p\n", fmap->find("mydiv")->second); - for (const auto& [key, value] : *fmap){ - printf("%s %p\n", key.c_str(), value); - } + // for (const auto& [key, value] : *fmap){ + // printf("%s %p\n", key.c_str(), value); + // } auto ret = fmap->find(fname); return ret == fmap->end() ? nullptr : ret->second; } @@ -145,9 +145,9 @@ int dll_main(int argc, char** argv, Context* cxt){ case 'F': { auto fname = n_recvd[i] + 1; - printf("%s: %p, %p\n", fname, user_module_handle, dlsym(user_module_handle, fname)); + //printf("%s: %p, %p\n", fname, user_module_handle, dlsym(user_module_handle, fname)); module_fn_map->insert_or_assign(fname, dlsym(user_module_handle, fname)); - printf("%p\n", module_fn_map->find("mydiv") != module_fn_map->end() ? module_fn_map->find("mydiv")->second : nullptr); + //printf("%p\n", module_fn_map->find("mydiv") != module_fn_map->end() ? module_fn_map->find("mydiv")->second : nullptr); } break; case 'U': @@ -193,6 +193,16 @@ int dll_main(int argc, char** argv, Context* cxt){ return 0; } +int launcher(int argc, char** argv){ + std::string str = " "; + for (int i = 0; i < argc; i++){ + str += argv[i]; + str += " "; + } + str = std::string("python3 prompt.py ") + str; + return system(str.c_str()); +} + extern "C" int __DLLEXPORT__ main(int argc, char** argv) { puts("running"); Context* cxt = new Context(); @@ -201,14 +211,17 @@ extern "C" int __DLLEXPORT__ main(int argc, char** argv) { #ifdef THREADING auto tp = new ThreadPool(); cxt->thread_pool = tp; - #endif const char* shmname; if (argc < 0) - return dll_main(argc, argv, cxt); + return dll_main(argc, argv, cxt); else if (argc <= 1) - return test_main(); +#ifdef __AQ__TESTING__ + return test_main(); +#else + return launcher(argc, argv); +#endif else shmname = argv[1]; SharedMemory shm = SharedMemory(shmname); @@ -241,18 +254,29 @@ extern "C" int __DLLEXPORT__ main(int argc, char** argv) { shm.FreeMemoryMap(); return 0; } + #include "utils.h" +#include "table_ext_monetdb.hpp" int test_main() { - //vector_type t; - //t = 1; - //t.emplace_back(2); - //print(t); - //return 0; Context* cxt = new Context(); if (cxt->alt_server == 0) cxt->alt_server = new Server(cxt); Server* server = reinterpret_cast(cxt->alt_server); + + + TableInfo table("sibal"); + int col0[] = { 1,2,3,4,5 }; + float col1[] = { 5.f, 4.f, 3.f, 2.f, 1.f }; + table.get_col<0>().initfrom(5, col0, "a"); + table.get_col<1>().initfrom(5, col1, "b"); + table.monetdb_append_table(server); + + server->exec("select * from sibal;"); + auto aa = server->getCol(0); + auto bb = server->getCol(1); + printf("sibal: %p %p\n", aa, bb); + const char* qs[]= { "SELECT MIN(3)-MAX(2);", "CREATE TABLE stocks(timestamp INT, price INT);", @@ -302,6 +326,5 @@ int test_main() //static_assert(std::is_same_v()), std::integer_sequence>, ""); return 0; - std::unordered_map a; } diff --git a/server/table.cpp b/server/table.cpp index 0eafe73..5e5d194 100644 --- a/server/table.cpp +++ b/server/table.cpp @@ -6,14 +6,13 @@ template <> void print<__int128_t>(const __int128_t& v, const char* delimiter){ char s[41]; s[40] = 0; - std::cout<< get_int128str(v, s+40); + std::cout<< get_int128str(v, s+40)<< delimiter; } template <> void print<__uint128_t>(const __uint128_t&v, const char* delimiter){ char s[41]; s[40] = 0; - std::cout<< get_uint128str(v, s+40); - + std::cout<< get_uint128str(v, s+40) << delimiter; } std::ostream& operator<<(std::ostream& os, __int128 & v) { @@ -26,3 +25,8 @@ std::ostream& operator<<(std::ostream& os, __uint128_t & v) return os; } #endif + +template <> +void print(const bool&v, const char* delimiter){ + std::cout<< (v?"true":"false") << delimiter; +} diff --git a/server/table.h b/server/table.h index 4dcc4ec..639d761 100644 --- a/server/table.h +++ b/server/table.h @@ -90,6 +90,15 @@ public: } template ColRef scast(); + + ColRef<_Ty>* rename(const char* name) { + this->name = name; + return this; + } + + // defined in table_ext_monetdb.hpp + void* monetdb_get_col(); + }; template @@ -297,9 +306,9 @@ struct TableInfo { const auto& this_value = get(*this)[i]; const auto& next = [&](auto &v) { if constexpr (sizeof...(rem_cols) == 0) - func(args..., printi128(v)); + func(args..., print_hook(v)); else - print2_impl(func, i, args ..., printi128(v)); + print2_impl(func, i, args ..., print_hook(v)); }; if constexpr (is_vector_type) for (int j = 0; j < this_value.size; ++j) @@ -368,6 +377,13 @@ struct TableInfo { const vector_type* __restrict view = nullptr, FILE* __restrict fp = nullptr) { applyIntegerSequence::apply(*this, sep, end, view, fp); } + + TableInfo< Types... >* rename(const char* name) { + this->name = name; + return this; + } + // defined in monetdb_conn.cpp + void monetdb_append_table(void* srv, const char* alt_name = nullptr); }; @@ -399,14 +415,14 @@ constexpr static inline bool is_vector(const vector_type&) { template TableInfo::TableInfo(const char* name, uint32_t n_cols) : name(name), n_cols(n_cols) { this->colrefs = (ColRef*)malloc(sizeof(ColRef) * n_cols); - for (int i = 0; i < n_cols; ++i) { + for (uint32_t i = 0; i < n_cols; ++i) { this->colrefs[i].init(); } } template TableInfo::TableInfo(const char* name) : name(name), n_cols(sizeof...(Types)) { this->colrefs = (ColRef*)malloc(sizeof(ColRef) * this->n_cols); - for (int i = 0; i < n_cols; ++i) { + for (uint32_t i = 0; i < n_cols; ++i) { this->colrefs[i].init(); } } @@ -573,6 +589,8 @@ void print<__int128_t>(const __int128_t& v, const char* delimiter); template <> void print<__uint128_t>(const __uint128_t&v, const char* delimiter); #endif +template <> +void print(const bool&v, const char* delimiter); template void inline print_impl(const T& v, const char* delimiter, const char* endline) { @@ -591,4 +609,5 @@ print(const VT& v, const char* delimiter = " ", const char* endline = "\n") { print_impl(v, delimiter, endline); } + #endif diff --git a/server/table_ext_monetdb.hpp b/server/table_ext_monetdb.hpp new file mode 100644 index 0000000..94b244e --- /dev/null +++ b/server/table_ext_monetdb.hpp @@ -0,0 +1,79 @@ +// TODO: Think of a way of decoupling table.h part and +// monetdbe.h part to speed up compilation. + +#ifndef __TABLE_EXT_MONETDB_HPP__ +#define __TABLE_EXT_MONETDB_HPP__ + +#include "table.h" +#include "monetdb_conn.h" +#include "monetdbe.h" + +inline constexpr monetdbe_types AQType_2_monetdbe[] = { + monetdbe_int32_t, monetdbe_float, monetdbe_str, monetdbe_double, monetdbe_int64_t, +#ifdef HAVE_HGE + monetdbe_int128_t, +#else + monetdbe_int64_t, +#endif + monetdbe_int16_t, monetdbe_date, monetdbe_time, monetdbe_int8_t, + monetdbe_int32_t, monetdbe_int64_t, +#ifdef HAVE_HGE + monetdbe_int128_t, +#else + monetdbe_int64_t, +#endif + monetdbe_int16_t, monetdbe_int8_t, monetdbe_bool, monetdbe_int64_t, monetdbe_int64_t, monetdbe_int64_t +}; + +template +void TableInfo::monetdb_append_table(void* srv, const char* alt_name) { + if (!alt_name){ + alt_name = this->name; + } + + monetdbe_column** monetdbe_cols = new monetdbe_column * [sizeof...(Ts)]; + + uint32_t i = 0; + const auto get_col = [&monetdbe_cols, &i](auto v) { + monetdbe_cols[i++] = (monetdbe_column*)v->monetdb_get_col(); + }; + (get_col((ColRef*)(colrefs + i)), ...); + + std::string create_table_str = "CREATE TABLE "; + create_table_str += alt_name; + create_table_str += " ("; + i = 0; + const auto get_name_type = [&i, *this]() { + return std::string(colrefs[i++].name) + ' '; + }; + create_table_str += ((get_name_type() + types::SQL_Type[types::Types::getType()] + ", ") + ... + std::string("")); + auto last_comma = create_table_str.find_last_of(','); + if (last_comma != static_cast(-1)) { + create_table_str[last_comma] = ')'; + Server* server = (Server*)srv; + server->exec(create_table_str.c_str()); + if (!server->last_error) { + auto err = monetdbe_append(*((monetdbe_database*)server->server), "sys", alt_name, monetdbe_cols, sizeof...(Ts)); + if (err) + puts(err); + return; + } + } + puts("Error! Empty table."); +} + + +template +void* ColRef::monetdb_get_col() { + auto aq_type = AQType_2_monetdbe[types::Types::getType()]; + monetdbe_column* col = (monetdbe_column*)malloc(sizeof(monetdbe_column)); + + col->type = aq_type; + col->count = this->size; + col->data = this->container; + col->name = const_cast(this->name); + + return col; +} + +#endif diff --git a/server/types.h b/server/types.h index 562a0c4..16d824b 100644 --- a/server/types.h +++ b/server/types.h @@ -22,19 +22,23 @@ constexpr static bool is_vector_type = is_vector_impl::value; namespace types { enum Type_t { AINT32, AFLOAT, ASTR, ADOUBLE, ALDOUBLE, AINT64, AINT128, AINT16, ADATE, ATIME, AINT8, - AUINT32, AUINT64, AUINT128, AUINT16, AUINT8, VECTOR, NONE, ERROR + AUINT32, AUINT64, AUINT128, AUINT16, AUINT8, ABOOL, VECTOR, NONE, ERROR }; - static constexpr const char* printf_str[] = { "%d", "%f", "%s", "%lf", "%Lf", "%ld", "%s", "%hi", "%s", "%s", "%c", - "%u", "%lu", "%s", "%hu", "%hhu", "Vector<%s>", "NULL", "ERROR" }; + static constexpr const char* printf_str[] = { "%d", "%f", "%s", "%lf", "%Lf", "%ld", "%d", "%hi", "%s", "%s", "%c", + "%u", "%lu", "%s", "%hu", "%hhu", "%s", "%s", "Vector<%s>", "NULL", "ERROR" }; + static constexpr const char* SQL_Type[] = { "INT", "REAL", "TEXT", "DOUBLE", "DOUBLE", "BIGINT", "HUGEINT", "SMALLINT", "DATE", "TIME", "TINYINT", + "INT", "BIGINT", "HUGEINT", "SMALLINT", "TINYINT", "BIGINT", "BOOL", "BIGINT", "NULL", "ERROR"}; + + // TODO: deal with data/time <=> str/uint conversion struct date_t { - uint32_t val; + uint32_t val = 0; date_t(const char* d) { } std::string toString() const; }; struct time_t { - uint32_t val; + uint32_t val = 0; time_t(const char* d) { } std::string toString() const; @@ -69,6 +73,7 @@ namespace types { f(unsigned long, AUINT64) \ f(unsigned short, AUINT16) \ f(unsigned char, AUINT8) \ + f(bool, ABOOL) \ F_INT128(f) inline constexpr static Type_t getType() { diff --git a/server/vector_type.hpp b/server/vector_type.hpp index d88b1ed..378ed01 100644 --- a/server/vector_type.hpp +++ b/server/vector_type.hpp @@ -255,17 +255,17 @@ public: #define Ops(o, x) \ template\ vector_type::type> operator o (const vector_type& r) const {\ - [[likely]] if (r.size == size) {\ + /*[[likely]] if (r.size == size) {*/\ return x(r);\ - }\ + /*}*/\ } #define Opseq(o, x) \ template\ vector_type::type> operator o##= (const vector_type& r) {\ - [[likely]] if (r.size == size) {\ + /*[[likely]] if (r.size == size) {*/\ return x##eq(r);\ - }\ + /*}*/\ } #define _Make_Ops(M) \