Added select into

Makefile

@@ -15,6 +15,8 @@ ifeq ($(OS),Windows_NT)
 	OS_SUPPORT += server/winhelper.cpp
 	MonetDB_LIB += -Imonetdb/msvc msc-plugin/monetdbe.dll 
 else
+	UNAME_S = $(shell uname -s)
+	UNAME_M = $(shell uname -m)
 	NULL_DEVICE = /dev/null
 	MonetDB_LIB += -I/usr/local/include/monetdb -I/usr/include/monetdb -lmonetdbe
 endif

README.md

@@ -48,6 +48,7 @@ AQuery++ Database is a cross-platform, In-Memory Column-Store Database that inco
 - [x] Bug fixes: type deduction misaligned in Hybrid Engine
 - [ ] Investigation: Using postproc only for q1 in Hybrid Engine (make is_special always on)
 - [ ] Limitation: putting ColRefs back to monetdb. 
+- [ ] Limitation: String operations and Date/Time data type. 
 - [ ] C++ Meta-Programming: Eliminate template recursions as much as possible.
 
 # Installation

aquery_config.py

@@ -1,9 +1,9 @@
 # put environment specific configuration here
 
 ## GLOBAL CONFIGURATION FLAGS
-version_string = '0.4.1a'
+version_string = '0.4.2a'
 add_path_to_ldpath = True
-rebuild_backend = True
+rebuild_backend = False
 run_backend = True
 have_hge = False
 os_platform = 'unkown'
@@ -15,6 +15,8 @@ class Build_Config:
     def __init__(self) -> None:
         self.OptimizationLv = '4' # [O0, O1, O2, O3, Ofast]
         self.Platform = 'x64'
+        self.PCH = ''
+        self.StaticLib = ''
         self.fLTO = True
         self.fmarchnative = True
         

aquery_parser/sql_parser.py

@@ -401,6 +401,7 @@ def parser(literal_string, ident, sqlserver=False):
         unordered_sql = Group(
             values
             | selection
+            + Optional(INTO + table_source("into"))
             + Optional(
                 (FROM + delimited_list(table_source) + ZeroOrMore(join))("from")
                 + Optional(WHERE + expr("where"))

prompt.py

@@ -1,3 +1,4 @@
+from multiprocessing.sharedctypes import Value
 import aquery_config
 help_message = '''\
 ======================================================
@@ -472,7 +473,7 @@ def prompt(running = lambda:True, next = input, state = None):
         except SystemExit:
             print("\nBye.")
             raise
-        except BaseException as e:
+        except ValueError as e:
             import code, traceback
             raise_exception = True
             sh = code.InteractiveConsole({**globals(), **locals()})

reconstruct/ast.py

@@ -91,7 +91,12 @@ class projection(ast_node):
 
     def consume(self, node):
         # deal with projections
-        self.out_table = TableInfo('out_'+base62uuid(4), [], self.context)
+        if 'into' not in node:
+            out_table_name = 'out_'+base62uuid(6)
+        else:
+            out_table_name = node['into']
+            
+        self.out_table : TableInfo = TableInfo(out_table_name, [], self.context)
         cols = []
         self.col_ext : Set[ColRef]= set()
         col_exprs : List[Tuple[str, Types]] = []
@@ -139,6 +144,8 @@ class projection(ast_node):
             # TODO: Type deduction in Python
             cols.append(ColRef(this_type, self.out_table, None, disp_name, i, compound=compound))
         
+        self.out_table.add_cols(cols, new = False)
+        
         if 'groupby' in node:
             self.group_node = groupby(self, node['groupby'])
         else:
@@ -200,7 +207,7 @@ class projection(ast_node):
             self.context.emitc(f'auto {vname} = ColRef<{typenames[idx].cname}>({length_name}, server->getCol({idx}));')
             vid2cname[idx] = vname
         # Create table into context
-        self.outtable_name = 'out_' + base62uuid(6)
+        self.outtable_name = self.out_table.table_name
         out_typenames = [None] * len(proj_map)
         
         for key, val in proj_map.items():
@@ -240,9 +247,11 @@ class projection(ast_node):
                 gb_cexprs.append((col_name, val[2]))
             self.group_node.finalize(gb_cexprs, gb_vartable)
         else:
-            for key, val in proj_map.items():
+            for i, (key, val) in enumerate(proj_map.items()):
                 if type(val[1]) is int:
-                    self.context.emitc(f'{self.outtable_name}->get_col<{key}>().initfrom({vid2cname[val[1]]});')
+                    self.context.emitc(
+                        f'{self.outtable_name}->get_col<{key}>().initfrom({vid2cname[val[1]]}, "{cols[i].name}");'
+                    )
                 else:
                     # for funcs evaluate f_i(x, ...)
                     self.context.emitc(f'{self.outtable_name}->get_col<{key}>() = {val[1]};')
@@ -251,6 +260,10 @@ class projection(ast_node):
         
         if self.outfile:
             self.outfile.finalize()
+
+        if 'into' in node:
+            self.context.emitc(select_into(self, node['into']).ccode)
+        
         self.context.emitc(f'puts("done.");')
 
         if self.parent is None:
@@ -258,6 +271,29 @@ class projection(ast_node):
             self.context.postproc_end(self.postproc_fname)
         
 
+
+class select_into(ast_node):
+    def init(self, node):
+        if type(self.parent) is projection:
+            if self.context.has_dll:
+                # has postproc put back to monetdb
+                self.produce = self.produce_cpp
+            else:
+                self.produce = self.produce_sql
+        else:
+            raise ValueError('parent must be projection')
+    def produce_cpp(self, node):
+        assert(type(self.parent) is projection)
+        if not hasattr(self.parent, 'out_table'):
+            raise Exception('No out_table found.')
+        else:
+            self.context.headers.add('"./server/table_ext_monetdb.hpp"')
+            self.ccode = f'{self.parent.out_table.table_name}->monetdb_append_table(cxt->alt_server, \"{node}\");'
+            
+    def produce_sql(self, node):
+        self.sql = f' INTO {node}'
+    
+
 class orderby(ast_node):
     name = 'order by'
     def produce(self, node):

reconstruct/storage.py

@@ -1,5 +1,7 @@
 from engine.types import *
 from engine.utils import enlist
+from typing import List, Dict, Set
+
 class ColRef:
     def __init__(self, _ty, cobj, table:'TableInfo', name, id, compound = False, _ty_args = None):
         self.type : Types = AnyT
@@ -30,10 +32,10 @@ class ColRef:
 class TableInfo:
     def __init__(self, table_name, cols, cxt:'Context'):
         # statics
-        self.table_name = table_name
+        self.table_name : str= table_name
-        self.alias = set([table_name])
+        self.alias : Set[str] = set([table_name])
-        self.columns_byname = dict() # column_name, type
+        self.columns_byname : Dict[str, ColRef] = dict() # column_name, type
-        self.columns = []
+        self.columns : List[ColRef] = []
         self.cxt = cxt
         # keep track of temp vars
         self.rec = None 

server/io.h

@@ -15,6 +15,16 @@ std::string generate_printf_string(const char* sep = " ", const char* end = "\n"
 	return str;
 }
 
+template<class T>
+inline decltype(auto) print_hook(const T& v){
+    return v;
+}
+
+template<>
+inline decltype(auto) print_hook<bool>(const bool& v) {
+	return v? "true" : "false";
+}
+
 #ifdef __SIZEOF_INT128__
 constexpr struct __int128__struct{
     uint64_t low, high;
@@ -53,24 +63,19 @@ extern char* gbuf;
 
 void setgbuf(char* buf = 0);
 
-template<class T>
-inline decltype(auto) printi128(const T& v){
-    return v;
-}
-
 template<>
-inline decltype(auto) printi128<__int128_t>(const __int128_t& v) {
+inline decltype(auto) print_hook<__int128_t>(const __int128_t& v) {
     *(gbuf+=40) = 0;
     return get_int128str(v, gbuf++);
 }
 
 template<>
-inline decltype(auto) printi128<__uint128_t>(const __uint128_t& v) {
+inline decltype(auto) print_hook<__uint128_t>(const __uint128_t& v) {
     *(gbuf+=40) = 0;
     return get_uint128str(v, gbuf++);
 }
 
 #else
-#define printi128(x) x
+
 #define setgbuf()
 #endif

server/monetdb_conn.cpp

@@ -2,6 +2,7 @@
 #include <cstdio>
 #include "monetdb_conn.h"
 #include "monetdbe.h"
+#include "table.h"
 #undef static_assert
 
 const char* monetdbe_type_str[] = {
@@ -16,6 +17,7 @@ const char* monetdbe_type_str[] = {
 	// should be last:
 	"monetdbe_type_unknown"
 } ;
+
 const unsigned char monetdbe_type_szs[] = {
     sizeof(monetdbe_column_bool::null_value), sizeof(monetdbe_column_int8_t::null_value), 
     sizeof(monetdbe_column_int16_t::null_value), sizeof(monetdbe_column_int32_t::null_value), 
@@ -31,11 +33,13 @@ const unsigned char monetdbe_type_szs[] = {
     // should be last:
     1
 };
+
+
+
 Server::Server(Context* cxt){
     if (cxt){
         connect(cxt);
     } 
-    
 }
 
 void Server::connect(Context *cxt){

server/monetdb_conn.h

@@ -25,4 +25,10 @@ struct Server{
     ~Server();
 };
 
+struct monetdbe_table_data{
+    const char* table_name;
+    const char* create_table_sql;
+    void* cols;
+};
+
 #endif

server/server.cpp

@@ -81,9 +81,9 @@ void* Context::get_module_function(const char* fname){
     auto fmap = static_cast<std::unordered_map<std::string, void*>*>
         (this->module_function_maps);
     //printf("%p\n", fmap->find("mydiv")->second);
-    for (const auto& [key, value] : *fmap){
+    // for (const auto& [key, value] : *fmap){
-        printf("%s %p\n", key.c_str(), value);
+    //     printf("%s %p\n", key.c_str(), value);
-    }
+    // }
     auto ret = fmap->find(fname);
     return ret == fmap->end() ? nullptr : ret->second;
 }
@@ -145,9 +145,9 @@ int dll_main(int argc, char** argv, Context* cxt){
                         case 'F':
                             {
                                 auto fname = n_recvd[i] + 1;
-                                printf("%s: %p, %p\n", fname, user_module_handle, dlsym(user_module_handle, fname));
+                                //printf("%s: %p, %p\n", fname, user_module_handle, dlsym(user_module_handle, fname));
                                 module_fn_map->insert_or_assign(fname, dlsym(user_module_handle, fname));
-                                printf("%p\n", module_fn_map->find("mydiv") != module_fn_map->end() ? module_fn_map->find("mydiv")->second : nullptr);
+                                //printf("%p\n", module_fn_map->find("mydiv") != module_fn_map->end() ? module_fn_map->find("mydiv")->second : nullptr);
                             }
                             break;
                         case 'U':
@@ -193,6 +193,16 @@ int dll_main(int argc, char** argv, Context* cxt){
     return 0;
 }
 
+int launcher(int argc, char** argv){
+    std::string str = " ";
+    for (int i = 0; i < argc; i++){
+        str += argv[i];
+        str += " ";
+    }
+    str = std::string("python3 prompt.py ") + str;
+    return system(str.c_str());
+}
+
 extern "C" int __DLLEXPORT__ main(int argc, char** argv) {
    puts("running");
    Context* cxt = new Context();
@@ -201,14 +211,17 @@ extern "C" int __DLLEXPORT__ main(int argc, char** argv) {
 #ifdef THREADING
     auto tp = new ThreadPool();
     cxt->thread_pool = tp;
-    
 #endif
     
    const char* shmname;
    if (argc < 0)
-       return dll_main(argc, argv, cxt);
+        return dll_main(argc, argv, cxt);
    else if (argc <= 1)
-       return test_main();
+#ifdef __AQ__TESTING__
+        return test_main();
+#else
+        return launcher(argc, argv);
+#endif
    else
        shmname = argv[1];
    SharedMemory shm = SharedMemory(shmname);
@@ -241,18 +254,29 @@ extern "C" int __DLLEXPORT__ main(int argc, char** argv) {
    shm.FreeMemoryMap();
    return 0;
 }
+
 #include "utils.h"
+#include "table_ext_monetdb.hpp"
 int test_main()
 {
-    //vector_type<int> t;
-    //t = 1;
-    //t.emplace_back(2);
-    //print(t);
-    //return 0;
     Context* cxt = new Context();
     if (cxt->alt_server == 0)
         cxt->alt_server = new Server(cxt);
     Server* server = reinterpret_cast<Server*>(cxt->alt_server);
+ 
+
+    TableInfo<int, float> table("sibal");
+    int col0[] = { 1,2,3,4,5 };
+    float col1[] = { 5.f, 4.f, 3.f, 2.f, 1.f };
+    table.get_col<0>().initfrom(5, col0, "a");
+    table.get_col<1>().initfrom(5, col1, "b");
+    table.monetdb_append_table(server);
+    
+    server->exec("select * from sibal;");
+    auto aa = server->getCol(0);
+    auto bb = server->getCol(1);
+    printf("sibal: %p %p\n", aa, bb);
+
     const char* qs[]= {
         "SELECT MIN(3)-MAX(2);",
         "CREATE TABLE stocks(timestamp INT, price INT);",
@@ -302,6 +326,5 @@ int test_main()
     //static_assert(std::is_same_v<decltype(fill_integer_array<5, 1>()), std::integer_sequence<bool, 1,1,1,1,1>>, "");
     
     return 0;
-    std::unordered_map<int, int> a;
 }
 

server/table.cpp

@@ -6,14 +6,13 @@ template <>
 void print<__int128_t>(const __int128_t& v, const char* delimiter){
 	char s[41];
 	s[40] = 0;
-	std::cout<< get_int128str(v, s+40);
+	std::cout<< get_int128str(v, s+40)<< delimiter;
 }
 template <>
 void print<__uint128_t>(const __uint128_t&v, const char* delimiter){
 	char s[41];
 	s[40] = 0;
-    std::cout<< get_uint128str(v, s+40);
+    std::cout<< get_uint128str(v, s+40) << delimiter;
-
 }
 std::ostream& operator<<(std::ostream& os, __int128 & v)
 {
@@ -26,3 +25,8 @@ std::ostream& operator<<(std::ostream& os, __uint128_t & v)
 	return os;
 }
 #endif
+
+template <>
+void print<bool>(const bool&v, const char* delimiter){
+	std::cout<< (v?"true":"false") << delimiter;
+}

server/table.h

@@ -90,6 +90,15 @@ public:
 	}
 	template<typename T>
 	ColRef<T> scast();
+	
+	ColRef<_Ty>* rename(const char* name) {
+		this->name = name;
+		return this;
+	}
+
+	// defined in table_ext_monetdb.hpp
+	void* monetdb_get_col();
+	
 };
 
 template<typename _Ty>
@@ -297,9 +306,9 @@ struct TableInfo {
 		const auto& this_value = get<col>(*this)[i];
 		const auto& next = [&](auto &v) {
 			if constexpr (sizeof...(rem_cols) == 0)
-				func(args..., printi128(v));
+				func(args..., print_hook(v));
 			else
-				print2_impl<rem_cols...>(func, i, args ..., printi128(v));
+				print2_impl<rem_cols...>(func, i, args ..., print_hook(v));
 		};
 		if constexpr (is_vector_type<this_type>)
 			for (int j = 0; j < this_value.size; ++j)
@@ -368,6 +377,13 @@ struct TableInfo {
 		const vector_type<uint32_t>* __restrict view = nullptr, FILE* __restrict fp = nullptr) {
 		applyIntegerSequence<sizeof...(Types), applier>::apply(*this, sep, end, view, fp);
 	}
+
+	TableInfo< Types... >* rename(const char* name) {
+		this->name = name;
+		return this;
+	}
+	// defined in monetdb_conn.cpp
+	void monetdb_append_table(void* srv, const char* alt_name = nullptr);
 };
 
 
@@ -399,14 +415,14 @@ constexpr static inline bool is_vector(const vector_type<T>&) {
 template<class ...Types>
 TableInfo<Types...>::TableInfo(const char* name, uint32_t n_cols) : name(name), n_cols(n_cols) {
 	this->colrefs = (ColRef<void>*)malloc(sizeof(ColRef<void>) * n_cols);
-	for (int i = 0; i < n_cols; ++i) {
+	for (uint32_t i = 0; i < n_cols; ++i) {
 		this->colrefs[i].init();
 	}
 }
 template<class ...Types>
 TableInfo<Types...>::TableInfo(const char* name) : name(name), n_cols(sizeof...(Types)) {
 	this->colrefs = (ColRef<void>*)malloc(sizeof(ColRef<void>) * this->n_cols);
-	for (int i = 0; i < n_cols; ++i) {
+	for (uint32_t i = 0; i < n_cols; ++i) {
 		this->colrefs[i].init();
 	}
 }
@@ -573,6 +589,8 @@ void print<__int128_t>(const __int128_t& v, const char* delimiter);
 template <>
 void print<__uint128_t>(const __uint128_t&v, const char* delimiter);
 #endif
+template <>
+void print<bool>(const bool&v, const char* delimiter);
 
 template <class T>
 void inline print_impl(const T& v, const char* delimiter, const char* endline) {
@@ -591,4 +609,5 @@ print(const VT<T>& v, const char* delimiter = " ", const char* endline = "\n") {
 	print_impl(v, delimiter, endline);
 }
  
+
 #endif

server/table_ext_monetdb.hpp

@@ -0,0 +1,79 @@
+// TODO: Think of a way of decoupling table.h part and 
+//		monetdbe.h part to speed up compilation.
+
+#ifndef __TABLE_EXT_MONETDB_HPP__
+#define __TABLE_EXT_MONETDB_HPP__
+
+#include "table.h"
+#include "monetdb_conn.h"
+#include "monetdbe.h"
+
+inline constexpr monetdbe_types AQType_2_monetdbe[] = {
+		monetdbe_int32_t, monetdbe_float, monetdbe_str, monetdbe_double, monetdbe_int64_t,
+#ifdef HAVE_HGE
+		monetdbe_int128_t,
+#else 
+		monetdbe_int64_t,
+#endif
+		monetdbe_int16_t, monetdbe_date, monetdbe_time, monetdbe_int8_t,
+		monetdbe_int32_t, monetdbe_int64_t,
+#ifdef HAVE_HGE
+		monetdbe_int128_t,
+#else 
+		monetdbe_int64_t,
+#endif
+		monetdbe_int16_t, monetdbe_int8_t, monetdbe_bool, monetdbe_int64_t, monetdbe_int64_t, monetdbe_int64_t
+};
+
+template<class ...Ts>
+void TableInfo<Ts ...>::monetdb_append_table(void* srv, const char* alt_name) {
+	if (!alt_name){
+		alt_name = this->name;
+	}
+
+	monetdbe_column** monetdbe_cols = new monetdbe_column * [sizeof...(Ts)];
+	
+	uint32_t i = 0;
+	const auto get_col = [&monetdbe_cols, &i](auto v) {
+		monetdbe_cols[i++] = (monetdbe_column*)v->monetdb_get_col();
+	};
+	(get_col((ColRef<Ts>*)(colrefs + i)), ...);
+
+	std::string create_table_str = "CREATE TABLE ";
+	create_table_str += alt_name;
+	create_table_str += " (";
+	i = 0;
+	const auto get_name_type = [&i, *this]() {
+		return std::string(colrefs[i++].name) + ' ';
+	};
+	create_table_str += ((get_name_type() + types::SQL_Type[types::Types<Ts>::getType()] + ", ") + ... + std::string(""));
+	auto last_comma = create_table_str.find_last_of(',');
+	if (last_comma != static_cast<decltype(last_comma)>(-1)) {
+		create_table_str[last_comma] = ')';
+		Server* server = (Server*)srv;
+		server->exec(create_table_str.c_str());
+		if (!server->last_error) {
+			auto err = monetdbe_append(*((monetdbe_database*)server->server), "sys", alt_name, monetdbe_cols, sizeof...(Ts));
+			if (err)
+				puts(err);
+			return;
+		}
+	}
+	puts("Error! Empty table.");
+}
+
+
+template<class Type>
+void* ColRef<Type>::monetdb_get_col() {
+	auto aq_type = AQType_2_monetdbe[types::Types<Type>::getType()];
+	monetdbe_column* col = (monetdbe_column*)malloc(sizeof(monetdbe_column));
+
+	col->type = aq_type;
+	col->count = this->size;
+	col->data = this->container;
+	col->name = const_cast<char*>(this->name);
+
+	return col;
+}
+
+#endif

server/types.h

@@ -22,19 +22,23 @@ constexpr static bool is_vector_type = is_vector_impl<T>::value;
 namespace types {
 	enum Type_t {
 		AINT32, AFLOAT, ASTR, ADOUBLE, ALDOUBLE, AINT64, AINT128, AINT16, ADATE, ATIME, AINT8,
-		AUINT32, AUINT64, AUINT128, AUINT16, AUINT8, VECTOR, NONE, ERROR
+		AUINT32, AUINT64, AUINT128, AUINT16, AUINT8, ABOOL, VECTOR, NONE, ERROR
 	};
-	static constexpr const char* printf_str[] = { "%d", "%f", "%s", "%lf", "%Lf", "%ld", "%s", "%hi", "%s", "%s", "%c",
+	static constexpr const char* printf_str[] = { "%d", "%f", "%s", "%lf", "%Lf", "%ld", "%d", "%hi", "%s", "%s", "%c",
-		"%u", "%lu", "%s", "%hu", "%hhu", "Vector<%s>", "NULL", "ERROR" };
+		"%u", "%lu", "%s", "%hu", "%hhu", "%s", "%s", "Vector<%s>", "NULL", "ERROR" };
+	static constexpr const char* SQL_Type[] = { "INT", "REAL", "TEXT", "DOUBLE", "DOUBLE", "BIGINT", "HUGEINT", "SMALLINT", "DATE", "TIME", "TINYINT",
+		"INT", "BIGINT", "HUGEINT", "SMALLINT", "TINYINT", "BIGINT", "BOOL", "BIGINT", "NULL", "ERROR"};
+	
+	
 	// TODO: deal with data/time <=> str/uint conversion
 	struct date_t {
-		uint32_t val;
+		uint32_t val = 0;
 		date_t(const char* d) {
 		}
 		std::string toString() const;
 	};
 	struct time_t {
-		uint32_t val;
+		uint32_t val = 0;
 		time_t(const char* d) {
 		}
 		std::string toString() const;
@@ -69,6 +73,7 @@ namespace types {
 		f(unsigned long, AUINT64) \
 		f(unsigned short, AUINT16) \
 		f(unsigned char, AUINT8) \
+		f(bool, ABOOL) \
 		F_INT128(f)
 
 		inline constexpr static Type_t getType() {

server/vector_type.hpp

@@ -255,17 +255,17 @@ public:
 #define Ops(o, x) \
 	template<typename T>\
 	vector_type<typename types::Coercion<_Ty, T>::type> operator o (const vector_type<T>& r) const {\
-		[[likely]] if (r.size == size) {\
+		/*[[likely]] if (r.size == size) {*/\
 			return x(r);\
-		}\
+		/*}*/\
 	}
 
 #define Opseq(o, x) \
 	template<typename T>\
 	vector_type<typename types::Coercion<_Ty, T>::type> operator o##= (const vector_type<T>& r) {\
-		[[likely]] if (r.size == size) {\
+		/*[[likely]] if (r.size == size) {*/\
 			return x##eq(r);\
-		}\
+		/*}*/\
 	}
 
 #define _Make_Ops(M) \