update

Makefile

@@ -4,13 +4,11 @@ MonetDB_INC =
 Defines = 
 CC = $(CXX) -xc 
 CXXFLAGS = --std=c++2a
-ifeq ($(AQ_DEBUG), 1)
-	OPTFLAGS = -g3 #-static-libsan -fsanitize=address 
-	LINKFLAGS = 
-else
-	OPTFLAGS = -Ofast -DNDEBUG -fno-stack-protector 
-	LINKFLAGS = -flto -s
+
+ifdef AQ_LINKER
+	CXX += -fuse-ld=$(AQ_LINKER)
 endif
+
 SHAREDFLAGS = -shared  
 FPIC = -fPIC
 _COMPILER = $(shell $(CXX) --version | grep -q clang && echo clang|| echo gcc) 
@@ -47,7 +45,7 @@ else
 		LIBTOOL = gcc-ar rcs
 	endif
 endif
-LINKFLAGS += $(SEMANTIC_INTERPOSITION)
+LINKFLAGS = $(SEMANTIC_INTERPOSITION)
 
 ifeq ($(PCH), 1)
 	PCHFLAGS = -include server/pch.hpp
@@ -79,10 +77,10 @@ else
 			LIBTOOL = libtool -static -o
 		endif
 		ifneq ($(UNAME_M),arm64)
-			OPTFLAGS += -march=native
+			OPTFLAGS = -march=native
 		endif
 	else
-		OPTFLAGS += -march=native
+		OPTFLAGS = -march=native
 		MonetDB_LIB += $(AQ_MONETDB_LIB)
 		MonetDB_INC += $(AQ_MONETDB_INC)
 		MonetDB_INC += -I/usr/local/include/monetdb -I/usr/include/monetdb 
@@ -90,6 +88,15 @@ else
 	MonetDB_LIB += -lmonetdbe -lmonetdbsql -lbat
 endif
 
+
+ifeq ($(AQ_DEBUG), 1)
+	OPTFLAGS = -g3 #-static-libsan -fsanitize=address 
+#	LINKFLAGS = 
+else
+	OPTFLAGS += -Ofast -DNDEBUG -fno-stack-protector 
+	LINKFLAGS += -flto -s
+endif
+
 ifeq ($(THREADING),1)
 	LIBAQ_SRC += server/threading.cpp
 	LIBAQ_OBJ += threading.o
@@ -133,7 +140,7 @@ pch:
 	$(CXX) -x c++-header server/pch.hpp $(FPIC) $(CXXFLAGS)
 libaquery:
 	$(CXX) -c $(FPIC) $(PCHFLAGS) $(LIBAQ_SRC) $(OS_SUPPORT) $(CXXFLAGS) &&\
-	$(CC) -c server/monetdb_ext.c $(OPTFLAGS) $(MonetDB_INC) &&\
+	$(CC) -c $(FPIC) server/monetdb_ext.c $(OPTFLAGS) $(MonetDB_INC) &&\
 	$(LIBTOOL) libaquery.a $(LIBAQ_OBJ) &&\
 	$(RANLIB) libaquery.a
 

aquery_config.py

@@ -7,7 +7,7 @@ add_path_to_ldpath = True
 rebuild_backend = False
 run_backend = True
 have_hge = False
-cygroot = 'c:/msys64/usr/bin'
+cygroot = 'c:/mingw64/usr/bin'
 msbuildroot = ''
 os_platform = 'unknown'
 build_driver = 'Auto'
@@ -49,9 +49,9 @@ def init_config():
                 os_platform = 'cygwin'
         # deal with msys dependencies:
         if os_platform == 'win':
-            add_dll_dir(cygroot)  
             add_dll_dir(os.path.abspath('./msc-plugin'))
             add_dll_dir(os.path.abspath('./deps'))
+            add_dll_dir(cygroot)  
             if build_driver == 'Auto':
                 try:
                     import vswhere
@@ -72,6 +72,8 @@ def init_config():
                 print("Warning: Readline module not present")
             if build_driver == 'Auto':
                 build_driver = 'Makefile'
+            if os_platform == 'linux':
+                os.environ['PATH'] = os.environ['PATH'] + os.pathsep + '/usr/lib'
             if os_platform == 'cygwin':
                 add_dll_dir('./lib')
         __config_initialized__ = True

aquery_parser/parser.py

@@ -591,24 +591,12 @@ def parser(literal_string, ident):
         
         drop_trigger = (keyword("drop trigger") + var_name("name")) ("drop_trigger")
         
-        cache_options = Optional((
-            keyword("options").suppress()
-            + LB
-            + Dict(delimited_list(Group(
-                literal_string / (lambda tokens: tokens[0]["literal"])
-                + Optional(EQ)
-                + var_name
-            )))
-            + RB
-        )("options"))
-
         create_cache = (
             keyword("cache").suppress()
             + Optional(flag("lazy"))
             + TABLE
-            + var_name("name")
-            + cache_options
-            + Optional(AS + query("query"))
+            + FROM
+            + var_name("source") # AQuery, MonetDB, DuckDB ...
         )("cache")
 
         drop_table = (

build.py

@@ -122,23 +122,23 @@ class build_manager:
                 os.environ['AQ_DEBUG'] = ('0' if mgr.OptimizationLv != '0' else '1')
 
         def libaquery_a(self):
-            self.build_cmd = [['rm', 'libaquery.a'],['make', 'libaquery']]
+            self.build_cmd = [['rm', 'libaquery.a'],['make', 'libaquery', '-j']]
             return self.build()
         def pch(self):
-            self.build_cmd = [['rm', 'server/pch.hpp.gch'], ['make', 'pch']]
+            self.build_cmd = [['rm', 'server/pch.hpp.gch'], ['make', 'pch', '-j']]
             return self.build()
         def server(self):
             if self.mgr.StaticLib:
-                self.build_cmd = [['rm', '*.o'],['rm', 'server.so'], ['make', 'server_uselib']]
+                self.build_cmd = [['rm', '*.o'],['rm', 'server.so'], ['make', 'server_uselib', '-j']]
             else:
-                self.build_cmd = [['rm', 'server.so'], ['make', 'server.so']]
+                self.build_cmd = [['rm', 'server.so'], ['make', 'server.so', '-j']]
             return self.build()
             
         def snippet(self):
             if self.mgr.StaticLib:
-                self.build_cmd = [['make', 'snippet_uselib']]
+                self.build_cmd = [['make', 'snippet_uselib', '-j']]
             else:
-                self.build_cmd = [['rm', 'dll.so'], ['make', 'snippet']]
+                self.build_cmd = [['rm', 'dll.so'], ['make', 'snippet', '-j']]
             return self.build()
 
     class MSBuildDriver(DriverBase):

common/types.py

@@ -4,14 +4,22 @@ from typing import Dict, List
 from aquery_config import have_hge
 from common.utils import base62uuid, defval
 
+aquery_types: Dict[str, int] = {}
 type_table: Dict[str, "Types"] = {}
 
+with open('server/aquery_types.h', 'r') as f:
+    f.readline()
+    types = f.read()
+    import re
+    types = re.compile(r'(\s|\\)+').sub('', types).split(',')
+    aquery_types = { t : i for i, t in enumerate(types)}
+
 class Types:
     def init_any(self):
         self.name : str = 'Any'
         self.sqlname : str = 'Int'
         self.cname : str = 'void*'
-        self.ctype_name : str = "types::NONE"
+        self.ctype_name : str = "None"
         self.null_value = 0
         self.priority : int= 0
         self.cast_to_dict = dict()
@@ -31,7 +39,7 @@ class Types:
             self.name = name
             self.cname = defval(cname, name.lower() + '_t')
             self.sqlname = defval(sqlname, name.upper())
-            self.ctype_name = defval(ctype_name, f'types::A{name.upper()}') 
+            self.ctype_name = defval(ctype_name, f'A{name.upper()}') 
             self.null_value = defval(null_value, 0)
             self.cast_to_dict = defval(cast_to, dict())
             self.cast_from_dict = defval(cast_from, dict())
@@ -89,8 +97,8 @@ class TypeCollection:
 type_table = dict()
 AnyT = Types(-1)
 LazyT = Types(240, name = 'Lazy', cname = '', sqlname = '', ctype_name = '')
-DateT = Types(200, name = 'DATE', cname = 'types::date_t', sqlname = 'DATE', ctype_name = 'types::ADATE')
-TimeT = Types(201, name = 'TIME', cname = 'types::time_t', sqlname = 'TIME', ctype_name = 'types::ATIME')
+DateT = Types(200, name = 'DATE', cname = 'types::date_t', sqlname = 'DATE', ctype_name = 'ADATE')
+TimeT = Types(201, name = 'TIME', cname = 'types::time_t', sqlname = 'TIME', ctype_name = 'ATIME')
 TimeStampT = Types(202, name = 'TIMESTAMP', cname = 'types::timestamp_t', sqlname = 'TIMESTAMP', ctype_name = 'ATIMESTAMP')
 DoubleT = Types(17, name = 'double', cname='double', sqlname = 'DOUBLE', is_fp = True)
 LDoubleT = Types(18, name = 'long double', cname='long double', sqlname = 'LDOUBLE', is_fp = True)
@@ -102,15 +110,15 @@ LongT = Types(4, name = 'int64', sqlname = 'BIGINT', fp_type = DoubleT)
 BoolT = Types(0, name = 'bool', cname='bool', sqlname = 'BOOL', long_type=LongT, fp_type=FloatT)
 ByteT = Types(1, name = 'int8', sqlname = 'TINYINT', long_type=LongT, fp_type=FloatT)
 ShortT = Types(2, name = 'int16', sqlname='SMALLINT', long_type=LongT, fp_type=FloatT)
-IntT = Types(3, name = 'int', cname = 'int', long_type=LongT, ctype_name = 'types::AINT32', fp_type=FloatT)
+IntT = Types(3, name = 'int', cname = 'int', long_type=LongT, ctype_name = 'AINT32', fp_type=FloatT)
 ULongT = Types(8, name = 'uint64', sqlname = 'UINT64', fp_type=DoubleT)
 UIntT = Types(7, name = 'uint32', sqlname = 'UINT32', long_type=ULongT, fp_type=FloatT)
 UShortT = Types(6, name = 'uint16', sqlname = 'UINT16', long_type=ULongT, fp_type=FloatT)
 UByteT = Types(5, name = 'uint8', sqlname = 'UINT8', long_type=ULongT, fp_type=FloatT)
-StrT = Types(200, name = 'str', cname = 'string_view', sqlname='TEXT', ctype_name = 'types::ASTR')
-TextT = Types(200, name = 'text', cname = 'string_view', sqlname='TEXT', ctype_name = 'types::ASTR')
-VarcharT = Types(200, name = 'varchar', cname = 'string_view', sqlname='VARCHAR', ctype_name = 'types::ASTR')
-VoidT = Types(200, name = 'void', cname = 'void', sqlname='Null', ctype_name = 'types::None')
+StrT = Types(200, name = 'str', cname = 'string_view', sqlname='TEXT', ctype_name = 'ASTR')
+TextT = Types(200, name = 'text', cname = 'string_view', sqlname='TEXT', ctype_name = 'ASTR')
+VarcharT = Types(200, name = 'varchar', cname = 'string_view', sqlname='VARCHAR', ctype_name = 'ASTR')
+VoidT = Types(200, name = 'void', cname = 'void', sqlname='Null', ctype_name = 'None')
 
 class VectorT(Types):
     def __init__(self, inner_type : Types, vector_type:str = 'vector_type'):

common/utils.py

@@ -130,16 +130,21 @@ class _Counter:
 import re
 
 ws = re.compile(r'\s+')
-import os
+def encode_integral(val : int):
+    return val.to_bytes(4, 'little').decode('latin-1')
 
+import os
 
 def add_dll_dir(dll: str):
     import sys
-    if sys.version_info.major >= 3 and sys.version_info.minor >7 and os.name == 'nt':
-        os.add_dll_directory(dll)
-    else:
-        os.environ['PATH'] = os.path.abspath(dll) + os.pathsep + os.environ['PATH']
-        
+    try:
+        if sys.version_info.major >= 3 and sys.version_info.minor >7 and os.name == 'nt':
+            os.add_dll_directory(dll)
+        else:
+            os.environ['PATH'] = os.path.abspath(dll) + os.pathsep + os.environ['PATH']
+    except FileNotFoundError:
+        print(f"Error: path not found")
+
 nullstream = open(os.devnull, 'w')
 
 

data/h2o

@@ -1 +0,0 @@
-/Users/bill/Downloads/db-benchmark-master/data

docs/paper

@@ -1 +1 @@
-Subproject commit c8d27fe131e952ca4a30b0de27ff58b28a9990ad
+Subproject commit 88d64456343506aae785a938c295864572fe6c0b

engine/ast.py

@@ -1746,13 +1746,35 @@ class user_module_function(OperatorBase):
         # builtin_operators[name] = self
         udf.try_init_udf(context)
         
+class cache(ast_node):
+    name = 'cache'
+    first_order = name
+    def init(self, node):
+        source = node['cache']['source']
+        # lazy = node['cache']['lazy']
+        lazy = 0
+        try:
+            tbl : TableInfo = self.context.tables_byname[source]
+        except KeyError:
+            raise ValueError(f'Cannot find table {source}.')
+        from common.utils import encode_integral
+        
+        tbl.cached = True
+        schema_string = encode_integral(len(tbl.columns))
+        for t in tbl.columns:
+            schema_string += t.name + '\0' + \
+            encode_integral(aquery_types[t.type.ctype_name])
+            
+        from common.utils import send_to_server
+        send_to_server(f'C{source}\0{"l" if lazy else "e"}\0{schema_string}\0')
+
 def include(objs):
     import inspect
     for _, cls in inspect.getmembers(objs):
         if inspect.isclass(cls) and issubclass(cls, ast_node) and type(cls.first_order) is str:
             ast_node.types[cls.first_order] = cls
-            
-            
+
+
 import sys
 
 include(sys.modules[__name__])

engine/storage.py

@@ -73,6 +73,7 @@ class TableInfo:
         self.columns : List[ColRef] = []
         self.triggers : Set[create_trigger] = set()
         self.cxt = cxt
+        self.cached = False
         # keep track of temp vars
         self.rec = None 
         self.add_cols(cols)
@@ -186,7 +187,7 @@ class Context:
         self.force_compiled = False
         self.use_gc = compile_use_gc 
         self.system_state: Optional[PromptState] = state 
-
+        self.use_cached_tables = True
         # self.new() called everytime new query batch is started
 
     def get_scan_var(self):
@@ -267,9 +268,10 @@ class Context:
         self.finalize_query()
     
     def direct_output(self, limit = -1, sep = ' ', end = '\n'):
+        from common.utils import encode_integral
         if type(limit) is not int or limit > 2**32 - 1 or limit < 0:
             limit = 2**32 - 1
-        limit = limit.to_bytes(4, 'little').decode('latin-1')
+        limit = encode_integral(limit)
         self.queries.append(
             'O' + limit + sep + end)
         

msvs-py/msvs-py.pyproj

@@ -11,7 +11,7 @@
     <Name>msvs-py</Name>
     <RootNamespace>msvs-py</RootNamespace>
     <IsWindowsApplication>False</IsWindowsApplication>
-    <InterpreterId>Global|PythonCore|3.10</InterpreterId>
+    <InterpreterId>Global|PythonCore|3.9</InterpreterId>
     <LaunchProvider>Standard Python launcher</LaunchProvider>
     <EnableNativeCodeDebugging>True</EnableNativeCodeDebugging>
     <Environment>AQ_DEBUG=1</Environment>
@@ -32,7 +32,6 @@
   <ItemGroup>
     <Folder Include="..\aquery_parser\" />
     <Folder Include="..\engine\" />
-    <Folder Include="parser\" />
   </ItemGroup>
   <ItemGroup>
     <Compile Include="..\aquery_parser\keywords.py" />

server/DataSource_conn.h

@@ -44,6 +44,7 @@ struct DataSource {
     virtual void connect(Context* cxt) = 0;
     virtual void exec(const char* q) = 0;
     virtual void* getCol(int col_idx, int type) = 0;
+    virtual void getDSTable(const char* name, void* tbl) = 0;
     // virtual long long getFirstElement() = 0;
     virtual void close() = 0;
     virtual bool haserror() = 0;

server/aquery_types.h

@@ -0,0 +1,5 @@
+#define __AQUERY_TYPES__ \
+    AINT32, AFLOAT, ASTR, ADOUBLE, ALDOUBLE, AINT64, AINT128, \
+    AINT16, ADATE, ATIME, AINT8, AUINT32, AUINT64, AUINT128, \
+    AUINT16, AUINT8, ABOOL, VECTOR, ATIMESTAMP, ACHAR, ASV, \
+    NONE, ERROR

server/duckdb_conn.cpp

@@ -1,6 +1,6 @@
 #include "pch_msc.hpp"
 #include "duckdb_conn.h"
-#include "../deps/duckdb.hpp"
+#include "duckdb.hpp"
 #include "libaquery.h"
 #include "types.h"
 #include <cstdio>
@@ -73,6 +73,10 @@ void* DuckdbServer::getCol(int col_idx, int ty) {
 	}
 }
 
+void DuckdbServer::getDSTable(const char* name, void* tbl) { 
+	// not implemented. 
+	puts("NOT IMPLEMENTED ERROR: DuckdbServer::getDSTable");
+}
 bool DuckdbServer::haserror() {
 	if (last_error) {
 		puts(last_error);

server/duckdb_conn.h

@@ -6,7 +6,8 @@ struct DuckdbServer : DataSource {
     explicit DuckdbServer(Context* cxt);
     void connect(Context* cxt);
     void exec(const char* q);
-    void* getCol(int col_idx, int type);
+    void* getCol(int col_idx, int type) override;
+    void getDSTable(const char* name, void* tbl) override;
     long long getFirstElement();
     void close();
     bool haserror();

server/gc.h

@@ -104,6 +104,7 @@ public:
 	} 
     constexpr static void(*_free) (void*) = free;
 };
+
 #else
 class GC {
 public:

server/hasher.h

@@ -7,6 +7,7 @@
 #include "types.h"
 // #include "robin_hood.h"
 #include "unordered_dense.h"
+
 template<typename Key, typename Val>
 using aq_map = ankerl::unordered_dense::map<Key, Val>;
 
@@ -137,3 +138,66 @@ namespace ankerl::unordered_dense{
 	struct hash<std::tuple<Types...>> : public hasher<Types...>{ };
 }
 
+template <
+	typename ValueType = bool, 
+	int PerfectHashingThreshold = 12
+>
+struct PerfectHashTable {
+	// static int m_PerfectHashingThreshold = 12;
+	using key_t = std::conditional_t<PerfectHashingThreshold <= 8, uint8_t, 
+					std::conditional_t<PerfectHashingThreshold <= 16, uint16_t, 
+					std::conditional_t<PerfectHashingThreshold <= 32, uint32_t, 
+					uint64_t
+		>>>;
+
+	int n_cols, n_rows = 0;
+	// char bits[32];
+	ValueType table[1 << PerfectHashingThreshold];
+	// PerfectHashTable(int n_cols, char* bits) {
+	// 	this->n_cols = n_cols;
+	// 	memcpy(this->bits, bits, 32);
+	// }
+	// template<typename ... Types, template <typename> class VT> 
+	// PerfectHashTable(VT<Types> ... args) {
+		
+	// }
+	template <typename ... Types, template <typename> class VT>
+	void construct(VT<Types>&... args) {
+		((this->n_cols = args.size), ...);
+		static_assert(
+			(sizeof...(Types) < PerfectHashingThreshold) &&
+			//(sizeof(Types) + ...) < PerfectHashingThreshold && 
+			(std::is_integral_v<Types> && ...),
+			"Types must be integral and less than 12 wide in total."
+			);
+		// this should be an attrib of VT.
+		key_t* // this better be automatically determined by Threshould 
+			hash_values = static_cast<key_t*>(
+					calloc(this->n_cols, sizeof(key_t))
+				);
+		//new short[this->n_cols] {0}; // use calloc/delete
+		auto get_hash = [&hash_values](auto& arg, int idx) {
+			uint32_t i = 0;
+			if(idx > 0)
+				for (auto& a : arg) {
+					hash_values[i] =
+						(hash_values[i] << arg.stats.bits) +
+						(a - arg.stats.minima);
+					++i;
+				}
+			else 
+				for (auto& a : arg) {
+					hash_values[i] = a - arg.stats.minima;
+					++i;
+				}
+		};
+		int idx = 0;
+		(get_hash(args, idx++), ...);
+		for (uint32_t i = 0; i < this->n_cols; ++i) {
+			this->table[hash_values[i]] = true;
+			// problem: random memory access
+		}
+		// delete[] hash_values;
+		free(hash_values);
+	}
+};

server/libaquery.h

@@ -235,6 +235,13 @@ inline _This_Type* AQ_DupObject(_This_Type* __val) {
 	return ret;
 }
 
+inline char* AQ_DupString(const char* __val) {
+	auto __len = strlen(__val) + 1;
+	auto ret = (char*)malloc(__len);
+	memcpy(ret, __val, __len);
+	return ret;
+}
+
 #ifdef __USE_STD_SEMAPHORE__
 	#include <semaphore>
 	class A_Semaphore {

server/monetdb_conn.cpp

@@ -267,6 +267,20 @@ long long MonetdbServer::getFirstElement() {
     return 0;
 }
 
+void MonetdbServer::getDSTable(const char* name, void* tbl) {
+    TableInfo<void> *table = static_cast<TableInfo<void>*>(tbl);
+    void*** cols = static_cast<void***>(alloca(table->n_cols * sizeof(void**)));
+    printf("\tncols: %d\n", table->n_cols);
+    for (int i = 0; i < table->n_cols; ++i) {
+        cols[i] = static_cast<void**>(
+            static_cast<void*>(
+                &(table->colrefs[i].container)
+            )
+        );
+    }
+    monetdbe_get_cols(*(void**)(this->server), name, cols, table->n_cols);
+} 
+
 MonetdbServer::~MonetdbServer(){
     close();
 }

server/monetdb_conn.h

@@ -7,6 +7,7 @@ struct MonetdbServer : DataSource {
     void connect(Context* cxt) override;
     void exec(const char* q) override;
     void *getCol(int col_idx, int) override;
+    void getDSTable(const char* name, void* tbl) override;
     long long getFirstElement();
     void close() override;
     bool haserror() override;
@@ -24,8 +25,12 @@ struct monetdbe_table_data{
 };
 
 extern "C" size_t 
-monetdbe_get_size(void* dbhdl, const char *table_name);
+monetdbe_get_size(void* dbhdl, const char *table_name, void*);
 
 extern "C" void* 
 monetdbe_get_col(void* dbhdl, const char *table_name, uint32_t col_id);
+
+extern "C" void
+monetdbe_get_cols(void* dbhdl, const char* table_name, void*** cols, int i);
+
 #endif

server/monetdb_ext.c

@@ -80,6 +80,19 @@ monetdbe_get_size(monetdbe_database dbhdl, const char *table_name)
 	return sz; 
 }
 
+void *
+monetdbe_list_fetch(list *l, int pos)
+{
+	node *n = NULL;
+	int i;
+
+	for (n = l->h, i=0; n && i<pos; n = n->next, i++)
+		;
+	if (n)
+		return n->data;
+	return NULL;
+}
+
 void* 
 monetdbe_get_col(monetdbe_database dbhdl, const char *table_name, uint32_t col_id) {
     monetdbe_database_internal* hdl = (monetdbe_database_internal*)dbhdl;
@@ -95,3 +108,24 @@ monetdbe_get_col(monetdbe_database dbhdl, const char *table_name, uint32_t col_i
 	//mvc_cancel_session(m); 
     return iter.base;
 }
+
+void monetdbe_get_cols(
+	monetdbe_database dbhdl, 
+	const char* table_name, 
+	void*** cols, 
+	int i
+) {
+	monetdbe_database_internal* hdl = (monetdbe_database_internal*)dbhdl;
+	backend* be = ((backend *)(((monetdbe_database_internal*)dbhdl)->c->sqlcontext));
+	mvc *m = be->mvc;
+	sql_table *t = find_table_or_view_on_scope(m, NULL, "sys", table_name, "CATALOG", false);
+	if (!i || !t) return;
+	node *n = t->columns->l->h;
+	sqlstore* store = m->store;
+	while(n && i-- > 0) {
+		BAT *b = store->storage_api.bind_col(m->session->tr, n->data, QUICK);
+		BATiter iter = bat_iterator(b);
+		*(cols++) = iter.base;
+		n = n->next;
+	}
+}

server/server.cpp

@@ -549,6 +549,37 @@ start:
                             }
                         }
                         break;
+                        case 'C': //Caching 
+                        {
+                            char* cached_table = n_recvd[i] + 1;
+                            char *lazy = (cached_table + 1);
+                            cached_table = AQ_DupString(cached_table);
+                            while(*lazy++);
+                            // get schema
+                            int* n_cols = reinterpret_cast<int *>(lazy + 2);
+                            char* col_schema = reinterpret_cast<char *>(n_cols + 1);
+                            TableInfo<void> *tbl = new TableInfo<void>;
+                            tbl->name = cached_table;//AQ_DupString(cached_table);
+                            tbl->n_cols = *n_cols;
+                            
+                            for (int i = 0; i < *n_cols; ++i) {
+                                char* col_name = col_schema;
+                                char* mem_coltype = col_name + 1;
+                                while(*mem_coltype++);
+                                int coltype = *(reinterpret_cast<int*>(mem_coltype));
+                                // 
+                                tbl->colrefs[i].name = AQ_DupString(col_name);
+                                tbl->colrefs[i].ty = static_cast<types::Type_t>(coltype);    
+                            }
+                            
+                            server->getDSTable(cached_table, tbl);
+                            // server->exec( (
+                            //     std::string("SELECT * FROM ") + cached_table + std::string(";")
+                            // ).c_str() );
+                            // server->getCol()
+                            // free(cached_table);
+                            break;
+                        }
                         }
                     }
                     

server/table.h

@@ -21,6 +21,8 @@ template <>
 class vector_type<void>;
 
 #ifdef _MSC_VER
+#include <intrin.h>
+#define __builtin_popcount __popcnt
 namespace types {
 	enum Type_t;
 	template <typename T>
@@ -54,6 +56,13 @@ std::ostream& operator<<(std::ostream& os, uint8_t& v);
 std::ostream& operator<<(std::ostream& os, types::date_t& v);
 std::ostream& operator<<(std::ostream& os, types::time_t& v);
 std::ostream& operator<<(std::ostream& os, types::timestamp_t& v);
+
+template<class T>
+struct TableStats {
+	T minima = 0;
+	unsigned char bits = 255;
+};
+
 template<typename _Ty>
 class ColView;
 template<typename _Ty>
@@ -63,6 +72,23 @@ public:
 	typedef ColRef<_Ty> Decayed_t;
 	const char* name;
 	types::Type_t ty = types::Type_t::ERROR;
+	TableStats<_Ty> stats;
+	bool populate_stats() {
+		if constexpr (std::is_integral_v<_Ty>) {
+			if (stats.bits <= 128) return true;
+			stats.minima = std::numeric_limits<_Ty>::max();
+			_Ty maxima = std::numeric_limits<_Ty>::min();
+			for (uint32_t i = 0; i < this->size; ++i) {
+				if (this->container[i] < stats.minima) 
+					stats.minima = this->container[i];
+				else if (this->container[i] > maxima) 
+					maxima = this->container[i];
+			}
+			stats.bits = ceil(log2(maxima - stats.minima));
+			return true;
+		}
+		return false;
+	}
 	ColRef(const ColRef<_Ty>& vt) : vector_type<_Ty>(vt) {}
 	ColRef(ColRef<_Ty>&& vt) : vector_type<_Ty>(std::move(vt)) {}
 	ColRef() : vector_type<_Ty>(0), name("") {}

server/types.h

@@ -6,6 +6,7 @@
 #include <string_view>
 #include <string>
 #include <utility>
+#include "aquery_types.h"
 using std::size_t;
 
 #if  defined(__SIZEOF_INT128__) and not defined(_WIN32)
@@ -68,8 +69,7 @@ constexpr bool aqis_same<T1, T2> = aqis_same_impl<T1, T2>::value;
 
 namespace types {
 	enum Type_t {
-		AINT32, AFLOAT, ASTR, ADOUBLE, ALDOUBLE, AINT64, AINT128, AINT16, ADATE, ATIME, AINT8,
-		AUINT32, AUINT64, AUINT128, AUINT16, AUINT8, ABOOL, VECTOR, ATIMESTAMP, ACHAR, ASV, NONE, ERROR
+		__AQUERY_TYPES__
 	};
 	static constexpr const char* printf_str[] = { "%d", "%f", "%s", "%lf", "%Lf", "%ld", "%s", "%hi", "%s", "%s", "%hhd",
 		"%u", "%lu", "%s", "%hu", "%hhu", "%s", "Vector<%s>", "%s", "%c", "%s", "NULL", "ERROR" };