initial support for pack expr

Makefile

@@ -3,7 +3,11 @@ MonetDB_LIB =
 MonetDB_INC = 
 Threading = 
 CXXFLAGS = --std=c++1z
-OPTFLAGS = -O3 -DNDEBUG 
+ifeq ($(AQ_DEBUG), 1)
+	OPTFLAGS = -g3 
+else
+	OPTFLAGS = -O3 -DNDEBUG -fno-stack-protector
+endif
 LINKFLAGS = -flto # + $(AQ_LINK_FLAG)
 SHAREDFLAGS = -shared  
 FPIC = -fPIC

README.md

@@ -8,7 +8,7 @@ AQuery++ Database is a cross-platform, In-Memory Column-Store Database that inco
    - See installation instructions from [docker.com](https://www.docker.com). Run **docker desktop** to start docker engine.
    - In AQuery root directory, type `make docker` to build the docker image from scratch. 
    - For Arm-based Mac users, you would have to build and run the **x86_64** docker image because MonetDB doesn't offer official binaries for arm64 Linux. (Run `docker buildx build --platform=linux/amd64 -t aquery .` instead of `make docker`)
-   - Finally run the image in **interactive** mode (`docker run -it aquery --name aquery`) 
+   - Finally run the image in **interactive** mode (`docker run --name aquery -it aquery`) 
    - When you need to access the container again run `docker start -ai aquery` 
    - If there is a need to access the system shell within AQuery, type `dbg` to activate python interpreter and type `os.system('sh')` to launch a shell.
 

aquery_config.py

@@ -2,7 +2,7 @@
 
 ## GLOBAL CONFIGURATION FLAGS
 
-version_string = '0.4.8a'
+version_string = '0.4.9a'
 add_path_to_ldpath = True
 rebuild_backend = False
 run_backend = True

build.py

@@ -109,7 +109,9 @@ class build_manager:
                 os.environ['CXX'] = mgr.cxx if mgr.cxx else 'c++'
             else:
                 mgr.cxx = os.environ['CXX']
-        
+            if 'AQ_DEBUG' not in os.environ:
+                os.environ['AQ_DEBUG'] = '0' if mgr.OptimizationLv else '1'
+                
         def libaquery_a(self):
             self.build_cmd = [['rm', 'libaquery.a'],['make', 'libaquery.a']]
             return self.build()

cims.sh

@@ -1,4 +1,4 @@
-#!/usr/bash
+#!/bin/bash
 echo  "Don't execute this script if it's not on CIMS servers."
 echo "run this script with source command. e.g. source ./cims.sh or . ./cims.sh"
 module load gcc-11.2

engine/types.py

@@ -226,7 +226,17 @@ def long_return(*_ : Types) -> Types:
     return LongT.long_type
 def lfp_return(*_ : Types) -> Types:
     return LongT.fp_type
-
+def pack_return(*args : Types) -> Types:
+    if len(args) == 0:
+        raise ValueError('0 arguments in pack expression')
+    inner_ty = args[0]
+    for a in args:
+        if a != inner_ty:
+            raise ValueError('pack expression with different types')
+    if isinstance(inner_ty, VectorT):
+        print('warning: packing vector types')
+        inner_ty = inner_ty.inner_type
+    return VectorT(args[0])
 def as_is (t: Types) -> Types:
     return t
 
@@ -269,6 +279,15 @@ def windowed_fn_behavor(op: OperatorBase, c_code, *x):
     else: 
         name = op.cname if len(x) == 1 else op.cname[:-1] + 'w'
         return f'{name}({", ".join([f"{xx}" for xx in x])})'
+
+def pack_behavior(op: OperatorBase, c_code, *x):
+    if len(x) == 0:
+        raise ValueError('0 arguments in pack expression')
+        
+    if not c_code:
+        return f'{op.sqlname}({", ".join([f"{xx}" for xx in x])})'
+    else:
+        return f'decltype({x[0]})::pack(len(x) + 1, {", ".join([f"{xx}.s()" for xx in x])})'
     
 # arithmetic 
 opadd = OperatorBase('add', 2, auto_extension, cname = '+', sqlname = '+', call = binary_op_behavior)
@@ -307,6 +326,7 @@ fnmins = OperatorBase('mins', [1, 2], ty_clamp(as_is, -1), cname = 'mins', sqlna
 fnsums = OperatorBase('sums', [1, 2], ext(ty_clamp(auto_extension, -1)), cname = 'sums', sqlname = 'SUMS', call = windowed_fn_behavor)
 fnavgs = OperatorBase('avgs', [1, 2], fp(ext(ty_clamp(auto_extension, -1))), cname = 'avgs', sqlname = 'AVGS', call = windowed_fn_behavor)
 fncnt = OperatorBase('count', 1, int_return, cname = 'count', sqlname = 'COUNT', call = count_behavior)
+fnpack = OperatorBase('pack', -1, pack_return, cname = 'pack', sqlname = 'PACK', call = pack_behavior)
 # special
 def is_null_call_behavior(op:OperatorBase, c_code : bool, x : str):
     if c_code : 
@@ -328,12 +348,16 @@ fnpow = OperatorBase('pow', 2, lambda *_ : DoubleT, cname = 'pow', sqlname = 'PO
 def _op_make_dict(*items : OperatorBase):
     return { i.name: i for i in items}
 builtin_binary_arith = _op_make_dict(opadd, opdiv, opmul, opsub, opmod)
-builtin_binary_logical = _op_make_dict(opand, opor, opxor, opgt, oplt, opge, oplte, opneq, opeq)
+builtin_binary_logical = _op_make_dict(opand, opor, opxor, opgt, oplt, 
+                                       opge, oplte, opneq, opeq)
 builtin_unary_logical = _op_make_dict(opnot)
 builtin_unary_arith = _op_make_dict(opneg)
 builtin_unary_special = _op_make_dict(spnull, opdistinct)
 builtin_cstdlib = _op_make_dict(fnsqrt, fnlog, fnsin, fncos, fntan, fnpow)
-builtin_func = _op_make_dict(fnmax, fnmin, fnsum, fnavg, fnmaxs, fnmins, fndeltas, fnratios, fnlast, fnfirst, fnsums, fnavgs, fncnt)
+builtin_func = _op_make_dict(fnmax, fnmin, fnsum, fnavg, fnmaxs, 
+                             fnmins, fndeltas, fnratios, fnlast,
+                             fnfirst, fnsums, fnavgs, fncnt, 
+                             fnpack)
 user_module_func = {}
 builtin_operators : Dict[str, OperatorBase] = {**builtin_binary_arith, **builtin_binary_logical, 
     **builtin_unary_arith, **builtin_unary_logical, **builtin_unary_special, **builtin_func, **builtin_cstdlib, 

prompt.py

@@ -497,6 +497,23 @@ def prompt(running = lambda:True, next = lambda:input('> '), state = None):
             elif q == 'exit' or q == 'exit()' or q == 'quit' or q == 'quit()' or q == '\\q':
                 rm(state)
                 exit()
+            elif q.startswith('sh'):
+                from distutils.spawn import find_executable
+                qs = re.split(r'[ \t]', q)
+                shells = ('zsh', 'bash', 'sh', 'fish', 'cmd', 'pwsh', 'powershell', 'csh', 'tcsh', 'ksh')
+                shell_path = ''
+                if len(qs) > 1 and qs[1] in shells:
+                    shell_path = find_executable(qs[1])
+                    if shell_path:
+                        os.system(shell_path)
+                else:
+                    for sh in shells:
+                        shell_path = find_executable(sh)
+                        if shell_path:
+                            os.system(shell_path)
+                            break
+                
+                continue
             elif q == 'r': # build and run
                 if state.server_mode == RunType.Threaded:
                     qs = [ctypes.c_char_p(bytes(q, 'utf-8')) for q in cxt.queries if len(q)]

server/table.h

@@ -8,6 +8,7 @@
 #include <iostream>
 #include <string>
 #include <algorithm>
+#include <cstdarg>
 #include "io.h"
 #include "hasher.h"
 
@@ -26,6 +27,13 @@ namespace types {
 	struct Coercion;
 }
 #endif
+struct ColRef_cstorage {
+	void* container;
+	unsigned int size, capacity;
+	const char* name;
+	int ty; // what if enum is not int?
+};
+
 template <template <class...> class VT, class T>
 std::ostream& operator<<(std::ostream& os, const VT<T>& v)
 {
@@ -80,6 +88,10 @@ public:
 		vt.capacity = 0; // rvalue's 
 		return *this;
 	}
+	template <class T>
+	ColRef<_Ty> getRef(){
+		return ColRef<_Ty>(this->size, this->container, this->name);
+	}
 	ColRef(const char* name, types::Type_t ty) : name(name), ty(ty) {}
 	using vector_type<_Ty>::operator[];
 	//using vector_type<_Ty>::operator=;
@@ -127,7 +139,37 @@ public:
 		this->name = name;
 		return this;
 	}
-
+	static ColRef<vector_type<_Ty>> pack(uint32_t cnt, ...){
+		va_list cols;
+		va_start(cols, cnt);
+		ColRef<_Ty> *col = (ColRef<_Ty>*)malloc(sizeof(ColRef<_Ty>) * cnt);
+		for (uint32_t i = 0; i < cnt; ++i){
+			ColRef_cstorage tmp = va_arg(cols, ColRef_cstorage);
+			memcpy(&col[i], &tmp, sizeof(ColRef_cstorage));
+			col[i].capacity = 0;
+		}
+		va_end(cols);
+		if(cnt > 0){
+			auto sz = col[0].size;
+			ColRef<vector_type<_Ty>> ret(sz);
+			for(uint32_t i = 0; i < sz; ++i){
+				decltype(auto) v = ret[i];
+				v.size = cnt;
+				v.capacity = cnt;
+				v.container = (_Ty*)malloc(sizeof(_Ty) * cnt);
+				for(uint32_t j = 0; j < cnt; ++j) {
+					v.container[j] = col[j][i];
+				}
+			}
+			free(col);
+			return ret;
+		}
+		else 
+			return ColRef<vector_type<_Ty>>();
+	}
+	ColRef_cstorage s() {
+		return *(ColRef_cstorage*)(this);
+	}
 	// defined in table_ext_monetdb.hpp
 	void* monetdb_get_col(void** gc_vecs, uint32_t& cnt);
 	

server/types.h

@@ -22,6 +22,10 @@ struct is_vector_impl : std::false_type {};
 template <class T>
 constexpr static bool is_vector_type = is_vector_impl<T>::value;
 #define Cond(c, x, y) typename std::conditional<c, x, y>::type
+template <class T>
+constexpr size_t aq_szof = sizeof(T);
+template <>
+inline constexpr size_t aq_szof<void> = 0;
 template <class T1, class T2>
 struct aqis_same_impl {
 	constexpr static bool value = 
@@ -30,7 +34,7 @@ struct aqis_same_impl {
 			Cond(
 				std::is_floating_point_v<T1> == std::is_floating_point_v<T2>,
 				Cond(
-					sizeof(T1) == sizeof(T2),
+					aq_szof<T1> == aq_szof<T2>, // deal with sizeof(void)
 					std::true_type,
 					std::false_type
 				),
@@ -40,8 +44,12 @@ struct aqis_same_impl {
 		>::value;
 };
 
+template <class T1, class T2, class ...Ts>
+constexpr bool aqis_same = aqis_same_impl<T1, T2>::value &&
+aqis_same<T2, Ts...>;
+
 template <class T1, class T2>
-constexpr bool aqis_same = aqis_same_impl<T1, T2>::value;
+constexpr bool aqis_same<T1, T2> = aqis_same_impl<T1, T2>::value;
 namespace types {
 	enum Type_t {
 		AINT32, AFLOAT, ASTR, ADOUBLE, ALDOUBLE, AINT64, AINT128, AINT16, ADATE, ATIME, AINT8,
@@ -165,15 +173,9 @@ namespace types {
 #define ATypeSize(t, at) sizeof(t),
 	static constexpr size_t AType_sizes[] = { ConnectTypes(ATypeSize) 1 };
 #define Comp(o) (sizeof(T1) o sizeof(T2))
-#define Same(x, y) (std::is_same_v<x, y>)
+#define Same(x, y) (aqis_same<x, y>)
 #define __U(x) std::is_unsigned<x>::value
 #define Fp(x) std::is_floating_point<x>::value
-	template <class T1, class T2>
-	struct Coercion {
-		using t1 = Cond(Comp(<= ), Cond(Comp(== ), Cond(Fp(T1), T1, Cond(Fp(T2), T2, Cond(__U(T1), T2, T1))), T2), T1);
-		using t2 = Cond(Same(T1, T2), T1, Cond(Same(T1, const char*) || Same(T2, const char*), const char*, void));
-		using type = Cond(Same(t2, void), Cond(Same(T1, date_t) && Same(T2, time_t) || Same(T1, time_t) && Same(T2, time_t), void, t1), t2);
-	};
 #define __Eq(x) (sizeof(T) == sizeof(x))
 	template<class T>
 	struct GetFPTypeImpl {
@@ -210,6 +212,50 @@ namespace types {
 	};
 	template<class T>
 	using GetLongerType = typename GetLongerTypeImpl<typename std::decay<T>::type>::type;
+#ifdef __AQ__HAS__INT128__
+    #define __AQ_HELPER_INT128__(x, y) x
+#else
+    #define __AQ_HELPER_INT128__(x, y) y
+#endif
+	template <class T>
+	struct GetSignedType_impl{
+		using type = Cond(Same(T, unsigned char), char,
+			Cond(Same(T, unsigned short), short,
+				Cond(Same(T, unsigned int), int,
+					Cond(Same(T, unsigned long), long,
+// #ifdef __AQ__HAS__INT128__				
+						__AQ_HELPER_INT128__(
+							Cond(Same(T, unsigned long long), long long,
+								Cond(Same(T, unsigned __int128), __int128_t,
+									T
+								)
+							), 
+// #else
+							T
+						)
+// #endif
+					)
+				)
+			)
+		);
+	};
+	template <class T>
+	using GetSignedType = typename GetSignedType_impl<T>::type;
+
+	template <class T1, class T2, class ...Ts>
+	struct Coercion{
+		using type = typename Coercion<T1, typename Coercion<T2, Ts...>::type>::type;
+	};
+	
+	template <class T1, class T2>
+	struct Coercion<T1, T2> {
+		using t0 = Cond(Comp(<= ), Cond(Comp(== ), Cond(Fp(T1), T1, Cond(Fp(T2), T2, Cond(__U(T1), T2, T1))), T2), T1);
+		using t1 = Cond(Fp(T1)||Fp(T2), GetFPType<t0>, Cond(!(__U(T1) && __U(T2)), GetSignedType<t0>,t0));
+		using t2 = Cond(Same(T1, T2), T1, Cond(Same(T1, const char*) || Same(T2, const char*), const char*, void));
+		using type = Cond(Same(t2, void), Cond(Same(T1, date_t) && Same(T2, time_t) || Same(T1, time_t) && Same(T2, time_t), void, t1), t2);
+	};
+	
+
 }
 
 

server/vector_type.hpp

@@ -17,10 +17,11 @@
 #include "types.h"
 
 #pragma pack(push, 1)
-template <typename _Ty>
-class slim_vector {
-
+struct vectortype_cstorage{
+	void* container;
+	unsigned int size, capacity;
 };
+
 template <typename _Ty>
 class vector_type {
 public:
@@ -291,7 +292,7 @@ public:
 	inline vector_type<_Ty> subvec(uint32_t start = 0) { return subvec(start, size); }
 	inline vector_type<_Ty> subvec_memcpy(uint32_t start = 0) { return subvec_memcpy(start, size); }
 	inline vector_type<_Ty> subvec_deep(uint32_t start = 0) { return subvec_deep(start, size); }
-	
+	vector_type<_Ty> getRef() { return vector_type<_Ty>(container, size); }
 	~vector_type() {
 		if (capacity > 0) free(container);
 		container = 0; size = capacity = 0;

tests/q.sql

@@ -1,112 +0,0 @@
-FUNCTION
-execStrategy ( alloc , mavgday , mavgmonth , px ) {
-buySignal := mavgday > mavgmonth ;
-f := a + b ;
-alloc * prd (
-CASE maxs ( buySignal )
-WHEN TRUE THEN
-CASE buySignal
-WHEN TRUE THEN 1 / px
-ELSE px
-END
-ELSE 1
-END )
-}
-FUNCTION covariance (x , y ) {
-xmean := avg (x) ;
-ymean := avg (y) ;
-avg (( x - xmean ) * (y - ymean ))
-}
-FUNCTION sd ( x) {
-sqrt ( covariance (x , x) )
-}
-FUNCTION pairCorr (x , y ) {
-covariance (x , y ) / ( sd (x) * sd (y ))
-}
-<k>
-`
-p:5
-q:2
-phi:(p+1)?1.
-theta:q?1.
-"p q phi theta"
-p
-q
-phi
-theta
-l:()
-e:()
-
-`
-L1:10?20
-Le1:10?2.
-L2:3?20
-Le2:3?2.
-"L1 Le1 L2 Le2"
-L1
-Le1
-L2
-Le2
-
-`
-"Add L1, then predict"
-l:l,L1
-e:e,Le1
-predict:(phi(0)) + (sum ({[x](phi(x+1)) * (l(((#l)-1)-x))}[!p])) - (sum ({[x](theta(x)) * (e(((#e)-1)-x))}[!q]))
-predict
-
-`
-"Add L2, then predict"
-l:l,L2
-e:e,Le2
-predict:(phi(0)) + (sum ({[x](phi(x+1)) * (l(((#l)-1)-x))}[!p])) - (sum ({[x](theta(x)) * (e(((#e)-1)-x))}[!q]))
-predict
-
- </k>
-
-WITH
-Target (Id , TradeDate , ClosePrice ) AS
-( SELECT
-Id , TradeDate , ClosePrice
-FROM price
-WHERE Id IN stock10 AND
-TradeDate >= startYear10 AND
-TradeDate <= startYear10 + 365 * 10),
-weekly (Id , bucket , name , low , high , mean ) AS
-( SELECT
-Id ,
-timeBucket ,
-" weekly " ,
-min ( ClosePrice ) ,
-max ( ClosePrice ) ,
-avg ( ClosePrice )
-FROM Target
-GROUP BY Id , getWeek ( TradeDate ) as
-timeBucket ),
-monthly ( Id , bucket , name , low , high , mean ) AS
-( SELECT
-Id ,
-timeBucket ,
-" monthly " ,
-min ( ClosePrice ) ,
-max ( ClosePrice ) ,
-avg ( ClosePrice )
-FROM Target
-GROUP BY Id , getMonth ( TradeDate ) as
-timeBucket ),
-yearly (Id , bucket , name , low , high , mean ) AS
-( SELECT
-Id ,
-timeBucket ,
-" yearly " ,
-min ( ClosePrice ) ,
-max ( ClosePrice ) ,
-avg ( ClosePrice )
-FROM Target
-GROUP BY Id , getYear ( TradeDate ) as
-timeBucket )
-SELECT
-Id , bucket , name , low , high , mean
-FROM
-CONCATENATE ( weekly , monthly , yearly )
-ASSUMING ASC Id , ASC name , ASC bucket

tests/udf2.a

@@ -1,14 +0,0 @@
-AGGREGATION FUNCTION covariances(x, y, w){
-    static xmeans := 0., ymeans := 0., cnt := 0; 
-    if (cnt < w) 
-    { 
-        xmeans += x; 
-        ymeans += y; 
-        cnt+=1;
-    }
-    else {       
-        xmeans += (x - x.vec[cnt - w]) / w;
-        ymeans += (y - y.vec[cnt - w]) / w;   
-    }
-    avg (( x.vec(x.len-w, x.len) - xmeans ) * (y.vec(y.len - w, y.len) - ymeans ))
-}

tests/udf4.a

@@ -1,32 +0,0 @@
-AGGREGATION FUNCTION covariances2(x, y, w){
-    xmeans := 0.;
-    ymeans := 0.;
-    l := _builtin_len;
-    if (w > l)
-        w := l;
-    elif (w > l + 2)
-    {
-        l := 3;
-        w := 4;
-    }
-    elif(w < 99){
-        l := 8;
-    }
-    elif(w<999)
-        w := 6;
-    else
-        l := l / 2;
-
-    for (i := 0, j:= 0; i < w; i := i+1) {
-        xmeans += x[i]; 
-        ymeans += y[i]; 
-        _builtin_ret[i] := avg (( x(l-w, l) - xmeans ) * (y(l - w, l) - ymeans ));
-    }
-    for (i := 0; i < l; i += 1) 
-    {
-        xmeans += (x[i] - x[i - w]) / w;
-        ymeans += (y[i] - y[i - w]) / w;   
-        _builtin_ret[i] := avg (( x(l-w, l) - xmeans ) * (y(l - w, l) - ymeans ));
-    }
-    Null
-}