initial support for pack expr

dev
Bill 2 years ago
parent 826c156d9f
commit 169b55a679

Binary file not shown.

@ -3,7 +3,11 @@ MonetDB_LIB =
MonetDB_INC = MonetDB_INC =
Threading = Threading =
CXXFLAGS = --std=c++1z CXXFLAGS = --std=c++1z
OPTFLAGS = -O3 -DNDEBUG ifeq ($(AQ_DEBUG), 1)
OPTFLAGS = -g3
else
OPTFLAGS = -O3 -DNDEBUG -fno-stack-protector
endif
LINKFLAGS = -flto # + $(AQ_LINK_FLAG) LINKFLAGS = -flto # + $(AQ_LINK_FLAG)
SHAREDFLAGS = -shared SHAREDFLAGS = -shared
FPIC = -fPIC FPIC = -fPIC

@ -8,7 +8,7 @@ AQuery++ Database is a cross-platform, In-Memory Column-Store Database that inco
- See installation instructions from [docker.com](https://www.docker.com). Run **docker desktop** to start docker engine. - See installation instructions from [docker.com](https://www.docker.com). Run **docker desktop** to start docker engine.
- In AQuery root directory, type `make docker` to build the docker image from scratch. - In AQuery root directory, type `make docker` to build the docker image from scratch.
- For Arm-based Mac users, you would have to build and run the **x86_64** docker image because MonetDB doesn't offer official binaries for arm64 Linux. (Run `docker buildx build --platform=linux/amd64 -t aquery .` instead of `make docker`) - For Arm-based Mac users, you would have to build and run the **x86_64** docker image because MonetDB doesn't offer official binaries for arm64 Linux. (Run `docker buildx build --platform=linux/amd64 -t aquery .` instead of `make docker`)
- Finally run the image in **interactive** mode (`docker run -it aquery --name aquery`) - Finally run the image in **interactive** mode (`docker run --name aquery -it aquery`)
- When you need to access the container again run `docker start -ai aquery` - When you need to access the container again run `docker start -ai aquery`
- If there is a need to access the system shell within AQuery, type `dbg` to activate python interpreter and type `os.system('sh')` to launch a shell. - If there is a need to access the system shell within AQuery, type `dbg` to activate python interpreter and type `os.system('sh')` to launch a shell.

@ -2,7 +2,7 @@
## GLOBAL CONFIGURATION FLAGS ## GLOBAL CONFIGURATION FLAGS
version_string = '0.4.8a' version_string = '0.4.9a'
add_path_to_ldpath = True add_path_to_ldpath = True
rebuild_backend = False rebuild_backend = False
run_backend = True run_backend = True

@ -109,7 +109,9 @@ class build_manager:
os.environ['CXX'] = mgr.cxx if mgr.cxx else 'c++' os.environ['CXX'] = mgr.cxx if mgr.cxx else 'c++'
else: else:
mgr.cxx = os.environ['CXX'] mgr.cxx = os.environ['CXX']
if 'AQ_DEBUG' not in os.environ:
os.environ['AQ_DEBUG'] = '0' if mgr.OptimizationLv else '1'
def libaquery_a(self): def libaquery_a(self):
self.build_cmd = [['rm', 'libaquery.a'],['make', 'libaquery.a']] self.build_cmd = [['rm', 'libaquery.a'],['make', 'libaquery.a']]
return self.build() return self.build()

@ -1,4 +1,4 @@
#!/usr/bash #!/bin/bash
echo "Don't execute this script if it's not on CIMS servers." echo "Don't execute this script if it's not on CIMS servers."
echo "run this script with source command. e.g. source ./cims.sh or . ./cims.sh" echo "run this script with source command. e.g. source ./cims.sh or . ./cims.sh"
module load gcc-11.2 module load gcc-11.2

@ -226,7 +226,17 @@ def long_return(*_ : Types) -> Types:
return LongT.long_type return LongT.long_type
def lfp_return(*_ : Types) -> Types: def lfp_return(*_ : Types) -> Types:
return LongT.fp_type return LongT.fp_type
def pack_return(*args : Types) -> Types:
if len(args) == 0:
raise ValueError('0 arguments in pack expression')
inner_ty = args[0]
for a in args:
if a != inner_ty:
raise ValueError('pack expression with different types')
if isinstance(inner_ty, VectorT):
print('warning: packing vector types')
inner_ty = inner_ty.inner_type
return VectorT(args[0])
def as_is (t: Types) -> Types: def as_is (t: Types) -> Types:
return t return t
@ -269,6 +279,15 @@ def windowed_fn_behavor(op: OperatorBase, c_code, *x):
else: else:
name = op.cname if len(x) == 1 else op.cname[:-1] + 'w' name = op.cname if len(x) == 1 else op.cname[:-1] + 'w'
return f'{name}({", ".join([f"{xx}" for xx in x])})' return f'{name}({", ".join([f"{xx}" for xx in x])})'
def pack_behavior(op: OperatorBase, c_code, *x):
if len(x) == 0:
raise ValueError('0 arguments in pack expression')
if not c_code:
return f'{op.sqlname}({", ".join([f"{xx}" for xx in x])})'
else:
return f'decltype({x[0]})::pack(len(x) + 1, {", ".join([f"{xx}.s()" for xx in x])})'
# arithmetic # arithmetic
opadd = OperatorBase('add', 2, auto_extension, cname = '+', sqlname = '+', call = binary_op_behavior) opadd = OperatorBase('add', 2, auto_extension, cname = '+', sqlname = '+', call = binary_op_behavior)
@ -307,6 +326,7 @@ fnmins = OperatorBase('mins', [1, 2], ty_clamp(as_is, -1), cname = 'mins', sqlna
fnsums = OperatorBase('sums', [1, 2], ext(ty_clamp(auto_extension, -1)), cname = 'sums', sqlname = 'SUMS', call = windowed_fn_behavor) fnsums = OperatorBase('sums', [1, 2], ext(ty_clamp(auto_extension, -1)), cname = 'sums', sqlname = 'SUMS', call = windowed_fn_behavor)
fnavgs = OperatorBase('avgs', [1, 2], fp(ext(ty_clamp(auto_extension, -1))), cname = 'avgs', sqlname = 'AVGS', call = windowed_fn_behavor) fnavgs = OperatorBase('avgs', [1, 2], fp(ext(ty_clamp(auto_extension, -1))), cname = 'avgs', sqlname = 'AVGS', call = windowed_fn_behavor)
fncnt = OperatorBase('count', 1, int_return, cname = 'count', sqlname = 'COUNT', call = count_behavior) fncnt = OperatorBase('count', 1, int_return, cname = 'count', sqlname = 'COUNT', call = count_behavior)
fnpack = OperatorBase('pack', -1, pack_return, cname = 'pack', sqlname = 'PACK', call = pack_behavior)
# special # special
def is_null_call_behavior(op:OperatorBase, c_code : bool, x : str): def is_null_call_behavior(op:OperatorBase, c_code : bool, x : str):
if c_code : if c_code :
@ -328,12 +348,16 @@ fnpow = OperatorBase('pow', 2, lambda *_ : DoubleT, cname = 'pow', sqlname = 'PO
def _op_make_dict(*items : OperatorBase): def _op_make_dict(*items : OperatorBase):
return { i.name: i for i in items} return { i.name: i for i in items}
builtin_binary_arith = _op_make_dict(opadd, opdiv, opmul, opsub, opmod) builtin_binary_arith = _op_make_dict(opadd, opdiv, opmul, opsub, opmod)
builtin_binary_logical = _op_make_dict(opand, opor, opxor, opgt, oplt, opge, oplte, opneq, opeq) builtin_binary_logical = _op_make_dict(opand, opor, opxor, opgt, oplt,
opge, oplte, opneq, opeq)
builtin_unary_logical = _op_make_dict(opnot) builtin_unary_logical = _op_make_dict(opnot)
builtin_unary_arith = _op_make_dict(opneg) builtin_unary_arith = _op_make_dict(opneg)
builtin_unary_special = _op_make_dict(spnull, opdistinct) builtin_unary_special = _op_make_dict(spnull, opdistinct)
builtin_cstdlib = _op_make_dict(fnsqrt, fnlog, fnsin, fncos, fntan, fnpow) builtin_cstdlib = _op_make_dict(fnsqrt, fnlog, fnsin, fncos, fntan, fnpow)
builtin_func = _op_make_dict(fnmax, fnmin, fnsum, fnavg, fnmaxs, fnmins, fndeltas, fnratios, fnlast, fnfirst, fnsums, fnavgs, fncnt) builtin_func = _op_make_dict(fnmax, fnmin, fnsum, fnavg, fnmaxs,
fnmins, fndeltas, fnratios, fnlast,
fnfirst, fnsums, fnavgs, fncnt,
fnpack)
user_module_func = {} user_module_func = {}
builtin_operators : Dict[str, OperatorBase] = {**builtin_binary_arith, **builtin_binary_logical, builtin_operators : Dict[str, OperatorBase] = {**builtin_binary_arith, **builtin_binary_logical,
**builtin_unary_arith, **builtin_unary_logical, **builtin_unary_special, **builtin_func, **builtin_cstdlib, **builtin_unary_arith, **builtin_unary_logical, **builtin_unary_special, **builtin_func, **builtin_cstdlib,

@ -497,6 +497,23 @@ def prompt(running = lambda:True, next = lambda:input('> '), state = None):
elif q == 'exit' or q == 'exit()' or q == 'quit' or q == 'quit()' or q == '\\q': elif q == 'exit' or q == 'exit()' or q == 'quit' or q == 'quit()' or q == '\\q':
rm(state) rm(state)
exit() exit()
elif q.startswith('sh'):
from distutils.spawn import find_executable
qs = re.split(r'[ \t]', q)
shells = ('zsh', 'bash', 'sh', 'fish', 'cmd', 'pwsh', 'powershell', 'csh', 'tcsh', 'ksh')
shell_path = ''
if len(qs) > 1 and qs[1] in shells:
shell_path = find_executable(qs[1])
if shell_path:
os.system(shell_path)
else:
for sh in shells:
shell_path = find_executable(sh)
if shell_path:
os.system(shell_path)
break
continue
elif q == 'r': # build and run elif q == 'r': # build and run
if state.server_mode == RunType.Threaded: if state.server_mode == RunType.Threaded:
qs = [ctypes.c_char_p(bytes(q, 'utf-8')) for q in cxt.queries if len(q)] qs = [ctypes.c_char_p(bytes(q, 'utf-8')) for q in cxt.queries if len(q)]

@ -8,6 +8,7 @@
#include <iostream> #include <iostream>
#include <string> #include <string>
#include <algorithm> #include <algorithm>
#include <cstdarg>
#include "io.h" #include "io.h"
#include "hasher.h" #include "hasher.h"
@ -26,6 +27,13 @@ namespace types {
struct Coercion; struct Coercion;
} }
#endif #endif
struct ColRef_cstorage {
void* container;
unsigned int size, capacity;
const char* name;
int ty; // what if enum is not int?
};
template <template <class...> class VT, class T> template <template <class...> class VT, class T>
std::ostream& operator<<(std::ostream& os, const VT<T>& v) std::ostream& operator<<(std::ostream& os, const VT<T>& v)
{ {
@ -80,6 +88,10 @@ public:
vt.capacity = 0; // rvalue's vt.capacity = 0; // rvalue's
return *this; return *this;
} }
template <class T>
ColRef<_Ty> getRef(){
return ColRef<_Ty>(this->size, this->container, this->name);
}
ColRef(const char* name, types::Type_t ty) : name(name), ty(ty) {} ColRef(const char* name, types::Type_t ty) : name(name), ty(ty) {}
using vector_type<_Ty>::operator[]; using vector_type<_Ty>::operator[];
//using vector_type<_Ty>::operator=; //using vector_type<_Ty>::operator=;
@ -127,7 +139,37 @@ public:
this->name = name; this->name = name;
return this; return this;
} }
static ColRef<vector_type<_Ty>> pack(uint32_t cnt, ...){
va_list cols;
va_start(cols, cnt);
ColRef<_Ty> *col = (ColRef<_Ty>*)malloc(sizeof(ColRef<_Ty>) * cnt);
for (uint32_t i = 0; i < cnt; ++i){
ColRef_cstorage tmp = va_arg(cols, ColRef_cstorage);
memcpy(&col[i], &tmp, sizeof(ColRef_cstorage));
col[i].capacity = 0;
}
va_end(cols);
if(cnt > 0){
auto sz = col[0].size;
ColRef<vector_type<_Ty>> ret(sz);
for(uint32_t i = 0; i < sz; ++i){
decltype(auto) v = ret[i];
v.size = cnt;
v.capacity = cnt;
v.container = (_Ty*)malloc(sizeof(_Ty) * cnt);
for(uint32_t j = 0; j < cnt; ++j) {
v.container[j] = col[j][i];
}
}
free(col);
return ret;
}
else
return ColRef<vector_type<_Ty>>();
}
ColRef_cstorage s() {
return *(ColRef_cstorage*)(this);
}
// defined in table_ext_monetdb.hpp // defined in table_ext_monetdb.hpp
void* monetdb_get_col(void** gc_vecs, uint32_t& cnt); void* monetdb_get_col(void** gc_vecs, uint32_t& cnt);

@ -22,6 +22,10 @@ struct is_vector_impl : std::false_type {};
template <class T> template <class T>
constexpr static bool is_vector_type = is_vector_impl<T>::value; constexpr static bool is_vector_type = is_vector_impl<T>::value;
#define Cond(c, x, y) typename std::conditional<c, x, y>::type #define Cond(c, x, y) typename std::conditional<c, x, y>::type
template <class T>
constexpr size_t aq_szof = sizeof(T);
template <>
inline constexpr size_t aq_szof<void> = 0;
template <class T1, class T2> template <class T1, class T2>
struct aqis_same_impl { struct aqis_same_impl {
constexpr static bool value = constexpr static bool value =
@ -30,7 +34,7 @@ struct aqis_same_impl {
Cond( Cond(
std::is_floating_point_v<T1> == std::is_floating_point_v<T2>, std::is_floating_point_v<T1> == std::is_floating_point_v<T2>,
Cond( Cond(
sizeof(T1) == sizeof(T2), aq_szof<T1> == aq_szof<T2>, // deal with sizeof(void)
std::true_type, std::true_type,
std::false_type std::false_type
), ),
@ -40,8 +44,12 @@ struct aqis_same_impl {
>::value; >::value;
}; };
template <class T1, class T2, class ...Ts>
constexpr bool aqis_same = aqis_same_impl<T1, T2>::value &&
aqis_same<T2, Ts...>;
template <class T1, class T2> template <class T1, class T2>
constexpr bool aqis_same = aqis_same_impl<T1, T2>::value; constexpr bool aqis_same<T1, T2> = aqis_same_impl<T1, T2>::value;
namespace types { namespace types {
enum Type_t { enum Type_t {
AINT32, AFLOAT, ASTR, ADOUBLE, ALDOUBLE, AINT64, AINT128, AINT16, ADATE, ATIME, AINT8, AINT32, AFLOAT, ASTR, ADOUBLE, ALDOUBLE, AINT64, AINT128, AINT16, ADATE, ATIME, AINT8,
@ -165,15 +173,9 @@ namespace types {
#define ATypeSize(t, at) sizeof(t), #define ATypeSize(t, at) sizeof(t),
static constexpr size_t AType_sizes[] = { ConnectTypes(ATypeSize) 1 }; static constexpr size_t AType_sizes[] = { ConnectTypes(ATypeSize) 1 };
#define Comp(o) (sizeof(T1) o sizeof(T2)) #define Comp(o) (sizeof(T1) o sizeof(T2))
#define Same(x, y) (std::is_same_v<x, y>) #define Same(x, y) (aqis_same<x, y>)
#define __U(x) std::is_unsigned<x>::value #define __U(x) std::is_unsigned<x>::value
#define Fp(x) std::is_floating_point<x>::value #define Fp(x) std::is_floating_point<x>::value
template <class T1, class T2>
struct Coercion {
using t1 = Cond(Comp(<= ), Cond(Comp(== ), Cond(Fp(T1), T1, Cond(Fp(T2), T2, Cond(__U(T1), T2, T1))), T2), T1);
using t2 = Cond(Same(T1, T2), T1, Cond(Same(T1, const char*) || Same(T2, const char*), const char*, void));
using type = Cond(Same(t2, void), Cond(Same(T1, date_t) && Same(T2, time_t) || Same(T1, time_t) && Same(T2, time_t), void, t1), t2);
};
#define __Eq(x) (sizeof(T) == sizeof(x)) #define __Eq(x) (sizeof(T) == sizeof(x))
template<class T> template<class T>
struct GetFPTypeImpl { struct GetFPTypeImpl {
@ -210,6 +212,50 @@ namespace types {
}; };
template<class T> template<class T>
using GetLongerType = typename GetLongerTypeImpl<typename std::decay<T>::type>::type; using GetLongerType = typename GetLongerTypeImpl<typename std::decay<T>::type>::type;
#ifdef __AQ__HAS__INT128__
#define __AQ_HELPER_INT128__(x, y) x
#else
#define __AQ_HELPER_INT128__(x, y) y
#endif
template <class T>
struct GetSignedType_impl{
using type = Cond(Same(T, unsigned char), char,
Cond(Same(T, unsigned short), short,
Cond(Same(T, unsigned int), int,
Cond(Same(T, unsigned long), long,
// #ifdef __AQ__HAS__INT128__
__AQ_HELPER_INT128__(
Cond(Same(T, unsigned long long), long long,
Cond(Same(T, unsigned __int128), __int128_t,
T
)
),
// #else
T
)
// #endif
)
)
)
);
};
template <class T>
using GetSignedType = typename GetSignedType_impl<T>::type;
template <class T1, class T2, class ...Ts>
struct Coercion{
using type = typename Coercion<T1, typename Coercion<T2, Ts...>::type>::type;
};
template <class T1, class T2>
struct Coercion<T1, T2> {
using t0 = Cond(Comp(<= ), Cond(Comp(== ), Cond(Fp(T1), T1, Cond(Fp(T2), T2, Cond(__U(T1), T2, T1))), T2), T1);
using t1 = Cond(Fp(T1)||Fp(T2), GetFPType<t0>, Cond(!(__U(T1) && __U(T2)), GetSignedType<t0>,t0));
using t2 = Cond(Same(T1, T2), T1, Cond(Same(T1, const char*) || Same(T2, const char*), const char*, void));
using type = Cond(Same(t2, void), Cond(Same(T1, date_t) && Same(T2, time_t) || Same(T1, time_t) && Same(T2, time_t), void, t1), t2);
};
} }

@ -17,10 +17,11 @@
#include "types.h" #include "types.h"
#pragma pack(push, 1) #pragma pack(push, 1)
template <typename _Ty> struct vectortype_cstorage{
class slim_vector { void* container;
unsigned int size, capacity;
}; };
template <typename _Ty> template <typename _Ty>
class vector_type { class vector_type {
public: public:
@ -291,7 +292,7 @@ public:
inline vector_type<_Ty> subvec(uint32_t start = 0) { return subvec(start, size); } inline vector_type<_Ty> subvec(uint32_t start = 0) { return subvec(start, size); }
inline vector_type<_Ty> subvec_memcpy(uint32_t start = 0) { return subvec_memcpy(start, size); } inline vector_type<_Ty> subvec_memcpy(uint32_t start = 0) { return subvec_memcpy(start, size); }
inline vector_type<_Ty> subvec_deep(uint32_t start = 0) { return subvec_deep(start, size); } inline vector_type<_Ty> subvec_deep(uint32_t start = 0) { return subvec_deep(start, size); }
vector_type<_Ty> getRef() { return vector_type<_Ty>(container, size); }
~vector_type() { ~vector_type() {
if (capacity > 0) free(container); if (capacity > 0) free(container);
container = 0; size = capacity = 0; container = 0; size = capacity = 0;

@ -1,112 +0,0 @@
FUNCTION
execStrategy ( alloc , mavgday , mavgmonth , px ) {
buySignal := mavgday > mavgmonth ;
f := a + b ;
alloc * prd (
CASE maxs ( buySignal )
WHEN TRUE THEN
CASE buySignal
WHEN TRUE THEN 1 / px
ELSE px
END
ELSE 1
END )
}
FUNCTION covariance (x , y ) {
xmean := avg (x) ;
ymean := avg (y) ;
avg (( x - xmean ) * (y - ymean ))
}
FUNCTION sd ( x) {
sqrt ( covariance (x , x) )
}
FUNCTION pairCorr (x , y ) {
covariance (x , y ) / ( sd (x) * sd (y ))
}
<k>
`
p:5
q:2
phi:(p+1)?1.
theta:q?1.
"p q phi theta"
p
q
phi
theta
l:()
e:()
`
L1:10?20
Le1:10?2.
L2:3?20
Le2:3?2.
"L1 Le1 L2 Le2"
L1
Le1
L2
Le2
`
"Add L1, then predict"
l:l,L1
e:e,Le1
predict:(phi(0)) + (sum ({[x](phi(x+1)) * (l(((#l)-1)-x))}[!p])) - (sum ({[x](theta(x)) * (e(((#e)-1)-x))}[!q]))
predict
`
"Add L2, then predict"
l:l,L2
e:e,Le2
predict:(phi(0)) + (sum ({[x](phi(x+1)) * (l(((#l)-1)-x))}[!p])) - (sum ({[x](theta(x)) * (e(((#e)-1)-x))}[!q]))
predict
</k>
WITH
Target (Id , TradeDate , ClosePrice ) AS
( SELECT
Id , TradeDate , ClosePrice
FROM price
WHERE Id IN stock10 AND
TradeDate >= startYear10 AND
TradeDate <= startYear10 + 365 * 10),
weekly (Id , bucket , name , low , high , mean ) AS
( SELECT
Id ,
timeBucket ,
" weekly " ,
min ( ClosePrice ) ,
max ( ClosePrice ) ,
avg ( ClosePrice )
FROM Target
GROUP BY Id , getWeek ( TradeDate ) as
timeBucket ),
monthly ( Id , bucket , name , low , high , mean ) AS
( SELECT
Id ,
timeBucket ,
" monthly " ,
min ( ClosePrice ) ,
max ( ClosePrice ) ,
avg ( ClosePrice )
FROM Target
GROUP BY Id , getMonth ( TradeDate ) as
timeBucket ),
yearly (Id , bucket , name , low , high , mean ) AS
( SELECT
Id ,
timeBucket ,
" yearly " ,
min ( ClosePrice ) ,
max ( ClosePrice ) ,
avg ( ClosePrice )
FROM Target
GROUP BY Id , getYear ( TradeDate ) as
timeBucket )
SELECT
Id , bucket , name , low , high , mean
FROM
CONCATENATE ( weekly , monthly , yearly )
ASSUMING ASC Id , ASC name , ASC bucket

@ -1,14 +0,0 @@
AGGREGATION FUNCTION covariances(x, y, w){
static xmeans := 0., ymeans := 0., cnt := 0;
if (cnt < w)
{
xmeans += x;
ymeans += y;
cnt+=1;
}
else {
xmeans += (x - x.vec[cnt - w]) / w;
ymeans += (y - y.vec[cnt - w]) / w;
}
avg (( x.vec(x.len-w, x.len) - xmeans ) * (y.vec(y.len - w, y.len) - ymeans ))
}

@ -1,32 +0,0 @@
AGGREGATION FUNCTION covariances2(x, y, w){
xmeans := 0.;
ymeans := 0.;
l := _builtin_len;
if (w > l)
w := l;
elif (w > l + 2)
{
l := 3;
w := 4;
}
elif(w < 99){
l := 8;
}
elif(w<999)
w := 6;
else
l := l / 2;
for (i := 0, j:= 0; i < w; i := i+1) {
xmeans += x[i];
ymeans += y[i];
_builtin_ret[i] := avg (( x(l-w, l) - xmeans ) * (y(l - w, l) - ymeans ));
}
for (i := 0; i < l; i += 1)
{
xmeans += (x[i] - x[i - w]) / w;
ymeans += (y[i] - y[i - w]) / w;
_builtin_ret[i] := avg (( x(l-w, l) - xmeans ) * (y(l - w, l) - ymeans ));
}
Null
}
Loading…
Cancel
Save