bill/AQuery
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

initial support for pack expr

Browse Source
This commit is contained in:
bill
2022-10-14 20:36:09 +08:00
parent 826c156d9f
commit 169b55a679
14 changed files with 158 additions and 180 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.test.cpp.swp
BIN
View File
Binary file not shown.
Makefile
+5 -1
View File
@@ -3,7 +3,11 @@ MonetDB_LIB =
MonetDB_INC =
Threading =
CXXFLAGS = --std=c++1z
OPTFLAGS = -O3 -DNDEBUG
ifeq ($(AQ_DEBUG), 1)
OPTFLAGS = -g3
else
OPTFLAGS = -O3 -DNDEBUG -fno-stack-protector
endif
LINKFLAGS = -flto # + $(AQ_LINK_FLAG)
SHAREDFLAGS = -shared
FPIC = -fPIC
README.md
+1 -1
View File
@@ -8,7 +8,7 @@ AQuery++ Database is a cross-platform, In-Memory Column-Store Database that inco
- See installation instructions from [docker.com](https://www.docker.com). Run **docker desktop** to start docker engine.
- In AQuery root directory, type `make docker` to build the docker image from scratch.
- For Arm-based Mac users, you would have to build and run the **x86_64** docker image because MonetDB doesn't offer official binaries for arm64 Linux. (Run `docker buildx build --platform=linux/amd64 -t aquery .` instead of `make docker`)
- Finally run the image in **interactive** mode (`docker run -it aquery --name aquery`)
- Finally run the image in **interactive** mode (`docker run --name aquery -it aquery`)
- When you need to access the container again run `docker start -ai aquery`
- If there is a need to access the system shell within AQuery, type `dbg` to activate python interpreter and type `os.system('sh')` to launch a shell.
aquery_config.py
+1 -1
View File
@@ -2,7 +2,7 @@
## GLOBAL CONFIGURATION FLAGS
version_string = '0.4.8a'
version_string = '0.4.9a'
add_path_to_ldpath = True
rebuild_backend = False
run_backend = True
build.py
+3 -1
View File
@@ -109,7 +109,9 @@ class build_manager:
os.environ['CXX'] = mgr.cxx if mgr.cxx else 'c++'
else:
mgr.cxx = os.environ['CXX']
if 'AQ_DEBUG' not in os.environ:
os.environ['AQ_DEBUG'] = '0' if mgr.OptimizationLv else '1'
def libaquery_a(self):
self.build_cmd = [['rm', 'libaquery.a'],['make', 'libaquery.a']]
return self.build()
cims.sh
+1 -1
View File
@@ -1,4 +1,4 @@
#!/usr/bash
#!/bin/bash
echo "Don't execute this script if it's not on CIMS servers."
echo "run this script with source command. e.g. source ./cims.sh or . ./cims.sh"
module load gcc-11.2
engine/types.py
+27 -3
View File
@@ -226,7 +226,17 @@ def long_return(*_ : Types) -> Types:
return LongT.long_type
def lfp_return(*_ : Types) -> Types:
return LongT.fp_type
def pack_return(*args : Types) -> Types:
if len(args) == 0:
raise ValueError('0 arguments in pack expression')
inner_ty = args[0]
for a in args:
if a != inner_ty:
raise ValueError('pack expression with different types')
if isinstance(inner_ty, VectorT):
print('warning: packing vector types')
inner_ty = inner_ty.inner_type
return VectorT(args[0])
def as_is (t: Types) -> Types:
return t
@@ -269,6 +279,15 @@ def windowed_fn_behavor(op: OperatorBase, c_code, *x):
else:
name = op.cname if len(x) == 1 else op.cname[:-1] + 'w'
return f'{name}({", ".join([f"{xx}" for xx in x])})'
def pack_behavior(op: OperatorBase, c_code, *x):
if len(x) == 0:
raise ValueError('0 arguments in pack expression')
if not c_code:
return f'{op.sqlname}({", ".join([f"{xx}" for xx in x])})'
else:
return f'decltype({x[0]})::pack(len(x) + 1, {", ".join([f"{xx}.s()" for xx in x])})'
# arithmetic
opadd = OperatorBase('add', 2, auto_extension, cname = '+', sqlname = '+', call = binary_op_behavior)
@@ -307,6 +326,7 @@ fnmins = OperatorBase('mins', [1, 2], ty_clamp(as_is, -1), cname = 'mins', sqlna
fnsums = OperatorBase('sums', [1, 2], ext(ty_clamp(auto_extension, -1)), cname = 'sums', sqlname = 'SUMS', call = windowed_fn_behavor)
fnavgs = OperatorBase('avgs', [1, 2], fp(ext(ty_clamp(auto_extension, -1))), cname = 'avgs', sqlname = 'AVGS', call = windowed_fn_behavor)
fncnt = OperatorBase('count', 1, int_return, cname = 'count', sqlname = 'COUNT', call = count_behavior)
fnpack = OperatorBase('pack', -1, pack_return, cname = 'pack', sqlname = 'PACK', call = pack_behavior)
# special
def is_null_call_behavior(op:OperatorBase, c_code : bool, x : str):
if c_code :
@@ -328,12 +348,16 @@ fnpow = OperatorBase('pow', 2, lambda *_ : DoubleT, cname = 'pow', sqlname = 'PO
def _op_make_dict(*items : OperatorBase):
return { i.name: i for i in items}
builtin_binary_arith = _op_make_dict(opadd, opdiv, opmul, opsub, opmod)
builtin_binary_logical = _op_make_dict(opand, opor, opxor, opgt, oplt, opge, oplte, opneq, opeq)
builtin_binary_logical = _op_make_dict(opand, opor, opxor, opgt, oplt,
opge, oplte, opneq, opeq)
builtin_unary_logical = _op_make_dict(opnot)
builtin_unary_arith = _op_make_dict(opneg)
builtin_unary_special = _op_make_dict(spnull, opdistinct)
builtin_cstdlib = _op_make_dict(fnsqrt, fnlog, fnsin, fncos, fntan, fnpow)
builtin_func = _op_make_dict(fnmax, fnmin, fnsum, fnavg, fnmaxs, fnmins, fndeltas, fnratios, fnlast, fnfirst, fnsums, fnavgs, fncnt)
builtin_func = _op_make_dict(fnmax, fnmin, fnsum, fnavg, fnmaxs,
fnmins, fndeltas, fnratios, fnlast,
fnfirst, fnsums, fnavgs, fncnt,
fnpack)
user_module_func = {}
builtin_operators : Dict[str, OperatorBase] = {**builtin_binary_arith, **builtin_binary_logical,
**builtin_unary_arith, **builtin_unary_logical, **builtin_unary_special, **builtin_func, **builtin_cstdlib,
prompt.py
+17
View File
@@ -497,6 +497,23 @@ def prompt(running = lambda:True, next = lambda:input('> '), state = None):
elif q == 'exit' or q == 'exit()' or q == 'quit' or q == 'quit()' or q == '\\q':
rm(state)
exit()
elif q.startswith('sh'):
from distutils.spawn import find_executable
qs = re.split(r'[ \t]', q)
shells = ('zsh', 'bash', 'sh', 'fish', 'cmd', 'pwsh', 'powershell', 'csh', 'tcsh', 'ksh')
shell_path = ''
if len(qs) > 1 and qs[1] in shells:
shell_path = find_executable(qs[1])
if shell_path:
os.system(shell_path)
else:
for sh in shells:
shell_path = find_executable(sh)
if shell_path:
os.system(shell_path)
break
continue
elif q == 'r': # build and run
if state.server_mode == RunType.Threaded:
qs = [ctypes.c_char_p(bytes(q, 'utf-8')) for q in cxt.queries if len(q)]
server/table.h
+43 -1
View File
@@ -8,6 +8,7 @@
#include <iostream>
#include <string>
#include <algorithm>
#include <cstdarg>
#include "io.h"
#include "hasher.h"
@@ -26,6 +27,13 @@ namespace types {
struct Coercion;
}
#endif
struct ColRef_cstorage {
void* container;
unsigned int size, capacity;
const char* name;
int ty; // what if enum is not int?
};
template <template <class...> class VT, class T>
std::ostream& operator<<(std::ostream& os, const VT<T>& v)
{
@@ -80,6 +88,10 @@ public:
vt.capacity = 0; // rvalue's
return *this;
}
template <class T>
ColRef<_Ty> getRef(){
return ColRef<_Ty>(this->size, this->container, this->name);
}
ColRef(const char* name, types::Type_t ty) : name(name), ty(ty) {}
using vector_type<_Ty>::operator[];
//using vector_type<_Ty>::operator=;
@@ -127,7 +139,37 @@ public:
this->name = name;
return this;
}
static ColRef<vector_type<_Ty>> pack(uint32_t cnt, ...){
va_list cols;
va_start(cols, cnt);
ColRef<_Ty> *col = (ColRef<_Ty>*)malloc(sizeof(ColRef<_Ty>) * cnt);
for (uint32_t i = 0; i < cnt; ++i){
ColRef_cstorage tmp = va_arg(cols, ColRef_cstorage);
memcpy(&col[i], &tmp, sizeof(ColRef_cstorage));
col[i].capacity = 0;
}
va_end(cols);
if(cnt > 0){
auto sz = col[0].size;
ColRef<vector_type<_Ty>> ret(sz);
for(uint32_t i = 0; i < sz; ++i){
decltype(auto) v = ret[i];
v.size = cnt;
v.capacity = cnt;
v.container = (_Ty*)malloc(sizeof(_Ty) * cnt);
for(uint32_t j = 0; j < cnt; ++j) {
v.container[j] = col[j][i];
}
}
free(col);
return ret;
}
else
return ColRef<vector_type<_Ty>>();
}
ColRef_cstorage s() {
return *(ColRef_cstorage*)(this);
}
// defined in table_ext_monetdb.hpp
void* monetdb_get_col(void** gc_vecs, uint32_t& cnt);
server/types.h
+55 -9
View File
@@ -22,6 +22,10 @@ struct is_vector_impl : std::false_type {};
template <class T>
constexpr static bool is_vector_type = is_vector_impl<T>::value;
#define Cond(c, x, y) typename std::conditional<c, x, y>::type
template <class T>
constexpr size_t aq_szof = sizeof(T);
template <>
inline constexpr size_t aq_szof<void> = 0;
template <class T1, class T2>
struct aqis_same_impl {
constexpr static bool value =
@@ -30,7 +34,7 @@ struct aqis_same_impl {
Cond(
std::is_floating_point_v<T1> == std::is_floating_point_v<T2>,
Cond(
sizeof(T1) == sizeof(T2),
aq_szof<T1> == aq_szof<T2>, // deal with sizeof(void)
std::true_type,
std::false_type
),
@@ -40,8 +44,12 @@ struct aqis_same_impl {
>::value;
};
template <class T1, class T2, class ...Ts>
constexpr bool aqis_same = aqis_same_impl<T1, T2>::value &&
aqis_same<T2, Ts...>;
template <class T1, class T2>
constexpr bool aqis_same = aqis_same_impl<T1, T2>::value;
constexpr bool aqis_same<T1, T2> = aqis_same_impl<T1, T2>::value;
namespace types {
enum Type_t {
AINT32, AFLOAT, ASTR, ADOUBLE, ALDOUBLE, AINT64, AINT128, AINT16, ADATE, ATIME, AINT8,
@@ -165,15 +173,9 @@ namespace types {
#define ATypeSize(t, at) sizeof(t),
static constexpr size_t AType_sizes[] = { ConnectTypes(ATypeSize) 1 };
#define Comp(o) (sizeof(T1) o sizeof(T2))
#define Same(x, y) (std::is_same_v<x, y>)
#define Same(x, y) (aqis_same<x, y>)
#define __U(x) std::is_unsigned<x>::value
#define Fp(x) std::is_floating_point<x>::value
template <class T1, class T2>
struct Coercion {
using t1 = Cond(Comp(<= ), Cond(Comp(== ), Cond(Fp(T1), T1, Cond(Fp(T2), T2, Cond(__U(T1), T2, T1))), T2), T1);
using t2 = Cond(Same(T1, T2), T1, Cond(Same(T1, const char*) || Same(T2, const char*), const char*, void));
using type = Cond(Same(t2, void), Cond(Same(T1, date_t) && Same(T2, time_t) || Same(T1, time_t) && Same(T2, time_t), void, t1), t2);
};
#define __Eq(x) (sizeof(T) == sizeof(x))
template<class T>
struct GetFPTypeImpl {
@@ -210,6 +212,50 @@ namespace types {
};
template<class T>
using GetLongerType = typename GetLongerTypeImpl<typename std::decay<T>::type>::type;
#ifdef __AQ__HAS__INT128__
#define __AQ_HELPER_INT128__(x, y) x
#else
#define __AQ_HELPER_INT128__(x, y) y
#endif
template <class T>
struct GetSignedType_impl{
using type = Cond(Same(T, unsigned char), char,
Cond(Same(T, unsigned short), short,
Cond(Same(T, unsigned int), int,
Cond(Same(T, unsigned long), long,
// #ifdef __AQ__HAS__INT128__
__AQ_HELPER_INT128__(
Cond(Same(T, unsigned long long), long long,
Cond(Same(T, unsigned __int128), __int128_t,
T
)
),
// #else
T
)
// #endif
)
)
)
);
};
template <class T>
using GetSignedType = typename GetSignedType_impl<T>::type;
template <class T1, class T2, class ...Ts>
struct Coercion{
using type = typename Coercion<T1, typename Coercion<T2, Ts...>::type>::type;
};
template <class T1, class T2>
struct Coercion<T1, T2> {
using t0 = Cond(Comp(<= ), Cond(Comp(== ), Cond(Fp(T1), T1, Cond(Fp(T2), T2, Cond(__U(T1), T2, T1))), T2), T1);
using t1 = Cond(Fp(T1)||Fp(T2), GetFPType<t0>, Cond(!(__U(T1) && __U(T2)), GetSignedType<t0>,t0));
using t2 = Cond(Same(T1, T2), T1, Cond(Same(T1, const char*) || Same(T2, const char*), const char*, void));
using type = Cond(Same(t2, void), Cond(Same(T1, date_t) && Same(T2, time_t) || Same(T1, time_t) && Same(T2, time_t), void, t1), t2);
};
}
server/vector_type.hpp
+5 -4
View File
@@ -17,10 +17,11 @@
#include "types.h"
#pragma pack(push, 1)
template <typename _Ty>
class slim_vector {
struct vectortype_cstorage{
void* container;
unsigned int size, capacity;
};
template <typename _Ty>
class vector_type {
public:
@@ -291,7 +292,7 @@ public:
inline vector_type<_Ty> subvec(uint32_t start = 0) { return subvec(start, size); }
inline vector_type<_Ty> subvec_memcpy(uint32_t start = 0) { return subvec_memcpy(start, size); }
inline vector_type<_Ty> subvec_deep(uint32_t start = 0) { return subvec_deep(start, size); }
vector_type<_Ty> getRef() { return vector_type<_Ty>(container, size); }
~vector_type() {
if (capacity > 0) free(container);
container = 0; size = capacity = 0;
tests/q.sql
-112
View File
@@ -1,112 +0,0 @@
FUNCTION
execStrategy ( alloc , mavgday , mavgmonth , px ) {
buySignal := mavgday > mavgmonth ;
f := a + b ;
alloc * prd (
CASE maxs ( buySignal )
WHEN TRUE THEN
CASE buySignal
WHEN TRUE THEN 1 / px
ELSE px
END
ELSE 1
END )
}
FUNCTION covariance (x , y ) {
xmean := avg (x) ;
ymean := avg (y) ;
avg (( x - xmean ) * (y - ymean ))
}
FUNCTION sd ( x) {
sqrt ( covariance (x , x) )
}
FUNCTION pairCorr (x , y ) {
covariance (x , y ) / ( sd (x) * sd (y ))
}
<k>
`
p:5
q:2
phi:(p+1)?1.
theta:q?1.
"p q phi theta"
p
q
phi
theta
l:()
e:()
`
L1:10?20
Le1:10?2.
L2:3?20
Le2:3?2.
"L1 Le1 L2 Le2"
L1
Le1
L2
Le2
`
"Add L1, then predict"
l:l,L1
e:e,Le1
predict:(phi(0)) + (sum ({[x](phi(x+1)) * (l(((#l)-1)-x))}[!p])) - (sum ({[x](theta(x)) * (e(((#e)-1)-x))}[!q]))
predict
`
"Add L2, then predict"
l:l,L2
e:e,Le2
predict:(phi(0)) + (sum ({[x](phi(x+1)) * (l(((#l)-1)-x))}[!p])) - (sum ({[x](theta(x)) * (e(((#e)-1)-x))}[!q]))
predict
</k>
WITH
Target (Id , TradeDate , ClosePrice ) AS
( SELECT
Id , TradeDate , ClosePrice
FROM price
WHERE Id IN stock10 AND
TradeDate >= startYear10 AND
TradeDate <= startYear10 + 365 * 10),
weekly (Id , bucket , name , low , high , mean ) AS
( SELECT
Id ,
timeBucket ,
" weekly " ,
min ( ClosePrice ) ,
max ( ClosePrice ) ,
avg ( ClosePrice )
FROM Target
GROUP BY Id , getWeek ( TradeDate ) as
timeBucket ),
monthly ( Id , bucket , name , low , high , mean ) AS
( SELECT
Id ,
timeBucket ,
" monthly " ,
min ( ClosePrice ) ,
max ( ClosePrice ) ,
avg ( ClosePrice )
FROM Target
GROUP BY Id , getMonth ( TradeDate ) as
timeBucket ),
yearly (Id , bucket , name , low , high , mean ) AS
( SELECT
Id ,
timeBucket ,
" yearly " ,
min ( ClosePrice ) ,
max ( ClosePrice ) ,
avg ( ClosePrice )
FROM Target
GROUP BY Id , getYear ( TradeDate ) as
timeBucket )
SELECT
Id , bucket , name , low , high , mean
FROM
CONCATENATE ( weekly , monthly , yearly )
ASSUMING ASC Id , ASC name , ASC bucket
tests/udf2.a
-14
View File
@@ -1,14 +0,0 @@
AGGREGATION FUNCTION covariances(x, y, w){
static xmeans := 0., ymeans := 0., cnt := 0;
if (cnt < w)
{
xmeans += x;
ymeans += y;
cnt+=1;
}
else {
xmeans += (x - x.vec[cnt - w]) / w;
ymeans += (y - y.vec[cnt - w]) / w;
}
avg (( x.vec(x.len-w, x.len) - xmeans ) * (y.vec(y.len - w, y.len) - ymeans ))
}
tests/udf4.a
-32
View File
@@ -1,32 +0,0 @@
AGGREGATION FUNCTION covariances2(x, y, w){
xmeans := 0.;
ymeans := 0.;
l := _builtin_len;
if (w > l)
w := l;
elif (w > l + 2)
{
l := 3;
w := 4;
}
elif(w < 99){
l := 8;
}
elif(w<999)
w := 6;
else
l := l / 2;
for (i := 0, j:= 0; i < w; i := i+1) {
xmeans += x[i];
ymeans += y[i];
_builtin_ret[i] := avg (( x(l-w, l) - xmeans ) * (y(l - w, l) - ymeans ));
}
for (i := 0; i < l; i += 1)
{
xmeans += (x[i] - x[i - w]) / w;
ymeans += (y[i] - y[i - w]) / w;
_builtin_ret[i] := avg (( x(l-w, l) - xmeans ) * (y(l - w, l) - ymeans ));
}
Null
}