bill/AQuery
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Added documentation for trigger demo

Browse Source
This commit is contained in:
bill
2023-03-06 10:21:39 +08:00
parent c944b5dfcf
commit 726ef535ea
14 changed files with 91 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+3
View File
@@ -1,4 +1,7 @@
# AQuery++ Database
## News:
**Demo workflow for Triggers now available** See [**DEMO**](/demo/README.md)
## Introduction
AQuery++ Database is a cross-platform, In-Memory Column-Store Database that incorporates compiled query execution. (**Note**: If you encounter any problems, feel free to contact me via ys3540@nyu.edu)
build.py
+1
View File
@@ -27,6 +27,7 @@ class checksums:
aquery_config.build_driver +
compiler_name +
aquery_config.version_string
+ str(os.environ['AQ_DEBUG'] == '1')
)
for key in self.__dict__.keys():
try:
demo/README.md
+28
View File
@@ -0,0 +1,28 @@
# Triggers Demo
This folder contains a demo workflow for the two types of triggers.
- An interval-based trigger will be set up to execute a stored procedure `demoi` defined in [demo/putdata.cpp](/demo/putdata.cpp) that inserts a .csv file from `data/electricity` to the table `source` every 5 seconds.
- A Conditional trigger will be triggered by condition `democq` defined in [demo/query.cpp](/demo/query.cpp) that checks and returns true when more than 200 rows of data are inserted into table `source`. Once triggered, it will execute a stored procedure `democa` defined in [demo/democa.cpp](/demo/action.cpp) that trains the incremental random forest by the new data.
- See [demo/prep.a](/demo/prep.a) for parameters of the random forest.
## Run the demo
### Preparation
- Preprocess data
- Put `electricity` dataset to `/data/electricity_orig`
- Run `python3 rfdata_preproc.py` to generate .csv files to `data/electricity/`
- Use [demo/setup.sh](/demo/setup.sh) to
- setup stored procedures for this demo
- compile random forest user module used in this demo
- compile queries used in this demo
### Running the demo
- Run AQuery prompt `python3 prompt.py`
- Use Automated AQuery script in [demo/demo.aquery](/demo/demo.aquery) to execute the workflow. It does the following things in order:
- Register user module, create a new random forest by running [`f demo/prep.a`](/demo/prep.a)
- Register stored procedures.
- Create an Interval-based Trigger that executes payload `demoi` every 5 seconds
- Create a Conditional Trigger that executes payload `democa` whenever condition `democq` returns a true. While condition `democq` is tested every time new data is inserted to table `source`.
- Loads test data by running [demo/test.a](/demo/test.a)
- Use query `select predict(x) from test` to get predictions of the test data from current random forest.
- In AQuery prompt, an extra `exec` command after the query is needed to execute the query.
- Use query `select test(x, y) from test` will also calculate l2 error.
demo/demo.aquery
+6 -6
View File
@@ -3,15 +3,15 @@
f demo/prep.a
exec
procedure demoi run
procedure democq run
procedure democa run
procedure demoi load
procedure democq load
procedure democa load
create trigger t action demoi interval 15000
create trigger t action demoi interval 5000
exec
create trigger c on source action democa when democq
exec
# f demo/test.a
# exec
f demo/test.a
exec
demo/prep.a
+4 -3
View File
@@ -1,13 +1,14 @@
create table source(x vecdouble, y int64);
LOAD MODULE FROM "./libirf.so" FUNCTIONS (
newtree(height:int, f:int64, sparse:vecint, forget:double, maxf:int64, noclasses:int64, e:int) -> bool,
newtree(height:int, f:int64, sparse:vecint, forget:double, noclasses:int64, e:int) -> bool,
fit_inc(X:vecvecdouble, y:vecint64) -> bool,
predict(X:vecvecdouble) -> vecint
predict(X:vecvecdouble) -> vecint ,
test(X:vecvecdouble, y:vecint64) -> double
);
create table elec_sparse(v int);
insert into elec_sparse values (0), (1), (1), (1), (1), (1), (1);
select newtree(30, 7, elec_sparse.v, 0, 4, 2, 1) from elec_sparse
select newtree(30, 7, elec_sparse.v, 0.3, 2, 1) from elec_sparse
demo/setup.sh
+6
View File
@@ -0,0 +1,6 @@
#!/bin/sh
make -C ../sdk irf
mkdir ../procedures
cp demo*.aqp ../procedures
make
demo/test.a
+5 -1
View File
@@ -1,3 +1,7 @@
-- select predict(x) from test;
-- select test(x, y) from test;
create table test(x vecdouble, y int64);
load complex data infile "data/electricity/electricity872.csv" into table test fields terminated by ',' element terminated by ';';
select predict(x) from test
paper
+1 -1
Submodule paper updated: a61c3122c4...0d0ec21429
prompt.py
+1 -1
View File
@@ -238,7 +238,7 @@ class PromptState():
server_bin = 'server.bin' if server_mode == RunType.IPC else 'server.so'
wait_engine = lambda: None
wake_engine = lambda: None
get_storedproc = lambda : StoredProcedure()
get_storedproc = lambda *_: StoredProcedure()
set_ready = lambda: None
get_ready = lambda: None
server_status = lambda: False
reconstruct/ast.py
+7 -6
View File
@@ -1336,11 +1336,11 @@ class load(ast_node):
self.context.postproc_begin(self.postproc_fname)
table:TableInfo = self.context.tables_byname[node['table']]
self.sql = F"SELECT {', '.join([c.name for c in table.columns])} FROM {table.table_name};"
self.emit(self.sql+';\n')
self.context.sql_end()
length_name = 'len_' + base62uuid(6)
self.context.emitc(f'auto {length_name} = server->cnt;')
# self.sql = F"SELECT {', '.join([c.name for c in table.columns])} FROM {table.table_name};"
# self.emit(self.sql+';\n')
# self.context.sql_end()
# length_name = 'len_' + base62uuid(6)
# self.context.emitc(f'auto {length_name} = server->cnt;')
out_typenames = [t.type.cname for t in table.columns]
outtable_col_nameslist = ', '.join([f'"{c.name}"' for c in table.columns])
@@ -1353,7 +1353,8 @@ class load(ast_node):
for i, c in enumerate(table.columns):
c.cxt_name = 'c_' + base62uuid(6)
self.context.emitc(f'decltype(auto) {c.cxt_name} = {self.out_table}->get_col<{i}>();')
self.context.emitc(f'{c.cxt_name}.initfrom({length_name}, server->getCol({i}), "{table.columns[i].name}");')
self.context.emitc(f'{c.cxt_name}.init("{table.columns[i].name}");')
#self.context.emitc(f'{c.cxt_name}.initfrom({length_name}, server->getCol({i}), "{table.columns[i].name}");')
csv_reader_name = 'csv_reader_' + base62uuid(6)
col_types = [c.type.cname for c in table.columns]
col_tmp_names = ['tmp_'+base62uuid(8) for _ in range(len(table.columns))]
sdk/aquery.h
+4 -4
View File
@@ -13,14 +13,14 @@ enum Backend_Type {
BACKEND_MariaDB
};
struct Config{
struct Config {
int running, new_query, server_mode,
backend_type, has_dll, n_buffers;
int buffer_sizes[];
};
struct Session{
struct Statistic{
struct Session {
struct Statistic {
unsigned long long total_active;
unsigned long long cnt_object;
unsigned long long total_alloc;
@@ -102,7 +102,7 @@ namespace std {
}
#endif
struct vectortype_storage{
struct vectortype_storage {
void* container = nullptr;
unsigned int size = 0, capacity = 0;
vectortype_storage(void* container, unsigned int size, unsigned int capacity) :
sdk/irf.cpp
+20 -3
View File
@@ -19,7 +19,7 @@ DecisionTree *dt = nullptr;
RandomForest *rf = nullptr;
__AQEXPORT__(bool)
newtree(int ntree, long f, ColRef<int> sparse, double forget, long maxf, long nclasses, Evaluation e)
newtree(int ntree, long f, ColRef<int> sparse, double forget, long nclasses, Evaluation e)
{
if (sparse.size != f)
return false;
@@ -27,8 +27,6 @@ newtree(int ntree, long f, ColRef<int> sparse, double forget, long maxf, long nc
memcpy(X_cpy, sparse.container, f);
if (maxf < 0)
maxf = f;
// dt = new DecisionTree(f, X_cpy, forget, maxf, noclasses, e);
rf = new RandomForest(ntree, f, X_cpy, forget, nclasses, e, true);
return true;
@@ -100,3 +98,22 @@ predict(vector_type<vector_type<double>> v)
auto ret = vectortype_cstorage{.container = container, .size = 1, .capacity = 0};
return ret;
}
template <typename T>
constexpr T sq(const T x) {
return x * x;
}
__AQEXPORT__(double)
test(vector_type<vector_type<double>> x, vector_type<long> y) {
int result = 0;
printf("y_hat = (");
double err = 0.;
for (uint32_t i = 0; i < x.size; i++) {
//result[i] = dt->Test(v.container[i].container, dt->DTree);
result = int(rf->Test(x[i].container));
err += sq(result - y.container[i]);
printf("%d ", result);
}
puts(")");
printf("error: %lf\n", err/=double(x.size));
return err;
}
server/server.cpp
+1
View File
@@ -460,6 +460,7 @@ start:
case 'S': // save procedure
break;
case 'L': // load procedure
current_procedure.name = copy_lpstr(proc_name);
if (!load_proc_fromfile(current_procedure)) {
cxt->stored_proc.insert_or_assign(proc_name, current_procedure);
}
server/table_ext_monetdb.hpp
+4 -4
View File
@@ -42,14 +42,14 @@ void TableInfo<Ts ...>::monetdb_append_table(void* srv, const char* alt_name) {
uint32_t i = 0;
constexpr auto n_vecs = count_vector_type((tuple_type*)(0));
void* gc_vecs[1 + n_vecs];
puts("getcols...");
// puts("getcols...");
uint32_t cnt = 0;
const auto get_col = [&monetdbe_cols, &i, *this, &gc_vecs, &cnt](auto v) {
// printf("%d %d\n", i, (ColRef<void>*)v - colrefs);
monetdbe_cols[i++] = (monetdbe_column*)v->monetdb_get_col(gc_vecs, cnt);
};
(get_col((ColRef<Ts>*)(colrefs + i)), ...);
puts("getcols done");
//puts("getcols done");
// for(int i = 0; i < sizeof...(Ts); ++i)
// {
// printf("no:%d name: %s count:%d data: %p type:%d \n",
@@ -68,8 +68,8 @@ void TableInfo<Ts ...>::monetdb_append_table(void* srv, const char* alt_name) {
if (last_comma != static_cast<decltype(last_comma)>(-1)) {
create_table_str[last_comma] = ')';
Server* server = (Server*)srv;
puts("create table...");
puts(create_table_str.c_str());
// puts("create table...");
// puts(create_table_str.c_str());
server->exec(create_table_str.c_str());
if (!server->last_error) {
auto err = monetdbe_append(*((monetdbe_database*)server->server), "sys", alt_name, monetdbe_cols, sizeof...(Ts));