bill/AQuery
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

triggers

Browse Source
This commit is contained in:
bill
2023-02-20 20:12:42 +08:00
parent d71fc77006
commit 64d4e3dd9a
15 changed files with 1068 additions and 1016 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/arch-hybrid.ai
+915 -960
View File
File diff suppressed because one or more lines are too long
engine/orderby.py
+1 -1
View File
@@ -43,7 +43,7 @@ class orderby(ast_node):
def merge(self, node):
self.produce(node)
def finialize(self, references):
def finalize(self, references):
self.order = [ o for o in self.order if o.name in references ]
def result(self, sep:str = ','):
reconstruct/ast.py
+7 -6
View File
@@ -281,11 +281,11 @@ class projection(ast_node):
# self.into_stub = f'{{INTOSTUB{base62uuid(20)}}}'
# self.add(self.into_stub, '')
def finialize(astnode:ast_node):
def finalize(astnode:ast_node):
if(astnode is not None):
self.add(astnode.sql)
finialize(self.datasource)
finialize(self.where)
finalize(self.datasource)
finalize(self.where)
if self.group_node and not self.group_node.use_sp_gb:
self.add(self.group_node.sql)
@@ -1103,7 +1103,7 @@ class create_trigger(ast_node):
if 'interval' in node: # executed periodically from server
self.type = self.Type.Interval
self.interval = node['interval']
send_to_server(f'TI{self.trigger_name}{self.action_name}{self.interval}')
send_to_server(f'TI{self.trigger_name}\0{self.action_name}\0{self.interval}')
else: # executed from sql backend
self.type = self.Type.Callback
self.query_name = node['query']
@@ -1125,14 +1125,15 @@ class create_trigger(ast_node):
def execute(self):
from engine.utils import send_to_server
send_to_server(f'TC{self.query_name}{self.action_name}')
send_to_server(f'TC{self.query_name}\0{self.action_name}')
def remove(self):
from engine.utils import send_to_server
send_to_server(f'TR{self.trigger_name}')
class drop_trigger(ast_node):
name = 'create_trigger'
name = 'drop_trigger'
first_order = name
def produce(self, node):
...
reconstruct/storage.py
+5
View File
@@ -271,6 +271,11 @@ class Context:
val.table.triggers.remove(val)
val.remove()
def post_exec_triggers(self):
for t in self.triggers_active:
t.execute()
self.triggers_active.clear()
def abandon_postproc(self):
self.ccode = ''
self.finalize_query()
sdk/Makefile
+1 -1
View File
@@ -5,7 +5,7 @@ else
OPT_FLAGS = -g3 -D_DEBUG -fsanitize=leak -fsanitize=address
endif
example:
$(CXX) -shared -fPIC example.cpp aquery_mem.cpp -fno-semantic-interposition -Ofast -march=native -flto --std=c++1z -o ../test.so
$(CXX) -shared -fPIC example.cpp aquery_mem.cpp -fno-semantic-interposition -Ofast -march=native -flto --std=c++1z -L.. -laquery -o ../test.so
irf:
$(CXX) -shared -fPIC RF.cpp irf.cpp incrementalDecisionTree.cpp aquery_mem.cpp Evaluation.cpp -fno-semantic-interposition $(OPT_FLAGS) --std=c++1z -o ../libirf.so
all: example
sdk/aquery.h
+5 -3
View File
@@ -84,7 +84,9 @@ __AQEXPORT__(void) init_session(Context* cxt);
#ifdef _WIN32
#include <cstring>
#else
void* memcpy(void*, const void*, unsigned long long);
namespace std {
void* memcpy(void*, const void*, unsigned long long);
}
#endif
struct vectortype_storage{
@@ -95,7 +97,7 @@ struct vectortype_storage{
vectortype_storage() = default;
template <class Ty, template <typename> class VT>
vectortype_storage(const VT<Ty>& vt) {
memcpy(this, &vt, sizeof(vectortype_storage));
std::memcpy(this, &vt, sizeof(vectortype_storage));
}
};
struct ColRef_storage {
@@ -108,7 +110,7 @@ struct ColRef_storage {
ColRef_storage() = default;
template <class Ty, template <typename> class VT>
ColRef_storage(const VT<Ty>& vt) {
memcpy(this, &vt, sizeof(ColRef_storage));
std::memcpy(this, &vt, sizeof(ColRef_storage));
}
};
#endif
sdk/example.cpp
+1 -1
View File
@@ -1,6 +1,6 @@
#include "aquery.h"
// __AQ_NO_SESSION__
#include "../server/table.h"
#include "aquery.h"
__AQEXPORT__(ColRef_storage) mulvec(int a, ColRef<float> b){
return a * b;
sdk/irf.cpp
+26 -3
View File
@@ -3,8 +3,15 @@
// __AQ_NO_SESSION__
#include "../server/table.h"
#include "aquery.h"
#include "./server/gc.h"
__AQEXPORT__(void) __AQ_Init_GC__(Context* cxt) {
GC::gc_handle = static_cast<GC*>(cxt->gc);
GC::scratch_space = nullptr;
}
DecisionTree *dt = nullptr;
RandomForest *rf = nullptr;
@@ -20,7 +27,7 @@ newtree(int height, long f, ColRef<int> X, double forget, long maxf, long noclas
if (maxf < 0)
maxf = f;
dt = new DecisionTree(f, X_cpy, forget, maxf, noclasses, e);
rf = new RandomForest(height, f, X_cpy, forget, noclasses, e)
rf = new RandomForest(height, f, X_cpy, forget, noclasses, e);
return true;
}
@@ -42,6 +49,21 @@ newtree(int height, long f, ColRef<int> X, double forget, long maxf, long noclas
// return 1;
// }
__AQEXPORT__(bool)
fit_inc(vector_type<vector_type<double>> v, vector_type<long> res)
{
static uint32_t last_offset = 0;
double **data = (double **)malloc(v.size * sizeof(double *));
if(last_offset >= v.size)
last_offset = 0;
for (int i = last_offset; i < v.size; ++i)
data[i] = v.container[i].container;
rf->fit(data, res.container, v.size);
free(data);
return true;
}
__AQEXPORT__(bool)
fit(vector_type<vector_type<double>> v, vector_type<long> res)
{
@@ -53,14 +75,15 @@ fit(vector_type<vector_type<double>> v, vector_type<long> res)
return true;
}
__AQEXPORT__(vectortype_cstorage)
predict(vector_type<vector_type<double>> v)
{
int *result = (int *)malloc(v.size * sizeof(int));
for (long i = 0; i < v.size; i++)
for (uint32_t i = 0; i < v.size; i++)
//result[i] = dt->Test(v.container[i].container, dt->DTree);
result[i] = rf->Test(v.container, rf->DTrees);
result[i] = int(rf->Test(v[i].container));
auto container = (vector_type<int> *)malloc(sizeof(vector_type<int>));
container->size = v.size;
container->capacity = 0;
server/hasher.h
+6 -1
View File
@@ -7,6 +7,12 @@
#include "types.h"
// #include "robin_hood.h"
#include "unordered_dense.h"
template<typename Key, typename Val>
using aq_map = ankerl::unordered_dense::map<Key, Val>;
template<typename Key>
using aq_set = ankerl::unordered_dense::set<Key>;
// only works for 64 bit systems
namespace hasher_consts{
constexpr size_t _FNV_offset_basis = 14695981039346656037ULL;
@@ -19,7 +25,6 @@ inline size_t append_bytes(const unsigned char* _First) noexcept {
_Val ^= static_cast<size_t>(*_First);
_Val *= hasher_consts::_FNV_prime;
}
return _Val;
}
server/libaquery.cpp
-21
View File
@@ -58,27 +58,6 @@ void print<bool>(const bool&v, const char* delimiter){
}
template<class T>
T getInt(const char*& buf){
T ret = 0;
while(*buf >= '0' and *buf <= '9'){
ret = ret*10 + *buf - '0';
buf++;
}
return ret;
}
template<class T>
char* intToString(T val, char* buf){
while (val > 0){
*--buf = val%10 + '0';
val /= 10;
}
return buf;
}
void skip(const char*& buf){
while(*buf && (*buf >'9' || *buf < '0')) buf++;
}
server/libaquery.h
+35 -1
View File
@@ -9,6 +9,7 @@
#include <unordered_map>
#include <chrono>
#include <filesystem>
#include <cstring>
class aq_timer {
private:
std::chrono::high_resolution_clock::time_point now;
@@ -32,6 +33,27 @@ public:
#include "table.h"
template<class T = int>
T getInt(const char*& buf){
T ret = 0;
while(*buf >= '0' and *buf <= '9'){
ret = ret*10 + *buf - '0';
buf++;
}
return ret;
}
template<class T>
char* intToString(T val, char* buf){
while (val > 0){
*--buf = val%10 + '0';
val /= 10;
}
return buf;
}
enum Log_level {
LOG_INFO,
@@ -72,7 +94,9 @@ struct StoredProcedure {
const char* name;
void **__rt_loaded_modules;
};
struct Trigger;
struct IntervalBasedTriggerHost;
struct CallbackBasedTriggerHost;
struct Context {
typedef int (*printf_type) (const char *format, ...);
@@ -113,6 +137,9 @@ struct Context {
std::unordered_map<std::string, void*> tables;
std::unordered_map<std::string, uColRef *> cols;
std::unordered_map<std::string, StoredProcedure> stored_proc;
std::unordered_map<std::string, Trigger> triggers;
IntervalBasedTriggerHost *it_host;
CallbackBasedTriggerHost *ct_host;
};
@@ -176,6 +203,13 @@ inline void AQ_ZeroMemory(_This_Struct& __val) {
memset(&__val, 0, sizeof(_This_Struct));
}
template <typename _This_Type>
inline _This_Type* AQ_DupObject(_This_Type* __val) {
auto ret = (_This_Type*)(malloc(sizeof(_This_Type)));
memcpy(ret, __val, sizeof(_This_Type));
return ret;
}
#ifdef __USE_STD_SEMAPHORE__
#include <semaphore>
class A_Semaphore {
server/server.cpp
+34 -1
View File
@@ -559,6 +559,37 @@ start:
}
}
break;
case 'T': // triggers
{
switch(n_recvd[i][1]){
case 'I': // register interval based trigger
{
const char* action_name = n_recvd[i] + 2;
while(*action_name++);
if(auto p = get_procedure(cxt, action_name); p.name == nullptr)
printf("Invalid action name: %s\n", action_name);
else {
auto action = AQ_DupObject(&p);
const char* interval = action_name;
while(*interval++);
const auto i_interval = getInt<uint32_t>(interval);
cxt->it_host->add_trigger(n_recvd[i] + 2, action, i_interval);
}
}
break;
case 'C': // activate callback based trigger
break;
case 'R': // remove trigger
{
cxt->it_host->remove_trigger(n_recvd[i] + 2);
}
break;
default:
printf("Corrupted message from prompt: %s\n", n_recvd[i]);
break;
}
}
break;
}
}
@@ -618,7 +649,7 @@ int launcher(int argc, char** argv){
str = std::string("cd ") + pwd + std::string("&& python3 ./prompt.py ") + str;
return system(str.c_str());
}
#if !defined(TESTMAIN) && !( defined(_MSC_VER) && defined(_DEBUG) )
#if true || !defined(TESTMAIN) && !( defined(_MSC_VER) && defined(_DEBUG) )
extern "C" int __DLLEXPORT__ main(int argc, char** argv) {
#ifdef __AQ_BUILD_LAUNCHER__
return launcher(argc, argv);
@@ -631,6 +662,8 @@ extern "C" int __DLLEXPORT__ main(int argc, char** argv) {
#ifdef THREADING
auto tp = new ThreadPool();
cxt->thread_pool = tp;
cxt->it_host = new IntervalBasedTriggerHost(tp);
cxt->ct_host = new CallbackBasedTriggerHost(tp);
#endif
const char* shmname;
server/threading.cpp
+18 -6
View File
@@ -156,16 +156,17 @@ bool ThreadPool::busy(){
IntervalBasedTriggerHost::IntervalBasedTriggerHost(ThreadPool* tp){
this->tp = tp;
this->triggers = new vector_type<IntervalBasedTrigger>;
this->triggers = new aq_map<std::string, IntervalBasedTrigger>;
trigger_queue_lock = new mutex();
this->now = std::chrono::high_resolution_clock::now().time_since_epoch().count();
}
void IntervalBasedTriggerHost::add_trigger(StoredProcedure *p, uint32_t interval) {
void IntervalBasedTriggerHost::add_trigger(const char* name, StoredProcedure *p, uint32_t interval) {
auto tr = IntervalBasedTrigger{.interval = interval, .time_remaining = 0, .sp = p};
auto vt_triggers = static_cast<vector_type<IntervalBasedTrigger> *>(this->triggers);
auto vt_triggers = static_cast<aq_map<std::string, IntervalBasedTrigger> *>(this->triggers);
trigger_queue_lock->lock();
vt_triggers->emplace_back(tr);
vt_triggers->emplace(name, tr);
//(*vt_triggers)[name] = tr;
trigger_queue_lock->unlock();
}
@@ -173,9 +174,9 @@ void IntervalBasedTriggerHost::tick() {
const auto current_time = std::chrono::high_resolution_clock::now().time_since_epoch().count();
const auto delta_t = static_cast<uint32_t>((current_time - now) / 1000000); // miliseconds precision
now = current_time;
auto vt_triggers = static_cast<vector_type<IntervalBasedTrigger> *>(this->triggers);
auto vt_triggers = static_cast<aq_map<std::string, IntervalBasedTrigger> *>(this->triggers);
trigger_queue_lock->lock();
for(auto& t : *vt_triggers) {
for(auto& [_, t] : vt_triggers->values()) {
if(t.tick(delta_t)) {
payload_t payload;
payload.f = execTriggerPayload;
@@ -187,6 +188,11 @@ void IntervalBasedTriggerHost::tick() {
trigger_queue_lock->unlock();
}
void IntervalBasedTriggerHost::remove_trigger(const char* name) {
auto vt_triggers = static_cast<aq_map<std::string, IntervalBasedTrigger> *>(this->triggers);
vt_triggers->erase(name);
}
void IntervalBasedTrigger::reset() {
time_remaining = interval;
}
@@ -201,3 +207,9 @@ bool IntervalBasedTrigger::tick(uint32_t delta_t) {
time_remaining = time_remaining - curr_dt;
return ret;
}
CallbackBasedTriggerHost::CallbackBasedTriggerHost(ThreadPool *tp) {
this->tp = tp;
}
void CallbackBasedTriggerHost::tick() {}
server/threading.h
+10 -7
View File
@@ -2,6 +2,8 @@
#define _AQ_THREADING_H
#include <stdint.h>
#include <thread>
#include <mutex>
typedef int(*payload_fn_t)(void*);
struct payload_t{
@@ -39,12 +41,11 @@ private:
};
#include <thread>
#include <mutex>
class A_Semphore;
class TriggerHost {
protected:
public:
void* triggers;
std::thread* handle;
ThreadPool *tp;
@@ -52,14 +53,15 @@ protected:
std::mutex* trigger_queue_lock;
virtual void tick() = 0;
public:
TriggerHost() = default;
virtual ~TriggerHost() = default;
};
struct StoredProcedure;
struct IntervalBasedTrigger {
struct Trigger{};
struct IntervalBasedTrigger : Trigger {
uint32_t interval; // in milliseconds
uint32_t time_remaining;
StoredProcedure* sp;
@@ -70,8 +72,8 @@ struct IntervalBasedTrigger {
class IntervalBasedTriggerHost : public TriggerHost {
public:
explicit IntervalBasedTriggerHost(ThreadPool *tp);
void add_trigger(StoredProcedure* stored_procedure, uint32_t interval);
void remove_trigger(uint32_t tid);
void add_trigger(const char* name, StoredProcedure* stored_procedure, uint32_t interval);
void remove_trigger(const char* name);
private:
unsigned long long now;
void tick() override;
@@ -79,6 +81,7 @@ private:
class CallbackBasedTriggerHost : public TriggerHost {
public:
explicit CallbackBasedTriggerHost(ThreadPool *tp);
void add_trigger();
private:
void tick() override;
server/vector_type.hpp
+4 -4
View File
@@ -18,7 +18,7 @@
#include "gc.h"
#pragma pack(push, 1)
struct vectortype_cstorage{
struct vectortype_cstorage {
void* container;
unsigned int size, capacity;
};
@@ -123,7 +123,7 @@ public:
_copy(vt);
return *this;
}
vector_type<_Ty>& operator =(vector_type<_Ty>&& vt) {
vector_type<_Ty>& operator =(vector_type<_Ty>&& vt) noexcept {
_move(std::move(vt));
return *this;
}
@@ -139,10 +139,10 @@ public:
return *this;
}
inline std::unordered_set<value_t> distinct_common(){
inline std::unordered_set<value_t> distinct_common() {
return std::unordered_set<value_t>(container, container + size);
}
vector_type<_Ty>& distinct_inplace(){
vector_type<_Ty>& distinct_inplace() {
uint32_t i = 0;
for(const auto& v : distinct_common()){
container[i++] = v;