More Date/Time, Complex expr on groupby, bug fixes

.gitignore

@@ -1,3 +1,4 @@
+*.dSYM
 test.lib
 test.exp
 *.pdb

README.md

@@ -13,7 +13,7 @@ AQuery++ Database is a cross-platform, In-Memory Column-Store Database that inco
 - Backend of AQuery++ Compiler generates target code dependent on the Execution Engine. It can either be the C++ code for AQuery Execution Engine or sql and C++ post-processor for Hybrid Engine or k9 for the k9 Engine.
 ### Execution Engines
 - AQuery++ supports different execution engines thanks to the decoupled compiler structure.
-- AQuery Execution Engine: executes query by compiling the query plan to C++ code. Doesn't support joins and udf functions. 
+- AQuery Execution Engine: executes queries by compiling the query plan to C++ code. Doesn't support joins and udf functions. 
 - Hybrid Execution Engine: decouples the query into two parts. The sql-compliant part is executed by an Embedded version of Monetdb and everything else is executed by a post-process module which is generated by AQuery++ Compiler in C++ and then compiled and executed.
 - K9 Execution Engine: (discontinued).
   
@@ -42,15 +42,14 @@ AQuery++ Database is a cross-platform, In-Memory Column-Store Database that inco
 
 - [x] User Module load syntax parsing (fn definition/registration)
 - [x] User Module initialize location
-   -> User Module test
+- [x] User Module test
    -> Interval based triggers
-   -> Optimize Compilation Process, using static libraries, hot reloading server binary
+- [x] Optimize Compilation Process, using static libraries, hot reloading server binary
 - [x] Bug fixes: type deduction misaligned in Hybrid Engine
    -> Investigation: Using postproc only for q1 in Hybrid Engine (make is_special always on)
 - [x] Limitation: putting ColRefs back to monetdb. 
-- [ ] Limitation: String operations and Date/Time data type. 
 - [ ] C++ Meta-Programming: Eliminate template recursions as much as possible.
-- [ ] Limitation: Date and Time, String operations, Funcs in groupby agg.
+- [x] Limitation: Date and Time, String operations, Funcs in groupby agg.
   
 # Installation
 ## Requirements

build.py

@@ -15,10 +15,13 @@ class checksums:
     pch_hpp_gch : Optional[Union[bytes, bool]] = None
     server : Optional[Union[bytes, bool]] = None
     sources : Union[Dict[str, bytes], bool] = None
+    env : str = ''
     def calc(self, libaquery_a = 'libaquery.a' , 
                 pch_hpp_gch = 'server/pch.hpp.gch', 
                 server = 'server.so'
         ):
+        from platform import machine
+        self.env = aquery_config.os_platform + machine() + aquery_config.build_driver
         for key in self.__dict__.keys():
             try:
                 with open(eval(key), 'rb') as file:
@@ -38,7 +41,10 @@ class checksums:
         ret = checksums()
         for key in self.__dict__.keys():
             try:
-                ret.__dict__[key] = self.__dict__[key] != __o.__dict__[key]
+                ret.__dict__[key] = (
+                    self.__dict__[key] and __o.__dict__[key] and
+                    self.__dict__[key] != __o.__dict__[key]
+                )
             except KeyError:
                 ret.__dict__[key] = True
         return ret
@@ -47,7 +53,10 @@ class checksums:
         ret = checksums()
         for key in self.__dict__.keys():
             try:
-                ret.__dict__[key] = self.__dict__[key] == __o.__dict__[key]
+                ret.__dict__[key] = (
+                    not (self.__dict__[key] and __o.__dict__[key]) or
+                    self.__dict__[key] == __o.__dict__[key]
+                )
             except KeyError:
                 ret.__dict__[key] = False
         return ret
@@ -55,7 +64,9 @@ class checksums:
         
         
 class build_manager:
-    sourcefiles = ['server/server.cpp', 'server/io.cpp', 
+    sourcefiles = [
+                   'build.py', 
+                   'server/server.cpp', 'server/io.cpp',  
                    'server/monetdb_conn.cpp', 'server/threading.cpp',
                    'server/winhelper.cpp'
                    ]

datagen.cpp

@@ -0,0 +1,110 @@
+#define _CRT_SECURE_NO_WARNINGS
+#include <cstdio>
+#include <cstring>
+#include <random>
+#include <vector>
+using uniform = std::uniform_int_distribution<unsigned int>;
+std::random_device rd;
+std::mt19937_64 engine = std::mt19937_64(rd());
+uniform ui, g_quantity = uniform(100, 10000);
+
+int getInt(char*& buf) {
+    while ((*buf < '0' || *buf > '9') && *buf != '.' && *buf) ++buf;
+    int res = 0;
+    while (*buf >= '0' && *buf <= '9')
+        res = res * 10 + *buf++ - '0';
+    return res;
+}
+
+float getFloat(char*& buf) {
+    float res = static_cast<float>(getInt(buf));
+    if (*buf == '.') {
+        ++buf;
+        float frac = 1.f;
+        while (*buf >= '0' && *buf <= '9')
+            res += (*buf++ - '0') * (frac /= 10.f);
+    }
+    return res;
+}
+
+void permutation(int *v, int n) {
+    for (int i = 0; i < n; ++i)
+    { // ensures each element have 1/N chance of being at place i
+        int j = i + ui(engine) % static_cast<unsigned int>(n - i);
+        int t = v[i];
+        v[i] = v[j];
+        v[j] = t;
+    }
+}
+
+int main(int argc, char* argv[])
+{
+    using std::vector;
+    float frac = .3;
+    int N = 70000;
+    int n_rows = 10000000;
+	if (argc > 2) {
+        frac = getFloat(argv[1]);
+        N = getInt(argv[2]);
+        n_rows = getInt(argv[3]);
+	}
+    else {
+        printf("No parameter supplied. Use default frac=%f, N=%d? [Y/n]\n", frac, N);
+        char buf[4096]; fgets(buf, 4095, stdin); 
+        if((buf[0] != 'y' && buf[0] !='Y') && buf[0] != '\n') { 
+            const auto &getParams = [&](){ puts("Type: frac N [ENTER]");
+                for(int i = 0; i < 4096; ++i) if(buf[i] == '\n') {buf[i] = 0; break;}
+                char* _buf = buf; frac = getFloat(_buf); N = getInt(_buf); n_rows=getInt(_buf);
+            }; getParams();
+            while ((frac <= 0 || frac >= 1) && N < 1&&n_rows<1) { fgets(buf, 4095, stdin); getParams(); }
+        }
+    }
+    printf("\nfrac = %f, N = %d\n", frac, N);
+    vector<int> v_lens;
+    int curr_len = N;
+    int s_len = 0;
+    while (curr_len >= 1) {
+        curr_len *= frac;
+        v_lens.push_back(s_len);
+        s_len += curr_len;
+    }
+    int* lens = v_lens.data();
+    int llens = v_lens.size();
+    for (int i = 0; i < llens; ++i)
+        lens[i] = s_len - lens[i];
+    lens[0] = s_len;
+    int *p = new int[lens[0] + N];
+    for (int i = 0; i < N; ++i) p[lens[0] + i] = i + 1;
+    permutation(p + lens[0], N);
+    for (int i = 1; i < llens; ++i)
+        memmove(p + lens[i], p + lens[0], (lens[i - 1] - lens[i]) * sizeof(int));
+    permutation(p, lens[0] + N);
+    // for (int i = 0; i < lens[0] + N; ++i) printf("%d ", p[i]);
+    FILE* fp = fopen("trade.csv", "w");
+    int* last_price = new int[N];
+    memset(last_price, -1, sizeof(int) * N);
+    fprintf(fp, "stocksymbol, time, quantity, price\n");
+    struct data{int ss, t, q, p;} *d = new data[n_rows];
+
+    using std::min; using std::max;
+    for (int i = 0; i < n_rows; ++i) {
+        int ss = p[ui(engine) % (lens[0]+N)];
+        int current_price = last_price[ss - 1];
+        if (current_price < 0)
+            current_price = ui(engine) % 451 + 50;
+        else {
+            int l = max(50, current_price - 5);
+            int h = min(500, current_price + 5);
+            unsigned int interval = max(0, current_price - l) + max(0, h - current_price);
+            int new_price = l + ui(engine) % interval; 
+            if (new_price >= current_price) //make sure every candidate have equal chance of being chosen
+                ++new_price;
+            current_price = new_price;
+        }
+        d[i]= {ss, i+1, (int)g_quantity(engine), current_price};
+        fprintf(fp, "s%d, %d, %d, %d\n", d[i].ss, d[i].t, d[i].q, d[i].p);
+        last_price[ss - 1] = current_price;
+    }
+    fclose(fp);
+    return 0;
+}

engine/ddl.py

@@ -55,7 +55,7 @@ class create_table(ast_node):
                     self.lineage = f"{lineage_var}.rid"
                 for i, c in enumerate(tbl.columns):
                     scanner.add(f'{c.cxt_name}.init("{c.name}");', "init")
-                    scanner.add(f"{c.cxt_name} = {self.cexprs[i](scanner.it_ver)};")
+                    scanner.add(f"{c.cxt_name} = {self.cexprs[i](scanner.it_var)};")
                     
 class insert(ast_node):
     name = 'insert'

engine/groupby.py

@@ -43,7 +43,7 @@ class groupby(ast_node):
             f'transTypes<{self.group_type}, hasher>> {self.group};')
         self.n_grps = len(node)
         self.scanner = scan(self, self.datasource, expr.toCExpr(first_col)()+'.size')
-        self.scanner.add(f'{self.group}[forward_as_tuple({g_contents(self.scanner.it_ver)})].emplace_back({self.scanner.it_ver});')
+        self.scanner.add(f'{self.group}[forward_as_tuple({g_contents(self.scanner.it_var)})].emplace_back({self.scanner.it_var});')
 
         
     def consume(self, _):
@@ -54,7 +54,7 @@ class groupby(ast_node):
     def deal_with_assumptions(self, assumption:assumption, out:TableInfo):
         gscanner = scan(self, self.group)
         val_var = 'val_'+base62uuid(7)
-        gscanner.add(f'auto &{val_var} = {gscanner.it_ver}.second;')
+        gscanner.add(f'auto &{val_var} = {gscanner.it_var}.second;')
         gscanner.add(f'{self.datasource.cxt_name}->order_by<{assumption.result()}>(&{val_var});')
         gscanner.finalize()
         
@@ -63,8 +63,8 @@ class groupby(ast_node):
         key_var = 'key_'+base62uuid(7)
         val_var = 'val_'+base62uuid(7)
         
-        gscanner.add(f'auto &{key_var} = {gscanner.it_ver}.first;')
+        gscanner.add(f'auto &{key_var} = {gscanner.it_var}.first;')
-        gscanner.add(f'auto &{val_var} = {gscanner.it_ver}.second;')
+        gscanner.add(f'auto &{val_var} = {gscanner.it_var}.second;')
         gscanner.add(';\n'.join([f'{out.columns[i].reference()}.emplace_back({ce(x=val_var, y=key_var)})' for i, ce in enumerate(cexprs)])+';')
         
         gscanner.finalize()

engine/scan.py

@@ -20,25 +20,25 @@ class scan(ast_node):
         self.mode = None
         self.filters = []
         scan_vars = set(s.it_var for s in self.context.scans)
-        self.it_ver = 'i' + base62uuid(2)
+        self.it_var = 'i' + base62uuid(2)
-        while(self.it_ver in scan_vars):
+        while(self.it_var in scan_vars):
-            self.it_ver = 'i' + base62uuid(6)
+            self.it_var = 'i' + base62uuid(6)
         self.parent.context.scans.append(self)
     def produce(self, node):
         if type(node) is ColRef:
             self.colref = node
             if self.size is None:
                 self.mode = ["col", node.table]
-                self.start += f'for ({self.const}auto& {self.it_ver} : {node.reference()}) {{\n'
+                self.start += f'for ({self.const}auto& {self.it_var} : {node.reference()}) {{\n'
             else:
                 self.mode = ["idx", node.table]
-                self.start += f"for (uint32_t {self.it_ver} = 0; {self.it_ver} < {node.reference()}.size; ++{self.it_ver}){{\\n"
+                self.start += f"for (uint32_t {self.it_var} = 0; {self.it_var} < {node.reference()}.size; ++{self.it_var}){{\\n"
         elif type(node) is str:
             self.mode = ["idx", None]
-            self.start+= f'for({self.const}auto& {self.it_ver} : {node}) {{\n'
+            self.start+= f'for({self.const}auto& {self.it_var} : {node}) {{\n'
         else:
             self.mode = ["idx", node] # Node is the TableInfo
-            self.start += f"for (uint32_t {self.it_ver} = 0; {self.it_ver} < {self.size}; ++{self.it_ver}){{\n"
+            self.start += f"for (uint32_t {self.it_var} = 0; {self.it_var} < {self.size}; ++{self.it_var}){{\n"
             
     def add(self, stmt, position = "body"):
         if position == "body":
@@ -95,5 +95,5 @@ class filter(ast_node):
         if self.scanner is None:
             self.scanner = scan(self, self.datasource, self.datasource.get_size())
         self.expr = expr(self, self.modified_node)
-        self.scanner.filters.append(f'if ({self.expr.cexpr(self.scanner.it_ver)}) {{\n')
+        self.scanner.filters.append(f'if ({self.expr.cexpr(self.scanner.it_var)}) {{\n')
     

engine/types.py

@@ -15,6 +15,7 @@ class Types:
         self.priority : int= 0
         self.cast_to_dict = dict()
         self.cast_from_dict = dict()
+        
     def __init__(self, priority = 0, *, 
                  name = None, cname = None, sqlname = None,
                  ctype_name = None, null_value = None,
@@ -87,6 +88,9 @@ class TypeCollection:
 type_table = dict()
 AnyT = Types(-1)
 LazyT = Types(240, name = 'Lazy', cname = '', sqlname = '', ctype_name = '')
+LazyT = Types(200, name = 'DATE', cname = 'types::date_t', sqlname = 'DATE', ctype_name = 'types::ADATE')
+LazyT = Types(201, name = 'TIME', cname = 'types::time_t', sqlname = 'TIME', ctype_name = 'types::ATIME')
+LazyT = Types(202, name = 'TIMESTAMP', cname = 'types::timestamp_t', sqlname = 'TIMESTAMP', ctype_name = 'ATIMESTAMP')
 DoubleT = Types(17, name = 'double', cname='double', sqlname = 'DOUBLE', is_fp = True)
 LDoubleT = Types(18, name = 'long double', cname='long double', sqlname = 'LDOUBLE', is_fp = True)
 FloatT = Types(16, name = 'float', cname = 'float', sqlname = 'REAL', 
@@ -102,7 +106,7 @@ ULongT = Types(8, name = 'uint64', sqlname = 'UINT64', fp_type=DoubleT)
 UIntT = Types(7, name = 'uint32', sqlname = 'UINT32', long_type=ULongT, fp_type=FloatT)
 UShortT = Types(6, name = 'uint16', sqlname = 'UINT16', long_type=ULongT, fp_type=FloatT)
 UByteT = Types(5, name = 'uint8', sqlname = 'UINT8', long_type=ULongT, fp_type=FloatT)
-StrT = Types(200, name = 'str', cname = 'const char*', sqlname='VARCHAR', ctype_name = 'types::STRING')
+StrT = Types(200, name = 'str', cname = 'const char*', sqlname='VARCHAR', ctype_name = 'types::ASTR')
 VoidT = Types(200, name = 'void', cname = 'void', sqlname='Null', ctype_name = 'types::None')
 
 class VectorT(Types):
@@ -212,6 +216,11 @@ def logical(*_ : Types) -> Types:
     return ByteT
 def int_return(*_ : Types) -> Types:
     return IntT
+def long_return(*_ : Types) -> Types:
+    return LongT.long_type
+def lfp_return(*_ : Types) -> Types:
+    return LongT.fp_type
+
 def as_is (t: Types) -> Types:
     return t
 
@@ -262,8 +271,12 @@ opnot = OperatorBase('not', 1, logical, cname = '!', sqlname = 'NOT', call = una
 # functional
 fnmax = OperatorBase('max', 1, as_is, cname = 'max', sqlname = 'MAX', call = fn_behavior)
 fnmin = OperatorBase('min', 1, as_is, cname = 'min', sqlname = 'MIN', call = fn_behavior)
-fnsum = OperatorBase('sum', 1, ext(auto_extension), cname = 'sum', sqlname = 'SUM', call = fn_behavior)
+fndeltas = OperatorBase('deltas', 1, as_is, cname = 'deltas', sqlname = 'DELTAS', call = fn_behavior)
-fnavg = OperatorBase('avg', 1, fp(ext(auto_extension)), cname = 'avg', sqlname = 'AVG', call = fn_behavior)
+fnlast = OperatorBase('last', 1, as_is, cname = 'last', sqlname = 'LAST', call = fn_behavior)
+#fnsum = OperatorBase('sum', 1, ext(auto_extension), cname = 'sum', sqlname = 'SUM', call = fn_behavior)
+#fnavg = OperatorBase('avg', 1, fp(ext(auto_extension)), cname = 'avg', sqlname = 'AVG', call = fn_behavior)
+fnsum = OperatorBase('sum', 1, long_return, cname = 'sum', sqlname = 'SUM', call = fn_behavior)
+fnavg = OperatorBase('avg', 1, lfp_return, cname = 'avg', sqlname = 'AVG', call = fn_behavior)
 fnmaxs = OperatorBase('maxs', [1, 2], ty_clamp(as_is, -1), cname = 'maxs', sqlname = 'MAXS', call = windowed_fn_behavor)
 fnmins = OperatorBase('mins', [1, 2], ty_clamp(as_is, -1), cname = 'mins', sqlname = 'MINS', call = windowed_fn_behavor)
 fnsums = OperatorBase('sums', [1, 2], ext(ty_clamp(auto_extension, -1)), cname = 'sums', sqlname = 'SUMS', call = windowed_fn_behavor)
@@ -295,7 +308,7 @@ builtin_unary_logical = _op_make_dict(opnot)
 builtin_unary_arith = _op_make_dict(opneg)
 builtin_unary_special = _op_make_dict(spnull)
 builtin_cstdlib = _op_make_dict(fnsqrt, fnlog, fnsin, fncos, fntan, fnpow)
-builtin_func = _op_make_dict(fnmax, fnmin, fnsum, fnavg, fnmaxs, fnmins, fnsums, fnavgs, fncnt)
+builtin_func = _op_make_dict(fnmax, fnmin, fnsum, fnavg, fnmaxs, fnmins, fndeltas, fnlast, fnsums, fnavgs, fncnt)
 user_module_func = {}
 builtin_operators : Dict[str, OperatorBase] = {**builtin_binary_arith, **builtin_binary_logical, 
     **builtin_unary_arith, **builtin_unary_logical, **builtin_unary_special, **builtin_func, **builtin_cstdlib, 

prompt.py

@@ -424,6 +424,15 @@ def prompt(running = lambda:True, next = lambda:input('> '), state = None):
                 rm(state)
                 exit()
             elif q == 'r': # build and run
+                if state.server_mode == RunType.Threaded:
+                    qs = [ctypes.c_char_p(bytes(q, 'utf-8')) for q in cxt.queries if len(q)]
+                    sz = len(qs)
+                    payload = (ctypes.c_char_p*sz)(*qs)
+                    state.payload = payload
+                    try:
+                        state.send(sz, payload)
+                    except TypeError as e:
+                        print(e)
                 if subprocess.call(['make', 'snippet']) == 0:
                     state.set_ready()
                 continue
@@ -526,5 +535,11 @@ if __name__ == '__main__':
         elif any([s in nextcmd for s in thread_string]):
             server_mode = RunType.Threaded
     
+    if check_param(['-r', '--rebuild']):
+        try:
+            os.remove('./.cached')
+        except FileNotFoundError:
+            pass
+    
     prompt(state=state)
     

reconstruct/ast.py

@@ -88,6 +88,8 @@ class projection(ast_node):
         else:
             self.where = None    
 
+        if 'groupby' in node:
+            self.context.special_gb = groupby.check_special(self, node['groupby'])
 
     def consume(self, node):
         # deal with projections
@@ -191,6 +193,8 @@ class projection(ast_node):
         else:
             self.outfile = None
         
+        # reset special_gb in case of subquery.
+        self.context.special_gb = False
         if self.parent is None:
             self.emit(self.sql+';\n')
         else: 
@@ -333,30 +337,28 @@ class scan(ast_node):
         foreach = auto()
         
     name = 'scan'
-    def __init__(self, parent: "ast_node", node, loop_style = 'for', context: Context = None, const = False):
+    def __init__(self, parent: "ast_node", node, loop_style = 'for', context: Context = None, const = False, it_name = None):
+        self.it_var = it_name
         self.const = "const " if const else ""
         self.loop_style = loop_style
         super().__init__(parent, node, context)
         
     def init(self, _):
+        self.it_var = self.context.get_scan_var() if not self.it_var else self.it_var
         self.datasource = self.context.datasource
         self.initializers = ''
         self.start = ''
         self.front = ''
         self.body = ''
         self.end = '}'
-        scan_vars = set(s.it_var for s in self.context.scans)
-        self.it_ver = 'i' + base62uuid(2)
-        while(self.it_ver in scan_vars):
-            self.it_ver = 'i' + base62uuid(6)
         self.parent.context.scans.append(self)
         
     def produce(self, node):
         if self.loop_style == 'for_each':
             self.colref = node
-            self.start += f'for ({self.const}auto& {self.it_ver} : {node}) {{\n'
+            self.start += f'for ({self.const}auto& {self.it_var} : {node}) {{\n'
         else:
-            self.start += f"for (uint32_t {self.it_ver} = 0; {self.it_ver} < {node}; ++{self.it_ver}){{\n"
+            self.start += f"for (uint32_t {self.it_var} = 0; {self.it_var} < {node}; ++{self.it_var}){{\n"
             
     def add(self, stmt, position = "body"):
         if position == "body":
@@ -387,25 +389,30 @@ class groupby_c(ast_node):
         g_contents = ''
         g_contents_list = []
         first_col = ''
-        
+        scanner_itname = self.context.get_scan_var()
         for g in self.glist:
             e = expr(self, g[0].node, c_code=True)
             g_str = e.eval(c_code = True, y = lambda c: self.proj.pyname2cname[c])
             # if v is compound expr, create tmp cols
-            if e.is_ColExpr:
+            if not e.is_ColExpr:
+                self.context.headers.add('"./server/aggregations.h"')                    
                 tmpcol = 't' + base62uuid(7)
                 self.context.emitc(f'auto {tmpcol} = {g_str};')
                 e = tmpcol
+            else:
+                e = g_str
             g_contents_list.append(e)
         first_col = g_contents_list[0]
         g_contents_decltype = [f'decays<decltype({c})::value_t>' for c in g_contents_list]
-        g_contents = ','.join(g_contents_list)
+        g_contents = ', '.join(
+            [f'{c}[{scanner_itname}]' for c in g_contents_list]
+        )
         self.context.emitc(f'typedef record<{",".join(g_contents_decltype)}> {self.group_type};')
         self.context.emitc(f'unordered_map<{self.group_type}, vector_type<uint32_t>, '
             f'transTypes<{self.group_type}, hasher>> {self.group};')
         self.n_grps = len(self.glist)
-        self.scanner = scan(self, first_col + '.size')
+        self.scanner = scan(self, first_col + '.size', it_name=scanner_itname)
-        self.scanner.add(f'{self.group}[forward_as_tuple({g_contents}[{self.scanner.it_ver}])].emplace_back({self.scanner.it_ver});')
+        self.scanner.add(f'{self.group}[forward_as_tuple({g_contents})].emplace_back({self.scanner.it_var});')
 
     def consume(self, _):
         self.scanner.finalize()
@@ -413,7 +420,7 @@ class groupby_c(ast_node):
     # def deal_with_assumptions(self, assumption:assumption, out:TableInfo):
     #     gscanner = scan(self, self.group)
     #     val_var = 'val_'+base62uuid(7)
-    #     gscanner.add(f'auto &{val_var} = {gscanner.it_ver}.second;')
+    #     gscanner.add(f'auto &{val_var} = {gscanner.it_var}.second;')
     #     gscanner.add(f'{self.datasource.cxt_name}->order_by<{assumption.result()}>(&{val_var});')
     #     gscanner.finalize()
         
@@ -422,8 +429,8 @@ class groupby_c(ast_node):
         key_var = 'key_'+base62uuid(7)
         val_var = 'val_'+base62uuid(7)
         
-        gscanner.add(f'auto &{key_var} = {gscanner.it_ver}.first;', position = 'front')
+        gscanner.add(f'auto &{key_var} = {gscanner.it_var}.first;', position = 'front')
-        gscanner.add(f'auto &{val_var} = {gscanner.it_ver}.second;', position = 'front')
+        gscanner.add(f'auto &{val_var} = {gscanner.it_var}.second;', position = 'front')
         len_var = None
         def define_len_var():
             nonlocal len_var
@@ -444,6 +451,14 @@ class groupby_c(ast_node):
             else:
                 return f'get<{var}>({key_var})'
         
+        def get_var_names_ex (varex : expr):
+            sql_code = varex.eval(c_code = False)
+            if (sql_code in var_table):
+                return get_var_names(sql_code)
+            else:
+                return varex.eval(c_code=True, y = get_var_names, 
+                        materialize_builtin = materialize_builtin)
+                
         for ce in cexprs:
             ex = ce[1]
             materialize_builtin = {}
@@ -457,7 +472,7 @@ class groupby_c(ast_node):
                     materialize_builtin['_builtin_ret'] = f'{ce[0]}.back()'
                     gscanner.add(f'{ex.eval(c_code = True, y=get_var_names, materialize_builtin = materialize_builtin)};\n')
                     continue
-            gscanner.add(f'{ce[0]}.emplace_back({ex.eval(c_code = True, y=get_var_names, materialize_builtin = materialize_builtin)});\n')
+            gscanner.add(f'{ce[0]}.emplace_back({get_var_names_ex(ex)});\n')
         
         gscanner.finalize()
         
@@ -466,6 +481,17 @@ class groupby_c(ast_node):
             
 class groupby(ast_node):
     name = 'group by'
+    
+    @staticmethod
+    def check_special(parent, node):
+        node = enlist(node)
+        for g in node:
+            if ('value' in g and 
+                expr(parent, g['value']).is_special
+            ):
+                return True
+        return False
+
     def produce(self, node):
         if type(self.parent) is not projection:
             raise ValueError('groupby can only be used in projection')
@@ -478,6 +504,7 @@ class groupby(ast_node):
         for g in node:
             self.datasource.rec = set()
             g_expr = expr(self, g['value'])
+            self.use_sp_gb = self.use_sp_gb or g_expr.is_special
             refs : Set[ColRef] = self.datasource.rec
             self.datasource.rec = None
             if self.parent.col_ext:

reconstruct/expr.py

@@ -75,6 +75,7 @@ class expr(ast_node):
         self.udf_map = parent.context.udf_map
         self.func_maps = {**builtin_func, **self.udf_map, **user_module_func}
         self.operators = {**builtin_operators, **self.udf_map, **user_module_func}
+        self.ext_aggfuncs = ['sum', 'avg', 'count', 'min', 'max']
         
     def produce(self, node):
         from engine.utils import enlist
@@ -105,7 +106,11 @@ class expr(ast_node):
                         self.type = AnyT
                         
                     self.sql = op(self.c_code, *str_vals)
-                    special_func = [*self.context.udf_map.keys(), *self.context.module_map.keys(), "maxs", "mins", "avgs", "sums"]
+                    special_func = [*self.context.udf_map.keys(), *self.context.module_map.keys(), 
+                                    "maxs", "mins", "avgs", "sums", "deltas", "last"]
+                    if self.context.special_gb:
+                        special_func = [*special_func, *self.ext_aggfuncs]
+                        
                     if key in special_func and not self.is_special:
                         self.is_special = True
                         if key in self.context.udf_map:

reconstruct/storage.py

@@ -1,5 +1,5 @@
 from engine.types import *
-from engine.utils import enlist
+from engine.utils import base62uuid, enlist
 from typing import List, Dict, Set
 
 class ColRef:
@@ -47,10 +47,10 @@ class TableInfo:
         cxt.tables_byname[self.table_name] = self # construct reverse map
 
     def add_cols(self, cols, new = True):
-        for i, c in enumerate(cols):
+        for c in cols:
-            self.add_col(c, new, i)
+            self.add_col(c, new)
             
-    def add_col(self, c, new = True, i = 0):
+    def add_col(self, c, new = True):
         _ty = c['type']
         _ty_args = None
         if type(_ty) is dict:
@@ -100,6 +100,7 @@ class Context:
         self.procs = []
         self.queries = []
         self.module_init_loc = 0
+        self.special_gb = False
         
     def __init__(self):
         self.tables_byname = dict()
@@ -120,6 +121,13 @@ class Context:
         self.Error = lambda *args: print(*args)
         self.Info = lambda *_: None
         
+    def get_scan_var(self):
+        it_var = 'i' + base62uuid(2)
+        scan_vars = set(s.it_var for s in self.scans)
+        while(it_var in scan_vars):
+            it_var = 'i' + base62uuid(6)
+        return it_var
+    
     def emit(self, sql:str):
         self.sql += sql + ' '
     def emitc(self, c:str):

server/aggregations.h

@@ -16,14 +16,16 @@ constexpr static inline size_t count(const T&) { return 1; }
 
 // TODO: Specializations for dt/str/none
 template<class T, template<typename ...> class VT>
-types::GetLongType<T> sum(const VT<T>& v) {
+// types::GetLongType<T> 
+LL_Type sum(const VT<T>& v) {
 	types::GetLongType<T> ret = 0;
 	for (const auto& _v : v)
 		ret += _v;
 	return ret;
 }
 template<class T, template<typename ...> class VT>
-types::GetFPType<T> avg(const VT<T>& v) {
+//types::GetFPType<T> 
+double avg(const VT<T>& v) {
 	return static_cast<types::GetFPType<T>>(
 		sum<T>(v) / static_cast<long double>(v.size));
 }
@@ -156,6 +158,31 @@ decayed_t<VT, types::GetFPType<types::GetLongType<T>>> avgw(uint32_t w, const VT
 	return ret;
 }
 
+// use getSignedType
+template<class T, template<typename ...> class VT>
+decayed_t<VT, T> deltas(const VT<T>& arr) {
+	const uint32_t& len = arr.size;
+	decayed_t<VT, T> ret(len);
+	uint32_t i = 0;
+	if(len) ret[i++] = 0;
+	for (; i < len; ++i) 
+		ret[i] = arr[i] - arr[i-1];
+	return ret;
+}
+
+template<class T, template<typename ...> class VT>
+T last(const VT<T>& arr) {
+	const uint32_t& len = arr.size;
+	decayed_t<VT, T> ret(len);
+	uint32_t i = 0;
+	if (len)
+		ret[i++] = arr[0];
+	for (; i < len; ++i) 
+		ret[i] = arr[i-1];
+	return ret;
+}
+
+// wrong behavior with count(0)
 template <class T> constexpr inline T count(const T& v) { return 1; }
 template <class T> constexpr inline T max(const T& v) { return v; }
 template <class T> constexpr inline T min(const T& v) { return v; }
@@ -169,3 +196,5 @@ template <class T> constexpr inline T maxs(const T& v) { return v; }
 template <class T> constexpr inline T mins(const T& v) { return v; }
 template <class T> constexpr inline T avgs(const T& v) { return v; }
 template <class T> constexpr inline T sums(const T& v) { return v; }
+template <class T> constexpr inline T last(const T& v) { return v; }
+template <class T> constexpr inline T daltas(const T& v) { return 0; }

server/io.cpp

@@ -1,13 +1,15 @@
 #include "pch.hpp"
 
 #include "io.h"
-
-
-
 #include "table.h"
+#include <limits>
+
+#include <chrono>
+#include <ctime>
 
+#include "utils.h"
+#include <random>
 
-#ifdef __SIZEOF_INT128__
 char* gbuf = nullptr;
 
 void setgbuf(char* buf) {
@@ -18,6 +20,8 @@ void setgbuf(char* buf) {
 		gbuf = buf;
 }
 
+#ifdef __AQ__HAS__INT128__
+
 template <>
 void print<__int128_t>(const __int128_t& v, const char* delimiter){
 	char s[41];
@@ -40,6 +44,7 @@ std::ostream& operator<<(std::ostream& os, __uint128_t & v)
 	print(v);
 	return os;
 }
+
 #endif
 
 template <>
@@ -47,25 +52,197 @@ void print<bool>(const bool&v, const char* delimiter){
 	std::cout<< (v?"true":"false") << delimiter;
 }
 
+template<class T> 
+T getInt(const char*& buf){
+	T ret = 0;
+	while(*buf >= '0' and *buf <= '9'){
+		ret = ret*10 + *buf - '0';
+		buf++;
+	}
+	return ret;
+}
+template<class T> 
+char* intToString(T val, char* buf){
+
+	while (val > 0){
+		*--buf = val%10 + '0';
+		val /= 10;
+	}
+	
+	return buf;
+}
+void skip(const char*& buf){ 
+	while(*buf && (*buf >'9' || *buf < '0')) buf++; 
+}
 
-#include <chrono>
-#include <ctime>
 namespace types {
 	using namespace std;
 	using namespace chrono;
-	string date_t::toString() const {
+
-		uint32_t curr_v = val;
+	date_t::date_t(const char* str) { fromString(str); }
-		return string() + string("/") + string() + string("/") + string();
+	date_t& date_t::fromString(const char* str)  {
+		if(str) {
+			skip(str);
+			year = getInt<short>(str);
+			skip(str);
+			month = getInt<unsigned char>(str);
+			skip(str);
+			day = getInt<unsigned char>(str);
+		}
+		else{
+			day = month = year = 0;
+		}
+		return *this;
+	}
+	bool date_t::validate() const{
+		return year <= 9999 && month <= 12 && day <= 31;
 	}
-	string time_t::toString() const {
-		uint32_t curr_v = val;
 	
-		return string() + string("/") + string() + string("/") + string();
+	char* date_t::toString(char* buf) const {
+		// if (!validate()) return "(invalid date)";
+		*--buf = 0;
+		buf = intToString(day, buf);
+		*--buf = '-';
+		buf = intToString(month, buf);
+		*--buf = '-';
+		buf = intToString(year, buf);
+		return buf;
 	}
+	bool date_t::operator > (const date_t& other) const {
+		return year > other.year || (year == other.year && (month > other.month || (month == other.month && day > other.day)));
 	}
 
-#include "utils.h"
+	bool date_t::operator < (const date_t& other) const {
-#include <random>
+		return year < other.year || (year == other.year && (month < other.month || (month == other.month && day < other.day)));
+	}
+
+	bool date_t::operator >= (const date_t& other) const {
+		return year >= other.year || (year == other.year && (month >= other.month || (month == other.month && day >= other.day)));
+	}
+
+	bool date_t::operator <= (const date_t& other) const {
+		return year <= other.year || (year == other.year && (month <= other.month || (month == other.month && day <= other.day)));
+	}
+
+	bool date_t::operator == (const date_t& other) const {
+		return year == other.year && month == other.month && day == other.day;
+	}
+
+	bool date_t::operator != (const date_t& other) const {
+		return !operator==(other);
+	}
+
+	time_t::time_t(const char* str) { fromString(str); }
+	time_t& time_t::fromString(const char* str)  {
+		if(str) {
+			skip(str);
+			hours = getInt<unsigned char>(str);
+			skip(str);
+			minutes = getInt<unsigned char>(str);
+			skip(str);
+			seconds = getInt<unsigned char>(str);
+			skip(str);
+			ms = getInt<unsigned int> (str);
+		}
+		else {
+			hours = minutes = seconds = ms = 0;
+		}
+		return *this;
+	}
+	
+	char* time_t::toString(char* buf) const {
+		// if (!validate()) return "(invalid date)";
+		*--buf = 0;
+		buf = intToString(ms, buf);
+		*--buf = ':';
+		buf = intToString(seconds, buf);
+		*--buf = ':';
+		buf = intToString(minutes, buf);
+		*--buf = ':';
+		buf = intToString(hours, buf);
+		return buf;
+	}
+	bool time_t::operator > (const time_t& other) const {
+		return hours > other.hours || (hours == other.hours && (minutes > other.minutes || (minutes == other.minutes && (seconds > other.seconds || (seconds == other.seconds && ms > other.ms)))));
+	}
+	bool time_t::operator< (const time_t& other) const {
+		return hours < other.hours || (hours == other.hours && (minutes < other.minutes || (minutes == other.minutes && (seconds < other.seconds || (seconds == other.seconds && ms < other.ms)))));
+	} 
+	bool time_t::operator>= (const time_t& other) const {
+		return hours >= other.hours || (hours == other.hours && (minutes >= other.minutes || (minutes == other.minutes && (seconds >= other.seconds || (seconds == other.seconds && ms >= other.ms)))));
+	}
+	bool time_t::operator<= (const time_t& other) const{
+		return hours <= other.hours || (hours == other.hours && (minutes <= other.minutes || (minutes == other.minutes && (seconds <= other.seconds || (seconds == other.seconds && ms <= other.ms)))));
+	}
+	bool time_t::operator==(const time_t& other) const {
+		return hours == other.hours && minutes == other.minutes && seconds == other.seconds && ms == other.ms;
+	}
+	bool time_t::operator!= (const time_t& other) const {
+		return !operator==(other);
+	}
+	bool time_t::validate() const{
+		return hours < 24 && minutes < 60 && seconds < 60 && ms < 1000;
+	}
+
+	timestamp_t::timestamp_t(const char* str) { fromString(str); }
+	timestamp_t& timestamp_t::fromString(const char* str) {
+		date.fromString(str);
+		time.fromString(str);
+
+		return *this;
+	}
+	bool timestamp_t::validate() const {
+		return date.validate() && time.validate();
+	}
+	
+	char* timestamp_t::toString(char* buf) const {
+		buf = time.toString(buf);
+		auto ret = date.toString(buf);
+		*(buf-1) = ' ';
+		return ret;
+	}
+	bool timestamp_t::operator > (const timestamp_t& other) const {
+		return date > other.date || (date == other.date && time > other.time);
+	}
+	bool timestamp_t::operator < (const timestamp_t& other) const {
+		return date < other.date || (date == other.date && time < other.time);
+	}
+	bool timestamp_t::operator >= (const timestamp_t& other) const {
+		return date >= other.date || (date == other.date && time >= other.time);
+	}
+	bool timestamp_t::operator <= (const timestamp_t& other) const {
+		return date <= other.date || (date == other.date && time <= other.time);
+	}
+	bool timestamp_t::operator == (const timestamp_t& other) const {
+		return date == other.date && time == other.time;
+	}
+	bool timestamp_t::operator!= (const timestamp_t & other) const {
+		return !operator==(other);
+	}
+}
+
+template<class T>
+void print_datetime(const T&v){
+	char buf[T::string_length()];
+	std::cout<<v.toString(buf + T::string_length());
+}
+std::ostream& operator<<(std::ostream& os, types::date_t & v)
+{
+	print_datetime(v);
+	return os;
+}
+std::ostream& operator<<(std::ostream& os, types::time_t & v)
+{
+	print_datetime(v);
+	return os;
+}
+std::ostream& operator<<(std::ostream& os, types::timestamp_t & v)
+{
+	print_datetime(v);
+	return os;
+}
+
+
 using std::string;
 string base62uuid(int l = 8) {
     using namespace std;

server/io.h

@@ -25,7 +25,27 @@ inline decltype(auto) print_hook<bool>(const bool& v) {
 	return v? "true" : "false";
 }
 
-#ifdef __SIZEOF_INT128__
+extern char* gbuf;
+
+void setgbuf(char* buf = 0);
+
+template<>
+inline decltype(auto) print_hook<types::date_t>(const types::date_t& v) {
+	*(gbuf += types::date_t::string_length()) = 0;
+
+	return v.toString(gbuf);
+}
+template<>
+inline decltype(auto) print_hook<types::time_t>(const types::time_t& v) {
+	*(gbuf += types::time_t::string_length()) = 0;
+	return v.toString(gbuf);
+}
+template<>
+inline decltype(auto) print_hook<types::timestamp_t>(const types::timestamp_t& v) {
+	*(gbuf += types::timestamp_t::string_length()) = 0;
+	return v.toString(gbuf);
+}
+#ifdef __AQ__HAS__INT128__
 constexpr struct __int128__struct{
     uint64_t low, high;
 	// constexpr bool operator==(__int128_t x) const{
@@ -59,9 +79,7 @@ inline const char* get_uint128str(__uint128_t v, char* buf){
 	} while(v);
 	return buf;
 }
-extern char* gbuf;
 
-void setgbuf(char* buf = 0);
 
 template<>
 inline decltype(auto) print_hook<__int128_t>(const __int128_t& v) {
@@ -77,5 +95,4 @@ inline decltype(auto) print_hook<__uint128_t>(const __uint128_t& v) {
 
 #else
 
-#define setgbuf()
 #endif

server/table.h

@@ -188,7 +188,10 @@ std::ostream& operator<<(std::ostream& os, const VT<T>& v)
 	v.out();
 	return os;
 }
-#ifdef __SIZEOF_INT128__
+std::ostream& operator<<(std::ostream& os, types::date_t & v);
+std::ostream& operator<<(std::ostream& os, types::time_t & v);
+std::ostream& operator<<(std::ostream& os, types::timestamp_t & v);
+#ifdef __AQ__HAS__INT128__
 std::ostream& operator<<(std::ostream& os, __int128 & v);
 std::ostream& operator<<(std::ostream& os, __uint128_t & v);
 #endif
@@ -329,14 +332,14 @@ struct TableInfo {
 				print2_impl<rem_cols...>(func, i, args ..., print_hook(v));
 		};
 		if constexpr (is_vector_type<this_type>)
-			for (int j = 0; j < this_value.size; ++j)
+			for (uint32_t j = 0; j < this_value.size; ++j)
 				next(this_value[j]);
 		else
 			next(this_value);
 	}
 	std::string get_header_string(const char* __restrict sep, const char* __restrict end) const{
 		std::string header_string = std::string();
-		for (int i = 0; i < sizeof...(Types); ++i)
+		for (uint32_t i = 0; i < sizeof...(Types); ++i)
 			header_string += std::string(this->colrefs[i].name) + sep + '|' + sep;
 		const size_t l_sep = strlen(sep) + 1;
 		if (header_string.size() - l_sep >= 0)
@@ -359,18 +362,27 @@ struct TableInfo {
 			header_string.resize(header_string.size() - l_sep);
 		
 		const auto& prt_loop = [&fp, &view, &printf_string, *this](const auto& f) {
-#ifdef __SIZEOF_INT128__			
+#ifdef __AQ__HAS__INT128__			
 			constexpr auto num_hge = count_type<__int128_t, __uint128_t>((tuple_type*)(0));
-			char cbuf[num_hge * 41];
+#else
-			setgbuf(cbuf);
+			constexpr auto num_hge = 0;
 #endif
+			constexpr auto num_date = count_type<types::date_t>((tuple_type*)(0));
+			constexpr auto num_time = count_type<types::time_t>((tuple_type*)(0));
+			constexpr auto num_timestamp = count_type<types::timestamp_t>((tuple_type*)(0));
+			char cbuf[ num_hge * 41 
+				+ num_time * types::time_t::string_length() 
+				+ num_date * types::date_t::string_length() 
+				+ num_timestamp * types::timestamp_t::string_length() 
+			];
+			setgbuf(cbuf);
 			if(view)
-				for (int i = 0; i < view->size; ++i){
+				for (uint32_t i = 0; i < view->size; ++i){
 					print2_impl<cols...>(f, (*view)[i], printf_string.c_str());
 					setgbuf();
 				}
 			else
-				for (int i = 0; i < colrefs[0].size; ++i){
+				for (uint32_t i = 0; i < colrefs[0].size; ++i){
 					print2_impl<cols...>(f, i, printf_string.c_str());		
 					setgbuf();
 				}
@@ -465,10 +477,10 @@ inline void TableView<Types...>::print(const char* __restrict sep, const char* _
 	std::string header_string = info.get_header_string(sep, end);
 	std::cout << header_string.c_str();
 	
-	int n_rows = 0;
+	uint32_t n_rows = 0;
 	if (info.colrefs[0].size > 0)
 		n_rows = info.colrefs[0].size;
-	for (int i = 0; i < n_rows; ++i) {
+	for (uint32_t i = 0; i < n_rows; ++i) {
 		print_impl(i);
 		std::cout << end;
 	}
@@ -495,10 +507,10 @@ inline void TableInfo<Types...>::print(const char* __restrict sep, const char* _
 	std::string header_string = get_header_string(sep, end);
 	std::cout << header_string.c_str();
 	
-	int n_rows = 0;
+	uint32_t n_rows = 0;
 	if (n_cols > 0 && colrefs[0].size > 0)
 		n_rows = colrefs[0].size;
-	for (int i = 0; i < n_rows; ++i) {
+	for (uint32_t i = 0; i < n_rows; ++i) {
 		print_impl(i);
 		std::cout << end;
 	}
@@ -506,88 +518,112 @@ inline void TableInfo<Types...>::print(const char* __restrict sep, const char* _
 template <class T1, class T2, template<typename ...> class VT, template<typename ...> class VT2>
 decayed_t<VT, typename types::Coercion<T1, T2>::type> operator -(const VT<T1>& lhs, const VT2<T2>& rhs) {
 	auto ret = decayed_t<VT, typename types::Coercion<T1, T2>::type>(lhs.size);
-	for (int i = 0; i < lhs.size; ++i)
+	for (uint32_t i = 0; i < lhs.size; ++i)
 		ret[i] = lhs[i] - rhs[i];
 	return ret;
 }
 template <class T1, class T2, template<typename ...> class VT>
 decayed_t<VT, typename types::Coercion<T1, T2>::type> operator -(const VT<T1>& lhs, const T2& rhs) {
 	auto ret = decayed_t<VT, typename types::Coercion<T1, T2>::type>(lhs.size);
-	for (int i = 0; i < lhs.size; ++i)
+	for (uint32_t i = 0; i < lhs.size; ++i)
 		ret[i] = lhs[i] - rhs;
 	return ret;
 }
 template <class T1, class T2, template<typename ...> class VT>
 decayed_t<VT, typename types::Coercion<T1, T2>::type> operator -(const T2& lhs, const VT<T1>& rhs) {
 	auto ret = decayed_t<VT, typename types::Coercion<T1, T2>::type>(rhs.size);
-	for (int i = 0; i < rhs.size; ++i)
+	for (uint32_t i = 0; i < rhs.size; ++i)
 		ret[i] = lhs - rhs[i];
 	return ret;
 }
 template <class T1, class T2, template<typename ...> class VT, template<typename ...> class VT2>
 decayed_t<VT, typename types::Coercion<T1, T2>::type> operator +(const VT<T1>& lhs, const VT2<T2>& rhs) {
 	auto ret = decayed_t<VT, typename types::Coercion<T1, T2>::type>(lhs.size);
-	for (int i = 0; i < lhs.size; ++i)
+	for (uint32_t i = 0; i < lhs.size; ++i)
 		ret[i] = lhs[i] + rhs[i];
 	return ret;
 }
 template <class T1, class T2, template<typename ...> class VT>
 decayed_t<VT, typename types::Coercion<T1, T2>::type> operator +(const VT<T1>& lhs, const T2& rhs) {
 	auto ret = decayed_t<VT, typename types::Coercion<T1, T2>::type>(lhs.size);
-	for (int i = 0; i < lhs.size; ++i)
+	for (uint32_t i = 0; i < lhs.size; ++i)
 		ret[i] = lhs[i] + rhs;
 	return ret;
 }
 template <class T1, class T2, template<typename ...> class VT>
 decayed_t<VT, typename types::Coercion<T1, T2>::type> operator +(const T2& lhs, const VT<T1>& rhs) {
 	auto ret = decayed_t<VT, typename types::Coercion<T1, T2>::type>(rhs.size);
-	for (int i = 0; i < rhs.size; ++i)
+	for (uint32_t i = 0; i < rhs.size; ++i)
 		ret[i] = lhs + rhs[i];
 	return ret;
 }
 template <class T1, class T2, template<typename ...> class VT, template<typename ...> class VT2>
 decayed_t<VT, typename types::Coercion<T1, T2>::type> operator *(const VT<T1>& lhs, const VT2<T2>& rhs) {
 	auto ret = decayed_t<VT, typename types::Coercion<T1, T2>::type>(lhs.size);
-	for (int i = 0; i < lhs.size; ++i)
+	for (uint32_t i = 0; i < lhs.size; ++i)
 		ret[i] = lhs[i] * rhs[i];
 	return ret;
 }
 template <class T1, class T2, template<typename ...> class VT>
 decayed_t<VT, typename types::Coercion<T1, T2>::type> operator *(const VT<T1>& lhs, const T2& rhs) {
 	auto ret = decayed_t<VT, typename types::Coercion<T1, T2>::type>(lhs.size);
-	for (int i = 0; i < lhs.size; ++i)
+	for (uint32_t i = 0; i < lhs.size; ++i)
 		ret[i] = lhs[i] * rhs;
 	return ret;
 }
 template <class T1, class T2, template<typename ...> class VT>
 decayed_t<VT, typename types::Coercion<T1, T2>::type> operator *(const T2& lhs, const VT<T1>& rhs) {
 	auto ret = decayed_t<VT, typename types::Coercion<T1, T2>::type>(rhs.size);
-	for (int i = 0; i < rhs.size; ++i)
+	for (uint32_t i = 0; i < rhs.size; ++i)
 		ret[i] = lhs * rhs[i];
 	return ret;
 }
 template <class T1, class T2, template<typename ...> class VT, template<typename ...> class VT2>
 decayed_t<VT, types::GetFPType<typename types::Coercion<T1, T2>::type>> operator /(const VT<T1>& lhs, const VT2<T2>& rhs) {
 	auto ret = decayed_t<VT, types::GetFPType<typename types::Coercion<T1, T2>::type>>(lhs.size);
-	for (int i = 0; i < lhs.size; ++i)
+	for (uint32_t i = 0; i < lhs.size; ++i)
 		ret[i] = lhs[i] / rhs[i];
 	return ret;
 }
 template <class T1, class T2, template<typename ...> class VT>
 decayed_t<VT, types::GetFPType<typename types::Coercion<T1, T2>::type>> operator /(const VT<T1>& lhs, const T2& rhs) {
 	auto ret = decayed_t<VT, types::GetFPType<typename types::Coercion<T1, T2>::type>>(lhs.size);
-	for (int i = 0; i < lhs.size; ++i)
+	for (uint32_t i = 0; i < lhs.size; ++i)
 		ret[i] = lhs[i] / rhs;
 	return ret;
 }
 template <class T1, class T2, template<typename ...> class VT>
 decayed_t<VT, types::GetFPType<typename types::Coercion<T1, T2>::type>> operator /(const T2& lhs, const VT<T1>& rhs) {
 	auto ret = decayed_t<VT, types::GetFPType<typename types::Coercion<T1, T2>::type>>(rhs.size);
-	for (int i = 0; i < rhs.size; ++i)
+	for (uint32_t i = 0; i < rhs.size; ++i)
 		ret[i] = lhs / rhs[i];
 	return ret;
 }
 
+template <class T1, class T2, template<typename ...> class VT, template<typename ...> class VT2>
+VT<bool> operator >(const VT<T1>& lhs, const VT2<T2>& rhs) {
+	auto ret = VT<bool>(lhs.size);
+	for (uint32_t i = 0; i < lhs.size; ++i)
+		ret[i] = lhs[i] > rhs[i];
+	return ret;
+}
+template <class T1, class T2, template<typename ...> class VT>
+VT<bool> operator >(const VT<T1>& lhs, const T2& rhs) {
+	auto ret = VT<bool>(lhs.size);
+	for (uint32_t i = 0; i < lhs.size; ++i)
+		ret[i] = lhs[i] > rhs;
+	return ret;
+}
+template <class T1, class T2, template<typename ...> class VT>
+VT<bool> operator >(const T2& lhs, const VT<T1>& rhs) {
+	auto ret = VT<bool>(rhs.size);
+	for (uint32_t i = 0; i < rhs.size; ++i)
+		ret[i] = lhs > rhs[i];
+	return ret;
+}
+
+
+
 template <class ...Types>
 void print(const TableInfo<Types...>& v, const char* delimiter = " ", const char* endline = "\n") {
 	v.print(delimiter, endline);
@@ -598,10 +634,11 @@ void print(const TableView<Types...>& v, const char* delimiter = " ", const char
 }
 template <class T>
 void print(const T& v, const char* delimiter = " ") {
-	std::cout<< v;
+	std::cout<< v<< delimiter;
 	// printf(types::printf_str[types::Types<T>::getType()], v);
 }
-#ifdef __SIZEOF_INT128__
+
+#ifdef __AQ__HAS__INT128__
 template <>
 void print<__int128_t>(const __int128_t& v, const char* delimiter);
 template <>

server/types.h

@@ -2,9 +2,12 @@
 #define _TYPES_H
 #include <cstdint>
 #include <type_traits>
-#include <string>
 #include <tuple>
 
+#if  defined(__SIZEOF_INT128__) and not defined(_WIN32)
+#define __AQ__HAS__INT128__
+#endif
+
 #ifdef _MSC_VER
 #define __restrict__ __restrict
 #endif
@@ -21,32 +24,85 @@ constexpr static bool is_vector_type = is_vector_impl<T>::value;
 namespace types {
 	enum Type_t {
 		AINT32, AFLOAT, ASTR, ADOUBLE, ALDOUBLE, AINT64, AINT128, AINT16, ADATE, ATIME, AINT8,
-		AUINT32, AUINT64, AUINT128, AUINT16, AUINT8, ABOOL, VECTOR, NONE, ERROR
+		AUINT32, AUINT64, AUINT128, AUINT16, AUINT8, ABOOL, VECTOR, ATIMESTAMP, NONE, ERROR
 	};
 	static constexpr const char* printf_str[] = { "%d", "%f", "%s", "%lf", "%Lf", "%ld", "%d", "%hi", "%s", "%s", "%c",
-		"%u", "%lu", "%s", "%hu", "%hhu", "%s", "%s", "Vector<%s>", "NULL", "ERROR" };
+		"%u", "%lu", "%s", "%hu", "%hhu", "%s", "%s", "Vector<%s>", "%s", "NULL", "ERROR" };
 	static constexpr const char* SQL_Type[] = { "INT", "REAL", "TEXT", "DOUBLE", "DOUBLE", "BIGINT", "HUGEINT", "SMALLINT", "DATE", "TIME", "TINYINT",
-		"INT", "BIGINT", "HUGEINT", "SMALLINT", "TINYINT", "BIGINT", "BOOL", "BIGINT", "NULL", "ERROR"};
+		"INT", "BIGINT", "HUGEINT", "SMALLINT", "TINYINT", "BIGINT", "BOOL", "BIGINT", "TIMESTAMP", "NULL", "ERROR"};
 	
 	
 	// TODO: deal with data/time <=> str/uint conversion
 	struct date_t{
-		uint32_t val = 0;
+		unsigned char day = 0;
-		date_t(const char* d) {
+		unsigned char month = 0;
-		}
+		short year = 0;
-		std::string toString() const;
+
+		date_t() = default;
+		date_t(unsigned char day, unsigned char month, short year) : 
+			day (day), month (month), year(year) {}
+		date_t(const char*);
+		date_t& fromString(const char*);
+		bool validate() const;
+		constexpr static unsigned string_length(){
+			return 11;
+		};
+		char* toString(char* buf) const;
+		bool operator > (const date_t&) const;
+		bool operator < (const date_t&) const;
+		bool operator >= (const date_t&) const;
+		bool operator <= (const date_t&) const;
+		bool operator == (const date_t&) const;
+		bool operator != (const date_t&) const;
 	};
+
 	struct time_t{
-		uint32_t val = 0;
+		unsigned int ms = 0;
-		time_t(const char* d) {
+		unsigned char seconds = 0;
-		}
+		unsigned char minutes = 0;
-		std::string toString() const;
+		unsigned char hours = 0;
+
+		time_t() = default; 
+		time_t(unsigned int ms, unsigned char seconds, unsigned char minutes, unsigned char hours) : 
+			ms (ms), seconds (seconds), minutes(minutes), hours(hours) {}; 
+		time_t(const char*); 
+		time_t& fromString(const char*);
+		bool validate() const;
+		constexpr static unsigned string_length() {
+			return 13;
+		};
+		char* toString(char* buf) const;
+		bool operator > (const time_t&) const;
+		bool operator < (const time_t&) const;
+		bool operator >= (const time_t&) const;
+		bool operator <= (const time_t&) const;
+		bool operator == (const time_t&) const;
+		bool operator != (const time_t&) const;
+	};
+	struct timestamp_t{
+		date_t date;
+		time_t time;
+		timestamp_t() = default;
+		timestamp_t(const date_t& d, const time_t& t) : date(d), time(t) {}
+		timestamp_t(const char*); 
+		timestamp_t& fromString(const char*);
+		bool validate() const;
+		constexpr static unsigned string_length(){
+			return date_t::string_length() + time_t::string_length();
+		};
+		char* toString(char* buf) const;
+		bool operator > (const timestamp_t&) const;
+		bool operator < (const timestamp_t&) const;
+		bool operator >= (const timestamp_t&) const;
+		bool operator <= (const timestamp_t&) const;
+		bool operator == (const timestamp_t&) const;
+		bool operator != (const timestamp_t&) const;
 	};
 	template <typename T>
 	struct Types {
 		typedef T type;
 		constexpr Types() noexcept = default;
-#ifdef __SIZEOF_INT128__
+#ifdef __AQ__HAS__INT128__
 #define F_INT128(__F_) __F_(__int128_t, AINT128) \
 				 __F_(__uint128_t, AUINT128)
 #define ULL_Type __uint128_t
@@ -73,6 +129,7 @@ namespace types {
 		f(unsigned short, AUINT16) \
 		f(unsigned char, AUINT8) \
 		f(bool, ABOOL) \
+		f(timestamp_t, ATIMESTAMP) \
 		F_INT128(f)
 
 		inline constexpr static Type_t getType() {

server/vector_type.hpp

@@ -9,7 +9,7 @@
 #include <cstring>
 #include <cstdlib>
 #include <cstdint>
-
+#include <iterator>
 #include <initializer_list>
 #include "types.h"
 
@@ -243,6 +243,21 @@ public:
 		if (capacity > 0) free(container);
 		container = 0; size = capacity = 0;
 	}
+#define Compare(_op) \
+	template<typename T> \
+	inline vector_type<bool> operator _op (const T& v){ \
+		vector_type<bool> res(size); \
+		for (uint32_t i = 0; i < size; ++i) \
+			res[i] = container[i] > v; \
+		return res; \
+	} 
+	// template <typename T> 
+	Compare(>)
+	Compare(<)
+	Compare(>=)
+	Compare(<=)
+	Compare(==)
+	Compare(!=)
 #define Op(o, x) \
 	template<typename T>\
 	vector_type<typename types::Coercion<_Ty, T>::type> inline x(const vector_type<T>& r) const {\

tests/network.a

@@ -0,0 +1,13 @@
+CREATE TABLE network(src varchar(3), dst varchar(3), len int, _time int)
+
+
+LOAD DATA INFILE "data/network.csv"
+INTO TABLE network
+FIELDS TERMINATED BY ","
+
+SELECT	 src, dst,  avg(len) 
+FROM	 network
+	     ASSUMING ASC src, ASC dst, ASC _time 
+GROUP BY src, dst, sums (deltas(_time) > 120)
+
+

tests/strings.a

@@ -5,3 +5,5 @@ INTO TABLE types_test
 FIELDS TERMINATED BY ","
 
 select names, val * 10000 + id from types_test
+
+create table date_time(id int, _date date, _time time, _timestamp timestamp);