triggers

Monetdb get col/sz
Updated RF/dataproc
Bug fixes and improvements

.gitignore

@@ -59,6 +59,10 @@ data/benchmark
 !nyctx100.csv
 !network.csv
 !test_complex.csv
+data/electricity*
+data/covtype*
+data/phishing*
+data/power*
 *.out
 *.asm
 !mmw.so
@@ -81,5 +85,7 @@ saves
 out*.cpp
 udf*.hpp
 *.ipynb
-
-
+saved_procedures/**
+procedures/**
+.mypy_cache
+__pycache__

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "paper"]
+	path = paper
+	url = https://github.com/sunyinqi0508/AQueryPaper

Dockerfile

@@ -10,7 +10,7 @@ RUN export OS_VER=`cat /etc/os-release | grep VERSION_CODENAME` &&\
 
 RUN wget --output-document=/etc/apt/trusted.gpg.d/monetdb.gpg https://dev.monetdb.org/downloads/MonetDB-GPG-KEY.gpg
 
-RUN apt update && apt install -y python3 python3-pip clang-14 libmonetdbe-dev git 
+RUN apt update && apt install -y python3 python3-pip clang-14 libmonetdbe-dev libmonetdb-client-dev monetdb5-sql-dev git 
 
 RUN git clone https://github.com/sunyinqi0508/AQuery2 
 

Makefile

@@ -2,12 +2,13 @@ OS_SUPPORT =
 MonetDB_LIB = 
 MonetDB_INC = 
 Defines = 
-CXXFLAGS = --std=c++1z
+CC = $(CXX) -xc 
+CXXFLAGS = --std=c++2a
 ifeq ($(AQ_DEBUG), 1)
-	OPTFLAGS = -g3 -fsanitize=address -fsanitize=leak
+	OPTFLAGS = -g3 #-fsanitize=address 
 	LINKFLAGS = 
 else
-	OPTFLAGS = -O3 -DNDEBUG -fno-stack-protector 
+	OPTFLAGS = -Ofast -DNDEBUG -fno-stack-protector 
 	LINKFLAGS = -flto -s
 endif
 SHAREDFLAGS = -shared  
@@ -17,7 +18,7 @@ COMPILER = $(strip $(_COMPILER))
 LIBTOOL = ar rcs
 USELIB_FLAG = -Wl,--whole-archive,libaquery.a -Wl,-no-whole-archive
 LIBAQ_SRC = server/monetdb_conn.cpp server/libaquery.cpp 
-LIBAQ_OBJ = monetdb_conn.o libaquery.o
+LIBAQ_OBJ = monetdb_conn.o libaquery.o monetdb_ext.o
 SEMANTIC_INTERPOSITION = -fno-semantic-interposition
 RANLIB = ranlib
 _LINKER_BINARY = $(shell `$(CXX) -print-prog-name=ld` -v 2>&1 | grep -q LLVM && echo lld || echo ld)
@@ -43,7 +44,7 @@ else
 		LIBTOOL = gcc-ar rcs
 	endif
 endif
-OPTFLAGS += $(SEMANTIC_INTERPOSITION)
+LINKFLAGS += $(SEMANTIC_INTERPOSITION)
 
 ifeq ($(PCH), 1)
 	PCHFLAGS = -include server/pch.hpp
@@ -82,7 +83,7 @@ else
 		MonetDB_INC += $(AQ_MONETDB_INC)
 		MonetDB_INC += -I/usr/local/include/monetdb -I/usr/include/monetdb 
 	endif
-	MonetDB_LIB += -lmonetdbe
+	MonetDB_LIB += -lmonetdbe -lmonetdbsql -lbat
 endif
 
 ifeq ($(THREADING),1)
@@ -128,6 +129,7 @@ pch:
 	$(CXX) -x c++-header server/pch.hpp $(FPIC) $(CXXFLAGS)
 libaquery:
 	$(CXX) -c $(FPIC) $(PCHFLAGS) $(LIBAQ_SRC) $(OS_SUPPORT) $(CXXFLAGS) &&\
+	$(CC) -c server/monetdb_ext.c $(OPTFLAGS) $(MonetDB_INC) &&\
 	$(LIBTOOL) libaquery.a $(LIBAQ_OBJ) &&\
 	$(RANLIB) libaquery.a
 

README.md

@@ -1,31 +1,20 @@
-
 # AQuery++ Database
-
-### Please try the latest code in dev branch if you encounter any problem. Use `git checkout dev` to switch branches.
-
 ## Introduction
 
 AQuery++ Database is a cross-platform, In-Memory Column-Store Database that incorporates compiled query execution. (**Note**: If you encounter any problems, feel free to contact me via ys3540@nyu.edu)
+# Architecture 
+![Architecture](./docs/arch-hybrid.svg)
 
-## Docker (Recommended): 
-   - See installation instructions from [docker.com](https://www.docker.com). Run **docker desktop** to start docker engine.
-   - In AQuery root directory, type `make docker` to build the docker image from scratch. 
-   - For Arm-based Mac users, you would have to build and run the **x86_64** docker image because MonetDB doesn't offer official binaries for arm64 Linux. (Run `docker buildx build --platform=linux/amd64 -t aquery .` instead of `make docker`)
-   - Finally run the image in **interactive** mode (`docker run --name aquery -it aquery`) 
-   - When you need to access the container again run `docker start -ai aquery` 
-   - If there is a need to access the system shell within AQuery, type `dbg` to activate python interpreter and type `os.system('sh')` to launch a shell.
-   - Docker image is available on [Docker Hub](https://hub.docker.com/repository/docker/sunyinqi0508/aquery) but building image yourself is highly recommended (see [#2](../../issues/2)) 
-## CIMS Computer Lab (Only for NYU affiliates who have access)
-  1. Clone this git repo in CIMS.
-  2. Download the [patch](https://drive.google.com/file/d/1YkykhM6u0acZ-btQb4EUn4jAEXPT81cN/view?usp=sharing) 
-  3. Decompress the patch to any directory and execute script inside by typing (`source ./cims.sh`). Please use the source command or `. ./cims.sh` (dot space) to execute the script because it contains configurations for environment variables. Also note that this script can only work with bash and compatible shells (e.g. dash, zsh. but not csh)
-  4. Execute `python3 ./prompt.py`
+## AQuery Compiler
+- The query is first processed by the AQuery Compiler which is composed of a frontend that parses the query into AST and a backend that generates target code that delivers the query.
+- Front end of AQuery++ Compiler is built on top of [mo-sql-parsing](https://github.com/klahnakoski/mo-sql-parsing) with modifications to handle AQuery dialect and extension.
+- Backend of AQuery++ Compiler generates target code dependent on the Execution Engine. It can either be the C++ code for AQuery Execution Engine or sql and C++ post-processor for Hybrid Engine or k9 for the k9 Engine.
+## Execution Engines
+- AQuery++ supports different execution engines thanks to the decoupled compiler structure.
+- Hybrid Execution Engine: decouples the query into two parts. The sql-compliant part is executed by an Embedded version of Monetdb and everything else is executed by a post-process module which is generated by AQuery++ Compiler in C++ and then compiled and executed.
+- AQuery Library: A set of header based libraries that provide column arithmetic and operations inspired by array programming languages like kdb. This library is used by C++ post-processor code which can significantly reduce the complexity of generated code, reducing compile time while maintaining the best performance. The set of libraries can also be used by UDFs as well as User modules which makes it easier for users to write simple, efficient yet powerful extensions. 
 
-## Singularity Container
- 1. build container `singularity build aquery.sif aquery.def`
- 2. execute container `singularity exec aquery.sif sh`
- 3. run AQuery `python3 ./prompt.py`
-# Native Installation:
+# Installation:
 ## Requirements
 1. Recent version of Linux, Windows or MacOS, with recent C++ compiler that has C++17 (1z) support. (however c++20 is recommended if available for heterogeneous lookup on unordered containers)
      - GCC: 9.0 or above (g++ 7.x, 8.x fail to handle fold-expressions due to a compiler bug)
@@ -38,10 +27,6 @@ AQuery++ Database is a cross-platform, In-Memory Column-Store Database that inco
    - On MacOS, Monetdb can be easily installed in homebrew `brew install monetdb`.
 
 3. Python 3.6 or above and install required packages in requirements.txt by `python3 -m pip install -r requirements.txt` 
-
-## Installation
-AQuery is tested on mainstream operating systems such as Windows, macOS and Linux
-
 ### Windows
 There're multiple options to run AQuery on Windows. But for better consistency I recommend using a simulated Linux environment such as **Windows Subsystem for Linux** (1 or 2), **Docker** or **Linux Virtual Machines**. You can also use the native toolchain from Microsoft Visual Studio or gcc from Winlabs/Cygwin/MinGW.
 
@@ -97,7 +82,24 @@ There're multiple options to run AQuery on Windows. But for better consistency I
    
    In this case, upgrade anaconda or your compiler or use the python from your OS or package manager instead. Or (**NOT recommended**) copy/link the library from your system (e.g. /usr/lib/x86_64-linux-gnu/libstdc++.so.6) to anaconda's library directory (e.g. ~/Anaconda3/lib/).
 
+## Docker (Recommended): 
+   - See installation instructions from [docker.com](https://www.docker.com). Run **docker desktop** to start docker engine.
+   - In AQuery root directory, type `make docker` to build the docker image from scratch. 
+   - For Arm-based Mac users, you would have to build and run the **x86_64** docker image because MonetDB doesn't offer official binaries for arm64 Linux. (Run `docker buildx build --platform=linux/amd64 -t aquery .` instead of `make docker`)
+   - Finally run the image in **interactive** mode (`docker run --name aquery -it aquery`) 
+   - When you need to access the container again run `docker start -ai aquery` 
+   - If there is a need to access the system shell within AQuery, type `dbg` to activate python interpreter and type `os.system('sh')` to launch a shell.
+   - Docker image is available on [Docker Hub](https://hub.docker.com/repository/docker/sunyinqi0508/aquery) but building image yourself is highly recommended (see [#2](../../issues/2)) 
+## CIMS Computer Lab (Only for NYU affiliates who have access)
+  1. Clone this git repo in CIMS.
+  2. Download the [patch](https://drive.google.com/file/d/1YkykhM6u0acZ-btQb4EUn4jAEXPT81cN/view?usp=sharing) 
+  3. Decompress the patch to any directory and execute script inside by typing (`source ./cims.sh`). Please use the source command or `. ./cims.sh` (dot space) to execute the script because it contains configurations for environment variables. Also note that this script can only work with bash and compatible shells (e.g. dash, zsh. but not csh)
+  4. Execute `python3 ./prompt.py`
 
+## Singularity Container
+ 1. build container `singularity build aquery.sif aquery.def`
+ 2. execute container `singularity exec aquery.sif sh`
+ 3. run AQuery `python3 ./prompt.py`
 # Usage
 `python3 prompt.py` will launch the interactive command prompt. The server binary will be automatically rebuilt and started.
 ### Commands:
@@ -268,17 +270,6 @@ SELECT * FROM my_table WHERE c1 > 10
 - `sqrt(x), trunc(x), and other builtin math functions`: value-wise math operations. `sqrt(x)[i] = sqrt(x[i])`
 - `pack(cols, ...)`: pack multiple columns with exact same type into a single column. 
 
-# Architecture 
-![Architecture](./docs/arch-hybrid.svg)
-
-## AQuery Compiler
-- The query is first processed by the AQuery Compiler which is composed of a frontend that parses the query into AST and a backend that generates target code that delivers the query.
-- Front end of AQuery++ Compiler is built on top of [mo-sql-parsing](https://github.com/klahnakoski/mo-sql-parsing) with modifications to handle AQuery dialect and extension.
-- Backend of AQuery++ Compiler generates target code dependent on the Execution Engine. It can either be the C++ code for AQuery Execution Engine or sql and C++ post-processor for Hybrid Engine or k9 for the k9 Engine.
-## Execution Engines
-- AQuery++ supports different execution engines thanks to the decoupled compiler structure.
-- Hybrid Execution Engine: decouples the query into two parts. The sql-compliant part is executed by an Embedded version of Monetdb and everything else is executed by a post-process module which is generated by AQuery++ Compiler in C++ and then compiled and executed.
-- AQuery Library: A set of header based libraries that provide column arithmetic and operations inspired by array programming languages like kdb. This library is used by C++ post-processor code which can significantly reduce the complexity of generated code, reducing compile time while maintaining the best performance. The set of libraries can also be used by UDFs as well as User modules which makes it easier for users to write simple but powerful extensions. 
 
 # Roadmap
 - [x] SQL Parser -> AQuery Parser (Front End)
@@ -304,10 +295,11 @@ SELECT * FROM my_table WHERE c1 > 10
 - [x] Query Optimization
   - [x] Selection/Order by push-down
   - [x] Join Optimization (Only in Hybrid Engine)
-  - [ ] Threaded GC
+  - [x] Threaded GC
 - [ ] Extensibility 
   - [x] UDFs (Hybrid Engine only)
   - [x] SDK and User Module 
+  - [x] Stored Procedures
   - [ ] Triggers 
 
 # Known Issues:
@@ -343,3 +335,7 @@ SELECT * FROM my_table WHERE c1 > 10
 
 - [MonetDB](https://www.monetdb.org) <br>
   License (Mozilla Public License): https://github.com/MonetDB/MonetDB/blob/master/license.txt
+
+- [ankerl::unordered_dense](https://github.com/martinus/unordered_dense)<br>
+  Author: Martin Ankerl <br>
+  License (MIT): http://opensource.org/licenses/MIT <br>

aquery_config.py

@@ -2,7 +2,7 @@
 
 ## GLOBAL CONFIGURATION FLAGS
 
-version_string = '0.5.3a'
+version_string = '0.7.0a'
 add_path_to_ldpath = True
 rebuild_backend = False
 run_backend = True

aquery_parser/__init__.py

@@ -5,20 +5,18 @@
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 #
 # Contact: Kyle Lahnakoski (kyle@lahnakoski.com)
-#
+# Bill Sun  2022 - 2023
 
 from __future__ import absolute_import, division, unicode_literals
 
 import json
 from threading import Lock
 
-from aquery_parser.sql_parser import scrub
+from aquery_parser.parser import scrub
 from aquery_parser.utils import ansi_string, simple_op, normal_op
-
+import aquery_parser.parser
 parse_locker = Lock()  # ENSURE ONLY ONE PARSING AT A TIME
 common_parser = None
-mysql_parser = None
-sqlserver_parser = None
 
 SQL_NULL = {"null": {}}
 
@@ -33,44 +31,10 @@ def parse(sql, null=SQL_NULL, calls=simple_op):
 
     with parse_locker:
         if not common_parser:
-            common_parser = sql_parser.common_parser()
+            common_parser = aquery_parser.parser.common_parser()
         result = _parse(common_parser, sql, null, calls)
         return result
 
-
-def parse_mysql(sql, null=SQL_NULL, calls=simple_op):
-    """
-    PARSE MySQL ASSUME DOUBLE QUOTED STRINGS ARE LITERALS
-    :param sql: String of SQL
-    :param null: What value to use as NULL (default is the null function `{"null":{}}`)
-    :return: parse tree
-    """
-    global mysql_parser
-
-    with parse_locker:
-        if not mysql_parser:
-            mysql_parser = sql_parser.mysql_parser()
-        return _parse(mysql_parser, sql, null, calls)
-
-
-def parse_sqlserver(sql, null=SQL_NULL, calls=simple_op):
-    """
-    PARSE MySQL ASSUME DOUBLE QUOTED STRINGS ARE LITERALS
-    :param sql: String of SQL
-    :param null: What value to use as NULL (default is the null function `{"null":{}}`)
-    :return: parse tree
-    """
-    global sqlserver_parser
-
-    with parse_locker:
-        if not sqlserver_parser:
-            sqlserver_parser = sql_parser.sqlserver_parser()
-        return _parse(sqlserver_parser, sql, null, calls)
-
-
-parse_bigquery = parse_mysql
-
-
 def _parse(parser, sql, null, calls):
     utils.null_locations = []
     utils.scrub_op = calls
@@ -85,4 +49,4 @@ def _parse(parser, sql, null, calls):
 
 _ = json.dumps
 
-__all__ = ["parse", "format", "parse_mysql", "parse_bigquery", "normal_op", "simple_op"]
+__all__ = ["parse", "format", "normal_op", "simple_op"]

aquery_parser/keywords.py

@@ -5,7 +5,7 @@
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 #
 # Contact: Kyle Lahnakoski (kyle@lahnakoski.com)
-#
+# Bill Sun  2022 - 2023
 
 # SQL CONSTANTS
 from mo_parsing import *

aquery_parser/parser.py

@@ -5,7 +5,7 @@
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 #
 # Contact: Kyle Lahnakoski (kyle@lahnakoski.com)
-#
+# Bill Sun  2022 - 2023
 
 from sre_parse import WHITESPACE
 
@@ -28,37 +28,13 @@ simple_ident = Regex(simple_ident.__regex__()[1])
 
 def common_parser():
     combined_ident = Combine(delimited_list(
-        ansi_ident | mysql_backtick_ident | simple_ident, separator=".", combine=True,
+        ansi_ident | aquery_backtick_ident | simple_ident, separator=".", combine=True,
     )).set_parser_name("identifier")
 
-    return parser(ansi_string | mysql_doublequote_string, combined_ident)
+    return parser(ansi_string | aquery_doublequote_string, combined_ident)
 
-
-def mysql_parser():
-    mysql_string = ansi_string | mysql_doublequote_string
-    mysql_ident = Combine(delimited_list(
-        mysql_backtick_ident | sqlserver_ident | simple_ident,
-        separator=".",
-        combine=True,
-    )).set_parser_name("mysql identifier")
-
-    return parser(mysql_string, mysql_ident)
-
-
-def sqlserver_parser():
-    combined_ident = Combine(delimited_list(
-        ansi_ident
-        | mysql_backtick_ident
-        | sqlserver_ident
-        | Word(FIRST_IDENT_CHAR, IDENT_CHAR),
-        separator=".",
-        combine=True,
-    )).set_parser_name("identifier")
-
-    return parser(ansi_string, combined_ident, sqlserver=True)
-
-
-def parser(literal_string, ident, sqlserver=False):
+    
+def parser(literal_string, ident):
     with Whitespace() as engine:
         engine.add_ignore(Literal("--") + restOfLine)
         engine.add_ignore(Literal("#") + restOfLine)
@@ -184,12 +160,10 @@ def parser(literal_string, ident, sqlserver=False):
             )
         )
 
-        if not sqlserver:
-            # SQL SERVER DOES NOT SUPPORT [] FOR ARRAY CONSTRUCTION (USED FOR IDENTIFIERS)
-            create_array = (
-                Literal("[") + delimited_list(Group(expr))("args") + Literal("]")
-                | create_array
-            )
+        create_array = (
+            Literal("[") + delimited_list(Group(expr))("args") + Literal("]")
+            | create_array
+        )
 
         create_array = create_array / to_array
 
@@ -596,8 +570,26 @@ def parser(literal_string, ident, sqlserver=False):
             + index_type
             + index_column_names
             + index_options
-        )("create index")
+        )("create_index")
 
+        create_trigger = (
+            keyword("create trigger")
+            + var_name("name")
+            + ((
+                ON
+                + var_name("table")
+                + keyword("action") 
+                + var_name("action")
+                + WHEN
+                + var_name("query")            )  
+            | (
+                keyword("action")
+                + var_name("action")
+                + INTERVAL 
+                + int_num("interval")
+            ))
+        )("create_trigger")
+        
         cache_options = Optional((
             keyword("options").suppress()
             + LB
@@ -720,7 +712,7 @@ def parser(literal_string, ident, sqlserver=False):
         sql_stmts = delimited_list( (
             query
             | (insert | update | delete | load)
-            | (create_table | create_view | create_cache | create_index)
+            | (create_table | create_view | create_cache | create_index | create_trigger)
             | (drop_table | drop_view | drop_index)
         )("stmts"), ";")
 
@@ -734,6 +726,5 @@ def parser(literal_string, ident, sqlserver=False):
             |other_stmt
             | keyword(";").suppress() # empty stmt
         )
-        
         return stmts.finalize()
-
+    

aquery_parser/types.py

@@ -5,7 +5,7 @@
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 #
 # Contact: Kyle Lahnakoski (kyle@lahnakoski.com)
-#
+# Bill Sun  2022 - 2023
 
 
 # KNOWN TYPES

aquery_parser/utils.py

@@ -5,7 +5,7 @@
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 #
 # Contact: Kyle Lahnakoski (kyle@lahnakoski.com)
-#
+# Bill Sun  2022 - 2023
 
 import ast
 
@@ -610,9 +610,8 @@ hex_num = (
 
 # STRINGS
 ansi_string = Regex(r"\'(\'\'|[^'])*\'") / to_string
-mysql_doublequote_string = Regex(r'\"(\"\"|[^"])*\"') / to_string
+aquery_doublequote_string = Regex(r'\"(\"\"|[^"])*\"') / to_string
 
 # BASIC IDENTIFIERS
 ansi_ident = Regex(r'\"(\"\"|[^"])*\"') / unquote
-mysql_backtick_ident = Regex(r"\`(\`\`|[^`])*\`") / unquote
-sqlserver_ident = Regex(r"\[(\]\]|[^\]])*\]") / unquote
+aquery_backtick_ident = Regex(r"\`(\`\`|[^`])*\`") / unquote

benchmark/quries/Aquery/load_data.a

@@ -0,0 +1,6 @@
+CREATE TABLE trade01m(stocksymbol STRING, time INT, quantity INT, price INT)
+load data infile "../tables/trade01m.csv" into table trade01m fields terminated by ','
+CREATE TABLE trade1m(stocksymbol STRING, time INT, quantity INT, price INT)
+load data infile "../tables/trade1m.csv" into table trade1m fields terminated by ','
+CREATE TABLE trade10m(stocksymbol STRING, time INT, quantity INT, price INT)
+load data infile "../tables/trade10m.csv" into table trade10m fields terminated by ','

benchmark/quries/Aquery/q0.a

@@ -0,0 +1,5 @@
+-- select rows
+<sql>
+CREATE TABLE res0 AS
+SELECT * FROM trade10m
+</sql>

benchmark/quries/Aquery/q1.a

@@ -0,0 +1,7 @@
+-- groupby_multi_different_functions
+<sql>
+CREATE TABLE res1 AS
+SELECT avg(quantity) AS avg_quan, min(price) AS min_p
+FROM trade1m
+GROUP BY stocksymbol, time
+</sql>

benchmark/quries/Aquery/q10.a

@@ -0,0 +1,4 @@
+SELECT stocksymbol, MAX(stddevs(3, price))
+FROM trade1m
+ASSUMING ASC time
+GROUP BY stocksymbol

benchmark/quries/Aquery/q2.a

@@ -0,0 +1,4 @@
+-- count values
+<sql>
+SELECT COUNT(*) FROM trade10m
+</sql>

benchmark/quries/Aquery/q3.a

@@ -0,0 +1,7 @@
+-- group by multiple keys
+<sql>
+create table res3 AS
+SELECT sum(quantity) as sum_quantity
+FROM trade01m
+GROUP BY stocksymbol, price
+</sql>

benchmark/quries/Aquery/q4.a

@@ -0,0 +1,5 @@
+-- append tables
+<sql>
+CREATE TABLE res4 AS
+SELECT * FROM trade10m UNION ALL SELECT * FROM trade10m
+</sql>

benchmark/quries/Aquery/q7.a

@@ -0,0 +1,5 @@
+CREATE table res7 AS
+SELECT stocksymbol, avgs(5, price)
+FROM trade10m
+ASSUMING ASC time
+GROUP BY stocksymbol

benchmark/quries/Aquery/q8.a

@@ -0,0 +1,6 @@
+<sql>
+CREATE TABLE res8 AS
+SELECT stocksymbol, quantity, price
+FROM trade10m
+WHERE time >= 5288 and time <= 7000
+</sql>

benchmark/quries/Aquery/q9.a

@@ -0,0 +1,6 @@
+<sql>
+CREATE TABLE res9 AS
+SELECT stocksymbol, MAX(price) - MIN(price)
+FROM trade10m
+GROUP BY stocksymbol
+</sql>

benchmark/quries/Clickhouse/q0

@@ -0,0 +1,3 @@
+-- q0 select rows
+CREATE TABLE res0 (a String, b Int32, c Int32, d Int32) ENGINE = MergeTree() ORDER BY b AS
+SELECT * FROM benchmark.trade10m

benchmark/quries/Clickhouse/q1

@@ -0,0 +1,4 @@
+-- groupby_multi_different_functions
+SELECT avg(quantity), min(price)
+FROM benchmark.trade10m
+GROUP BY stocksymbol, time

benchmark/quries/Clickhouse/q10

@@ -0,0 +1,8 @@
+-- max rolling std
+select
+    stocksymbol,
+    max(stddevPop(price)) over
+        (partition by stocksymbol rows between 2 preceding AND CURRENT row) as maxRollingStd
+from 
+(SELECT * FROM benchmark.trade01m ORDER BY time)
+GROUP BY stocksymbol

benchmark/quries/Clickhouse/q2

@@ -0,0 +1,2 @@
+-- count values
+SELECT COUNT(*) FROM benchmark.trade10m

benchmark/quries/Clickhouse/q3

@@ -0,0 +1,4 @@
+-- group by multiple keys
+SELECT sum(quantity)
+FROM benchmark.trade10m
+GROUP BY stocksymbol, price

benchmark/quries/Clickhouse/q4

@@ -0,0 +1,2 @@
+-- append two tables
+SELECT * FROM benchmark.trade10m UNION ALL SELECT * FROM benchmark.trade10m

benchmark/quries/Clickhouse/q7

@@ -0,0 +1,5 @@
+-- moving_avg
+SELECT stocksymbol, groupArrayMovingAvg(5)(price) AS moving_avg_price
+FROM 
+(SELECT * FROM benchmark.trade01m ORDER BY time)
+GROUP BY stocksymbol

benchmark/quries/Clickhouse/q8

@@ -0,0 +1,3 @@
+SELECT stocksymbol, quantity, price
+FROM benchmark.trade10m
+WHERE time >= 5288 and time <= 7000

benchmark/quries/Clickhouse/q9

@@ -0,0 +1,3 @@
+SELECT stocksymbol, MAX(price) - MIN(price)
+FROM benchmark.trade1m
+GROUP BY stocksymbol

benchmark/quries/Timescaledb/q0

@@ -0,0 +1,3 @@
+-- select rows
+CREATE TABLE res0 AS 
+SELECT * FROM trade10m;

benchmark/quries/Timescaledb/q1

@@ -0,0 +1,4 @@
+-- groupby_multi_different_functions
+SELECT avg(quantity), min(price)
+FROM trade10m
+GROUP BY stocksymbol, time;

benchmark/quries/Timescaledb/q10

@@ -0,0 +1,7 @@
+select
+    stocksymbol,
+    max(stddev(price)) over
+        (partition by stocksymbol rows between 2 preceding AND CURRENT row) as maxRollingStd
+from 
+(SELECT * FROM trade01m ORDER BY time) as t
+GROUP BY stocksymbol;

benchmark/quries/Timescaledb/q2

@@ -0,0 +1,2 @@
+-- count values
+SELECT COUNT(*) FROM trade10m;

benchmark/quries/Timescaledb/q3

@@ -0,0 +1,4 @@
+-- group by multiple keys
+SELECT sum(quantity)
+FROM trade10m
+GROUP BY stocksymbol, price;

benchmark/quries/Timescaledb/q4

@@ -0,0 +1,2 @@
+-- append tables
+SELECT * FROM trade10m UNION ALL SELECT * FROM trade10m;

benchmark/quries/Timescaledb/q7

@@ -0,0 +1,5 @@
+select
+    stocksymbol,
+    coalesce(avg(price) over
+        (partition by stocksymbol order by time rows between 4 preceding AND CURRENT row), price) as rollingAvg
+from trade10m;

benchmark/quries/Timescaledb/q8

@@ -0,0 +1,3 @@
+SELECT stocksymbol, quantity, price
+FROM trade01m
+WHERE time >= 5288 and time <= 7000

benchmark/quries/Timescaledb/q9

@@ -0,0 +1,3 @@
+SELECT stocksymbol, MAX(price) - MIN(price)
+FROM trade01m
+GROUP BY stocksymbol;

build.py

@@ -117,7 +117,7 @@ class build_manager:
             else:
                 mgr.cxx = os.environ['CXX']
             if 'AQ_DEBUG' not in os.environ:
-                os.environ['AQ_DEBUG'] = '0' if mgr.OptimizationLv else '1'
+                os.environ['AQ_DEBUG'] = ('0' if mgr.OptimizationLv != '0' else '1')
 
         def libaquery_a(self):
             self.build_cmd = [['rm', 'libaquery.a'],['make', 'libaquery']]

datagen.cpp

@@ -52,7 +52,7 @@ int gen_trade_data(int argc, char* argv[])
         printf("No parameter supplied. Use default frac=%f, N=%d? [Y/n]\n", frac, N);
         char buf[4096]; fgets(buf, 4095, stdin); 
         if((buf[0] != 'y' && buf[0] !='Y') && buf[0] != '\n') { 
-            const auto &getParams = [&](){ puts("Type: frac N [ENTER]");
+            const auto &getParams = [&](){ puts("Type: frac N n_rows [ENTER]");
                 for(int i = 0; i < 4096; ++i) if(buf[i] == '\n') {buf[i] = 0; break;}
                 char* _buf = buf; frac = getFloat(_buf); N = getInt(_buf); n_rows=getInt(_buf);
             }; getParams();
@@ -80,7 +80,7 @@ int gen_trade_data(int argc, char* argv[])
         memmove(p + lens[i], p + lens[0], (lens[i - 1] - lens[i]) * sizeof(int));
     permutation(p, lens[0] + N);
     // for (int i = 0; i < lens[0] + N; ++i) printf("%d ", p[i]);
-    FILE* fp = fopen("trade.csv", "w");
+    FILE* fp = fopen("trade.csv", "wb");
     int* last_price = new int[N];
     memset(last_price, -1, sizeof(int) * N);
     fprintf(fp, "stocksymbol, time, quantity, price\n");
@@ -102,7 +102,7 @@ int gen_trade_data(int argc, char* argv[])
             current_price = new_price;
         }
         d[i]= {ss, i+1, (int)g_quantity(engine), current_price};
-        fprintf(fp, "s%d, %d, %d, %d\n", d[i].ss, d[i].t, d[i].q, d[i].p);
+        fprintf(fp, "%d, %d, %d, %d\n", d[i].ss, d[i].t, d[i].q, d[i].p);
         last_price[ss - 1] = current_price;
     }
     fclose(fp);
@@ -131,7 +131,7 @@ int gen_stock_data(int argc, char* argv[]){
     }
     IDs[n_stocks] = "S";
     names[n_stocks] = "x";
-    FILE* fp = fopen("./data/stock.csv", "w");
+    FILE* fp = fopen("./data/stock.csv", "wb");
     fprintf(fp, "ID, timestamp, tradeDate, price\n");
     char date_str_buf [types::date_t::string_length()];
     int* timestamps = new int[n_data];
@@ -142,14 +142,15 @@ int gen_stock_data(int argc, char* argv[]){
         fprintf(fp, "%s,%d,%s,%d\n", IDs[ui(engine)%(n_stocks + 1)].c_str(), timestamps[i], date, ui(engine) % 1000);
     }
     fclose(fp);
-    fp = fopen("./data/base.csv", "w");
+    fp = fopen("./data/base.csv", "wb");
     fprintf(fp, "ID, name\n");
     for(int i = 0; i < n_stocks + 1; ++ i){
         fprintf(fp, "%s,%s\n", IDs[i].c_str(), names[i].c_str());
     }
     fclose(fp);
+    return 0;
 }
 
 int main(int argc, char* argv[]){
-    return gen_stock_data(argc, argv);
+    return gen_trade_data(argc, argv);
 }

engine/ddl.py

@@ -110,7 +110,7 @@ class outfile(ast_node):
         filename = node['loc']['literal'] if 'loc' in node else node['literal']
         sep = ',' if 'term' not in node else node['term']['literal']
         file_pointer = 'fp_' + base62uuid(6)
-        self.emit(f'FILE* {file_pointer} = fopen("{filename}", "w");')
+        self.emit(f'FILE* {file_pointer} = fopen("{filename}", "wb");')
         self.emit(f'{out_table.cxt_name}->printall("{sep}", "\\n", nullptr, {file_pointer});')
         self.emit(f'fclose({file_pointer});')
         # self.context.headers.add('fstream')

engine/orderby.py

@@ -43,7 +43,7 @@ class orderby(ast_node):
     def merge(self, node):
         self.produce(node)
         
-    def finialize(self, references):
+    def finalize(self, references):
         self.order = [ o for o in self.order if o.name in references ]
     
     def result(self, sep:str = ','):

engine/types.py

@@ -107,9 +107,9 @@ ULongT = Types(8, name = 'uint64', sqlname = 'UINT64', fp_type=DoubleT)
 UIntT = Types(7, name = 'uint32', sqlname = 'UINT32', long_type=ULongT, fp_type=FloatT)
 UShortT = Types(6, name = 'uint16', sqlname = 'UINT16', long_type=ULongT, fp_type=FloatT)
 UByteT = Types(5, name = 'uint8', sqlname = 'UINT8', long_type=ULongT, fp_type=FloatT)
-StrT = Types(200, name = 'str', cname = 'const char*', sqlname='TEXT', ctype_name = 'types::ASTR')
-TextT = Types(200, name = 'text', cname = 'const char*', sqlname='TEXT', ctype_name = 'types::ASTR')
-VarcharT = Types(200, name = 'varchar', cname = 'const char*', sqlname='VARCHAR', ctype_name = 'types::ASTR')
+StrT = Types(200, name = 'str', cname = 'string_view', sqlname='TEXT', ctype_name = 'types::ASTR')
+TextT = Types(200, name = 'text', cname = 'string_view', sqlname='TEXT', ctype_name = 'types::ASTR')
+VarcharT = Types(200, name = 'varchar', cname = 'string_view', sqlname='VARCHAR', ctype_name = 'types::ASTR')
 VoidT = Types(200, name = 'void', cname = 'void', sqlname='Null', ctype_name = 'types::None')
 
 class VectorT(Types):
@@ -305,7 +305,7 @@ opor = OperatorBase('or', 2, logical, cname = '||', sqlname = ' OR ', call = bin
 opxor = OperatorBase('xor', 2, logical, cname = '^', sqlname = ' XOR ', call = binary_op_behavior)
 opgt = OperatorBase('gt', 2, logical, cname = '>', sqlname = '>', call = binary_op_behavior)
 oplt = OperatorBase('lt', 2, logical, cname = '<', sqlname = '<', call = binary_op_behavior)
-opge = OperatorBase('gte', 2, logical, cname = '>=', sqlname = '>=', call = binary_op_behavior)
+opgte = OperatorBase('gte', 2, logical, cname = '>=', sqlname = '>=', call = binary_op_behavior)
 oplte = OperatorBase('lte', 2, logical, cname = '<=', sqlname = '<=', call = binary_op_behavior)
 opneq = OperatorBase('neq', 2, logical, cname = '!=', sqlname = '!=', call = binary_op_behavior)
 opeq = OperatorBase('eq', 2, logical, cname = '==', sqlname = '=', call = binary_op_behavior)
@@ -355,19 +355,27 @@ fnpow = OperatorBase('pow', 2, lambda *_ : DoubleT, cname = 'pow', sqlname = 'PO
 # type collections
 def _op_make_dict(*items : OperatorBase):
     return { i.name: i for i in items}
+#binary op
 builtin_binary_arith = _op_make_dict(opadd, opdiv, opmul, opsub, opmod)
 builtin_binary_logical = _op_make_dict(opand, opor, opxor, opgt, oplt, 
-                                       opge, oplte, opneq, opeq)
+                                       opgte, oplte, opneq, opeq)
+builtin_binary_ops = {**builtin_binary_arith, **builtin_binary_logical}
+#unary op
 builtin_unary_logical = _op_make_dict(opnot)
 builtin_unary_arith = _op_make_dict(opneg)
 builtin_unary_special = _op_make_dict(spnull, opdistinct)
+# functions 
 builtin_cstdlib = _op_make_dict(fnsqrt, fnlog, fnsin, fncos, fntan, fnpow)
-builtin_func = _op_make_dict(fnmax, fnmin, fnsum, fnavg, fnmaxs, 
-                             fnmins, fndeltas, fnratios, fnlast,
-                             fnfirst, fnsums, fnavgs, fncnt, 
-                             fnpack, fntrunc, fnprev, fnnext, 
-                             fnvar, fnvars, fnstd, fnstds)
+builtin_aggfunc = _op_make_dict(fnmax, fnmin, fnsum, fnavg, 
+                                fnlast, fnfirst, fncnt, fnvar, fnstd)
+builtin_vecfunc = _op_make_dict(fnmaxs, 
+                             fnmins, fndeltas, fnratios, fnsums, fnavgs, 
+                             fnpack, fntrunc, fnprev, fnnext, fnvars, fnstds)
+builtin_vecfunc = {**builtin_vecfunc, **builtin_cstdlib}
+builtin_func = {**builtin_vecfunc, **builtin_aggfunc}
+
 user_module_func = {}
+
 builtin_operators : Dict[str, OperatorBase] = {**builtin_binary_arith, **builtin_binary_logical, 
     **builtin_unary_arith, **builtin_unary_logical, **builtin_unary_special, **builtin_func, **builtin_cstdlib, 
     **user_module_func}

engine/utils.py

@@ -8,7 +8,7 @@ nums = '0123456789'
 base62alp = nums + lower_alp + upper_alp
 
 reserved_monet = ['month']
-
+session_context = None
 
 class CaseInsensitiveDict(MutableMapping):
     def __init__(self, data=None, **kwargs):
@@ -157,4 +157,36 @@ def get_innermost(sl):
     elif sl and type(sl) is list:
         return get_innermost(sl[0])
     else:
-        return sl
+        return sl
+
+
+def send_to_server(payload : str):
+    from prompt import PromptState
+    cxt : PromptState = session_context
+    if cxt is None:
+        raise RuntimeError("Error! no session specified.")
+    else:
+        from ctypes import c_char_p
+        cxt.payload = (c_char_p*1)(c_char_p(bytes(payload, 'utf-8')))
+        cxt.cfg.has_dll = 0
+        cxt.send(1, cxt.payload)
+        cxt.set_ready()
+
+def get_storedproc(name : str):
+    from prompt import PromptState, StoredProcedure
+    cxt : PromptState = session_context
+    if cxt is None:
+        raise RuntimeError("Error! no session specified.")
+    else:
+        ret : StoredProcedure = cxt.get_storedproc(bytes(name, 'utf-8'))
+        if (
+            ret.name.value and 
+            ret.name.value.decode('utf-8') != name
+        ):
+            print(f'Procedure {name} mismatch in server {ret.name.value}')
+            return None
+        else:
+            return ret
+
+def execute_procedure(proc):
+    pass

header.cxx

@@ -0,0 +1,13 @@
+#include "./server/libaquery.h"
+
+#ifndef __AQ_USE_THREADEDGC__
+
+#include "./server/gc.h"
+__AQEXPORT__(void) __AQ_Init_GC__(Context* cxt) {
+    GC::gc_handle = static_cast<GC*>(cxt->gc);
+    GC::scratch_space = nullptr;
+}
+
+#else // __AQ_USE_THREADEDGC__
+#define __AQ_Init_GC__(x)
+#endif // __AQ_USE_THREADEDGC__

maketest.sh

@@ -0,0 +1,2 @@
+make snippet_uselib
+cp ./dll.so procedures/q70.so

mem_opt.cpp

@@ -0,0 +1,72 @@
+#include "./server/libaquery.h"
+
+#ifndef __AQ_USE_THREADEDGC__
+
+#include "./server/gc.h"
+__AQEXPORT__(void) __AQ_Init_GC__(Context* cxt) {
+    GC::gc_handle = static_cast<GC*>(cxt->gc);
+}
+
+#else // __AQ_USE_THREADEDGC__
+#define __AQ_Init_GC__(x)
+#endif // __AQ_USE_THREADEDGC__
+#include "./server/hasher.h"
+#include "./server/monetdb_conn.h"
+#include "./server/aggregations.h"
+
+__AQEXPORT__(int) dll_2Cxoox(Context* cxt) {
+	using namespace std;
+	using namespace types;
+	auto server = static_cast<Server*>(cxt->alt_server);
+auto len_4ycjiV = server->cnt;
+auto mont_8AE = ColRef<const char*>(len_4ycjiV, server->getCol(0));
+auto sales_2RB = ColRef<int>(len_4ycjiV, server->getCol(1));
+const char* names_6pIt[] = {"mont", "minw2ysales"};
+auto out_2LuaMH = new TableInfo<const char*,vector_type<double>>("out_2LuaMH", names_6pIt);
+decltype(auto) col_EeW23s = out_2LuaMH->get_col<0>();
+decltype(auto) col_5gY1Dm = out_2LuaMH->get_col<1>();
+typedef record<decays<decltype(mont_8AE)::value_t>> record_typegj3e8Xf;
+ankerl::unordered_dense::map<record_typegj3e8Xf, uint32_t, transTypes<record_typegj3e8Xf, hasher>> gMzMTEvd;
+gMzMTEvd.reserve(mont_8AE.size);
+uint32_t* reversemap = new uint32_t[mont_8AE.size<<1], 
+	*mapbase = reversemap + mont_8AE.size;
+for (uint32_t i2E = 0; i2E < mont_8AE.size; ++i2E){
+	reversemap[i2E] = gMzMTEvd.hashtable_push(forward_as_tuple(mont_8AE[i2E]));
+}
+auto arr_values = gMzMTEvd.values().data();
+auto arr_len = gMzMTEvd.size();
+uint32_t* seconds = new uint32_t[gMzMTEvd.size()];
+
+auto vecs = static_cast<vector_type<uint32_t>*>(malloc(sizeof(vector_type<uint32_t>) * arr_len));
+vecs[0].init_from(arr_values[0].second, mapbase);
+for (uint32_t i = 1; i < arr_len; ++i) {
+	vecs[i].init_from(arr_values[i].second, mapbase + arr_values[i - 1].second);
+	arr_values[i].second += arr_values[i - 1].second;
+}
+for (uint32_t i = 0; i < mont_8AE.size; ++i) {
+	auto id = reversemap[i];
+	mapbase[--arr_values[id].second] = i;    
+}
+
+col_EeW23s.reserve(gMzMTEvd.size());
+col_5gY1Dm.reserve(gMzMTEvd.size());
+auto buf_col_5gY1Dm = new double[mont_8AE.size];
+for (uint32_t i = 0; i < arr_len; ++i) {
+	col_5gY1Dm[i].init_from(vecs[i].size, buf_col_5gY1Dm + arr_values[i].second);
+}
+for (uint32_t i = 0; i < arr_len; ++i) {
+
+auto &key_3iNX3qG = arr_values[i].first;
+auto &val_7jjv8Mo = arr_values[i].second;
+col_EeW23s.emplace_back(get<0>(key_3iNX3qG));
+
+avgw(10, sales_2RB[vecs[i]], col_5gY1Dm[i]);
+
+}
+//print(*out_2LuaMH);
+//FILE* fp_5LQeym = fopen("flatten.csv", "wb");
+out_2LuaMH->printall(",", "\n", nullptr, nullptr, 10);
+//fclose(fp_5LQeym);
+puts("done.");
+return 0;
+}

msc-plugin/launcher.vcxproj

@@ -17,6 +17,14 @@
       <Configuration>Release</Configuration>
       <Platform>x64</Platform>
     </ProjectConfiguration>
+    <ProjectConfiguration Include="RelWDbgInf|Win32">
+      <Configuration>RelWDbgInf</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="RelWDbgInf|x64">
+      <Configuration>RelWDbgInf</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
     <ProjectConfiguration Include="sharedlib|Win32">
       <Configuration>sharedlib</Configuration>
       <Platform>Win32</Platform>
@@ -52,6 +60,14 @@
     <CharacterSet>Unicode</CharacterSet>
     <EnableASAN>false</EnableASAN>
   </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='RelWDbgInf|Win32'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <UseDebugLibraries>false</UseDebugLibraries>
+    <PlatformToolset>v143</PlatformToolset>
+    <WholeProgramOptimization>true</WholeProgramOptimization>
+    <CharacterSet>Unicode</CharacterSet>
+    <EnableASAN>false</EnableASAN>
+  </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='sharedlib|Win32'" Label="Configuration">
     <ConfigurationType>Application</ConfigurationType>
     <UseDebugLibraries>false</UseDebugLibraries>
@@ -75,6 +91,14 @@
     <CharacterSet>Unicode</CharacterSet>
     <EnableASAN>false</EnableASAN>
   </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='RelWDbgInf|x64'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <UseDebugLibraries>false</UseDebugLibraries>
+    <PlatformToolset>v143</PlatformToolset>
+    <WholeProgramOptimization>true</WholeProgramOptimization>
+    <CharacterSet>Unicode</CharacterSet>
+    <EnableASAN>false</EnableASAN>
+  </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='sharedlib|x64'" Label="Configuration">
     <ConfigurationType>Application</ConfigurationType>
     <UseDebugLibraries>false</UseDebugLibraries>
@@ -94,6 +118,9 @@
   <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
     <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
   </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='RelWDbgInf|Win32'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
   <ImportGroup Condition="'$(Configuration)|$(Platform)'=='sharedlib|Win32'" Label="PropertySheets">
     <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
   </ImportGroup>
@@ -103,6 +130,9 @@
   <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
     <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
   </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='RelWDbgInf|x64'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
   <ImportGroup Condition="'$(Configuration)|$(Platform)'=='sharedlib|x64'" Label="PropertySheets">
     <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
   </ImportGroup>
@@ -117,6 +147,11 @@
     <TargetExt>.exe</TargetExt>
     <OutDir>$(SolutionDir)..\</OutDir>
   </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='RelWDbgInf|Win32'">
+    <LinkIncremental>false</LinkIncremental>
+    <TargetExt>.exe</TargetExt>
+    <OutDir>$(SolutionDir)..\</OutDir>
+  </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='sharedlib|Win32'">
     <LinkIncremental>false</LinkIncremental>
     <TargetExt>.exe</TargetExt>
@@ -132,6 +167,11 @@
     <TargetExt>.exe</TargetExt>
     <OutDir>$(SolutionDir)..\</OutDir>
   </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='RelWDbgInf|x64'">
+    <LinkIncremental>false</LinkIncremental>
+    <TargetExt>.exe</TargetExt>
+    <OutDir>$(SolutionDir)..\</OutDir>
+  </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='sharedlib|x64'">
     <LinkIncremental>false</LinkIncremental>
     <TargetExt>.exe</TargetExt>
@@ -145,7 +185,7 @@
       <ConformanceMode>true</ConformanceMode>
       <MultiProcessorCompilation>true</MultiProcessorCompilation>
       <EnableParallelCodeGeneration>true</EnableParallelCodeGeneration>
-      <LanguageStandard>stdcpp17</LanguageStandard>
+      <LanguageStandard>stdcpplatest</LanguageStandard>
       <LanguageStandard_C>stdc17</LanguageStandard_C>
       <AdditionalIncludeDirectories>$(ProjectDir)\..\monetdb\msvc</AdditionalIncludeDirectories>
     </ClCompile>
@@ -166,7 +206,32 @@
       <ConformanceMode>true</ConformanceMode>
       <MultiProcessorCompilation>true</MultiProcessorCompilation>
       <EnableParallelCodeGeneration>true</EnableParallelCodeGeneration>
-      <LanguageStandard>stdcpp17</LanguageStandard>
+      <LanguageStandard>stdcpplatest</LanguageStandard>
+      <LanguageStandard_C>stdc17</LanguageStandard_C>
+      <AdditionalIncludeDirectories>$(ProjectDir)\..\monetdb\msvc</AdditionalIncludeDirectories>
+      <DebugInformationFormat>None</DebugInformationFormat>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <EnableCOMDATFolding>true</EnableCOMDATFolding>
+      <OptimizeReferences>true</OptimizeReferences>
+      <GenerateDebugInformation>false</GenerateDebugInformation>
+      <AdditionalDependencies>..\libaquery.lib;$(ProjectDir)\..\monetdb\msvc\monetdbe.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <DataExecutionPrevention>false</DataExecutionPrevention>
+      <AdditionalOptions>/WHOLEARCHIVE:libaquery.lib %(AdditionalOptions)</AdditionalOptions>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='RelWDbgInf|Win32'">
+    <ClCompile>
+      <WarningLevel>Level3</WarningLevel>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <IntrinsicFunctions>true</IntrinsicFunctions>
+      <SDLCheck>true</SDLCheck>
+      <PreprocessorDefinitions>_CRT_SECURE_NO_WARNINGS;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <ConformanceMode>true</ConformanceMode>
+      <MultiProcessorCompilation>true</MultiProcessorCompilation>
+      <EnableParallelCodeGeneration>true</EnableParallelCodeGeneration>
+      <LanguageStandard>stdcpplatest</LanguageStandard>
       <LanguageStandard_C>stdc17</LanguageStandard_C>
       <AdditionalIncludeDirectories>$(ProjectDir)\..\monetdb\msvc</AdditionalIncludeDirectories>
       <DebugInformationFormat>None</DebugInformationFormat>
@@ -215,9 +280,10 @@
       <ConformanceMode>true</ConformanceMode>
       <MultiProcessorCompilation>true</MultiProcessorCompilation>
       <EnableParallelCodeGeneration>true</EnableParallelCodeGeneration>
-      <LanguageStandard>stdcpp17</LanguageStandard>
+      <LanguageStandard>stdcpplatest</LanguageStandard>
       <LanguageStandard_C>stdc17</LanguageStandard_C>
       <AdditionalIncludeDirectories>$(ProjectDir)\..\monetdb\msvc</AdditionalIncludeDirectories>
+      <BrowseInformation>true</BrowseInformation>
     </ClCompile>
     <Link>
       <SubSystem>Console</SubSystem>
@@ -225,6 +291,9 @@
       <AdditionalDependencies>..\libaquery.lib;$(ProjectDir)\..\monetdb\msvc\monetdbe.lib;%(AdditionalDependencies)</AdditionalDependencies>
       <AdditionalOptions>/WHOLEARCHIVE:libaquery.lib %(AdditionalOptions)</AdditionalOptions>
     </Link>
+    <Bscmake>
+      <PreserveSbr>true</PreserveSbr>
+    </Bscmake>
   </ItemDefinitionGroup>
   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
     <ClCompile>
@@ -236,7 +305,32 @@
       <ConformanceMode>true</ConformanceMode>
       <MultiProcessorCompilation>true</MultiProcessorCompilation>
       <EnableParallelCodeGeneration>true</EnableParallelCodeGeneration>
-      <LanguageStandard>stdcpp17</LanguageStandard>
+      <LanguageStandard>stdcpplatest</LanguageStandard>
+      <LanguageStandard_C>stdc17</LanguageStandard_C>
+      <AdditionalIncludeDirectories>$(ProjectDir)\..\monetdb\msvc</AdditionalIncludeDirectories>
+      <DebugInformationFormat>None</DebugInformationFormat>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <EnableCOMDATFolding>true</EnableCOMDATFolding>
+      <OptimizeReferences>true</OptimizeReferences>
+      <GenerateDebugInformation>false</GenerateDebugInformation>
+      <AdditionalDependencies>..\libaquery.lib;$(ProjectDir)\..\monetdb\msvc\monetdbe.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <DataExecutionPrevention>false</DataExecutionPrevention>
+      <AdditionalOptions>/WHOLEARCHIVE:libaquery.lib %(AdditionalOptions)</AdditionalOptions>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='RelWDbgInf|x64'">
+    <ClCompile>
+      <WarningLevel>Level3</WarningLevel>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <IntrinsicFunctions>true</IntrinsicFunctions>
+      <SDLCheck>true</SDLCheck>
+      <PreprocessorDefinitions>_CRT_SECURE_NO_WARNINGS;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <ConformanceMode>true</ConformanceMode>
+      <MultiProcessorCompilation>true</MultiProcessorCompilation>
+      <EnableParallelCodeGeneration>true</EnableParallelCodeGeneration>
+      <LanguageStandard>stdcpplatest</LanguageStandard>
       <LanguageStandard_C>stdc17</LanguageStandard_C>
       <AdditionalIncludeDirectories>$(ProjectDir)\..\monetdb\msvc</AdditionalIncludeDirectories>
       <DebugInformationFormat>None</DebugInformationFormat>

msc-plugin/libaquery.vcxproj

@@ -1,4 +1,4 @@
-<?xml version="1.0" encoding="utf-8"?>
+<?xml version="1.0" encoding="utf-8"?>
 <Project DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
   <ItemGroup Label="ProjectConfigurations">
     <ProjectConfiguration Include="Debug|Win32">
@@ -17,6 +17,14 @@
       <Configuration>Release</Configuration>
       <Platform>x64</Platform>
     </ProjectConfiguration>
+    <ProjectConfiguration Include="RelWDbgInf|Win32">
+      <Configuration>RelWDbgInf</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="RelWDbgInf|x64">
+      <Configuration>RelWDbgInf</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
   </ItemGroup>
   <PropertyGroup Label="Globals">
     <VCProjectVersion>16.0</VCProjectVersion>
@@ -39,6 +47,13 @@
     <WholeProgramOptimization>true</WholeProgramOptimization>
     <CharacterSet>Unicode</CharacterSet>
   </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='RelWDbgInf|Win32'" Label="Configuration">
+    <ConfigurationType>StaticLibrary</ConfigurationType>
+    <UseDebugLibraries>false</UseDebugLibraries>
+    <PlatformToolset>v143</PlatformToolset>
+    <WholeProgramOptimization>true</WholeProgramOptimization>
+    <CharacterSet>Unicode</CharacterSet>
+  </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
     <ConfigurationType>StaticLibrary</ConfigurationType>
     <UseDebugLibraries>true</UseDebugLibraries>
@@ -52,6 +67,13 @@
     <WholeProgramOptimization>true</WholeProgramOptimization>
     <CharacterSet>Unicode</CharacterSet>
   </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='RelWDbgInf|x64'" Label="Configuration">
+    <ConfigurationType>StaticLibrary</ConfigurationType>
+    <UseDebugLibraries>false</UseDebugLibraries>
+    <PlatformToolset>v143</PlatformToolset>
+    <WholeProgramOptimization>true</WholeProgramOptimization>
+    <CharacterSet>Unicode</CharacterSet>
+  </PropertyGroup>
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
   <ImportGroup Label="ExtensionSettings">
   </ImportGroup>
@@ -63,12 +85,18 @@
   <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
     <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
   </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='RelWDbgInf|Win32'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
   <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
     <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
   </ImportGroup>
   <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
     <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
   </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='RelWDbgInf|x64'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
   <PropertyGroup Label="UserMacros" />
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
     <LinkIncremental>true</LinkIncremental>
@@ -79,6 +107,11 @@
     <TargetExt>.lib</TargetExt>
     <OutDir>$(SolutionDir)..\</OutDir>
   </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='RelWDbgInf|Win32'">
+    <LinkIncremental>false</LinkIncremental>
+    <TargetExt>.lib</TargetExt>
+    <OutDir>$(SolutionDir)..\</OutDir>
+  </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
     <LinkIncremental>true</LinkIncremental>
     <OutDir>$(SolutionDir)..\</OutDir>
@@ -88,13 +121,18 @@
     <TargetExt>.lib</TargetExt>
     <OutDir>$(SolutionDir)..\</OutDir>
   </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='RelWDbgInf|x64'">
+    <LinkIncremental>false</LinkIncremental>
+    <TargetExt>.lib</TargetExt>
+    <OutDir>$(SolutionDir)..\</OutDir>
+  </PropertyGroup>
   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
     <ClCompile>
       <WarningLevel>Level3</WarningLevel>
       <SDLCheck>true</SDLCheck>
-      <PreprocessorDefinitions>_CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>QUERY_DECLSPEC=__declspec(dllexport);_CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <ConformanceMode>true</ConformanceMode>
-      <LanguageStandard>stdcpp17</LanguageStandard>
+      <LanguageStandard>stdcpplatest</LanguageStandard>
       <LanguageStandard_C>stdc17</LanguageStandard_C>
       <PrecompiledHeader>Create</PrecompiledHeader>
       <PrecompiledHeaderFile>pch_msc.hpp</PrecompiledHeaderFile>
@@ -118,9 +156,9 @@
       <FunctionLevelLinking>true</FunctionLevelLinking>
       <IntrinsicFunctions>true</IntrinsicFunctions>
       <SDLCheck>false</SDLCheck>
-      <PreprocessorDefinitions>_CRT_SECURE_NO_WARNINGS;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>QUERY_DECLSPEC=__declspec(dllexport);_CRT_SECURE_NO_WARNINGS;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <ConformanceMode>true</ConformanceMode>
-      <LanguageStandard>stdcpp17</LanguageStandard>
+      <LanguageStandard>stdcpplatest</LanguageStandard>
       <LanguageStandard_C>stdc17</LanguageStandard_C>
       <PrecompiledHeader>Create</PrecompiledHeader>
       <PrecompiledHeaderFile>pch_msc.hpp</PrecompiledHeaderFile>
@@ -153,13 +191,54 @@
       <AdditionalDependencies>$(ProjectDir)\..\monetdb\msvc\monetdbe.lib;</AdditionalDependencies>
     </Lib>
   </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='RelWDbgInf|Win32'">
+    <ClCompile>
+      <WarningLevel>Level3</WarningLevel>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <IntrinsicFunctions>true</IntrinsicFunctions>
+      <SDLCheck>false</SDLCheck>
+      <PreprocessorDefinitions>QUERY_DECLSPEC=__declspec(dllexport);_CRT_SECURE_NO_WARNINGS;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <ConformanceMode>true</ConformanceMode>
+      <LanguageStandard>stdcpplatest</LanguageStandard>
+      <LanguageStandard_C>stdc17</LanguageStandard_C>
+      <PrecompiledHeader>Create</PrecompiledHeader>
+      <PrecompiledHeaderFile>pch_msc.hpp</PrecompiledHeaderFile>
+      <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
+      <Optimization>Full</Optimization>
+      <InlineFunctionExpansion>AnySuitable</InlineFunctionExpansion>
+      <FavorSizeOrSpeed>Speed</FavorSizeOrSpeed>
+      <OmitFramePointers>true</OmitFramePointers>
+      <EnableFiberSafeOptimizations>true</EnableFiberSafeOptimizations>
+      <EnableEnhancedInstructionSet>AdvancedVectorExtensions2</EnableEnhancedInstructionSet>
+      <MultiProcessorCompilation>true</MultiProcessorCompilation>
+      <StringPooling>true</StringPooling>
+      <ExceptionHandling>false</ExceptionHandling>
+      <BufferSecurityCheck>false</BufferSecurityCheck>
+      <ControlFlowGuard>false</ControlFlowGuard>
+      <EnableParallelCodeGeneration>true</EnableParallelCodeGeneration>
+      <FloatingPointExceptions>false</FloatingPointExceptions>
+      <AdditionalIncludeDirectories>$(ProjectDir)\..\monetdb\msvc</AdditionalIncludeDirectories>
+      <PrecompiledHeaderOutputFile>$(SolutionDir)..\msc-plugin\pch_msc.pch</PrecompiledHeaderOutputFile>
+      <ProgramDataBaseFileName>$(IntDir)vc$(PlatformToolsetVersion).pdb</ProgramDataBaseFileName>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <EnableCOMDATFolding>true</EnableCOMDATFolding>
+      <OptimizeReferences>true</OptimizeReferences>
+      <GenerateDebugInformation>false</GenerateDebugInformation>
+      <AssemblyDebug>false</AssemblyDebug>
+    </Link>
+    <Lib>
+      <AdditionalDependencies>$(ProjectDir)\..\monetdb\msvc\monetdbe.lib;</AdditionalDependencies>
+    </Lib>
+  </ItemDefinitionGroup>
   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
     <ClCompile>
       <WarningLevel>Level3</WarningLevel>
       <SDLCheck>true</SDLCheck>
-      <PreprocessorDefinitions>_CRT_SECURE_NO_WARNINGS;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>QUERY_DECLSPEC=__declspec(dllexport);_CRT_SECURE_NO_WARNINGS;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <ConformanceMode>true</ConformanceMode>
-      <LanguageStandard>stdcpp17</LanguageStandard>
+      <LanguageStandard>stdcpplatest</LanguageStandard>
       <LanguageStandard_C>stdc17</LanguageStandard_C>
       <PrecompiledHeader>Create</PrecompiledHeader>
       <PrecompiledHeaderFile>pch_msc.hpp</PrecompiledHeaderFile>
@@ -168,6 +247,7 @@
       <MultiProcessorCompilation>true</MultiProcessorCompilation>
       <EnableParallelCodeGeneration>true</EnableParallelCodeGeneration>
       <ProgramDataBaseFileName>$(IntDir)vc$(PlatformToolsetVersion).pdb</ProgramDataBaseFileName>
+      <BrowseInformation>true</BrowseInformation>
     </ClCompile>
     <Link>
       <SubSystem>Console</SubSystem>
@@ -176,6 +256,9 @@
     <Lib>
       <AdditionalDependencies>$(ProjectDir)\..\monetdb\msvc\monetdbe.lib;</AdditionalDependencies>
     </Lib>
+    <Bscmake>
+      <PreserveSbr>true</PreserveSbr>
+    </Bscmake>
   </ItemDefinitionGroup>
   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
     <ClCompile>
@@ -183,9 +266,9 @@
       <FunctionLevelLinking>true</FunctionLevelLinking>
       <IntrinsicFunctions>true</IntrinsicFunctions>
       <SDLCheck>false</SDLCheck>
-      <PreprocessorDefinitions>_CRT_SECURE_NO_WARNINGS;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>QUERY_DECLSPEC=__declspec(dllexport);_CRT_SECURE_NO_WARNINGS;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <ConformanceMode>true</ConformanceMode>
-      <LanguageStandard>stdcpp17</LanguageStandard>
+      <LanguageStandard>stdcpplatest</LanguageStandard>
       <LanguageStandard_C>stdc17</LanguageStandard_C>
       <PrecompiledHeader>Create</PrecompiledHeader>
       <PrecompiledHeaderFile>pch_msc.hpp</PrecompiledHeaderFile>
@@ -218,6 +301,47 @@
       <AdditionalDependencies>$(ProjectDir)\..\monetdb\msvc\monetdbe.lib;</AdditionalDependencies>
     </Lib>
   </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='RelWDbgInf|x64'">
+    <ClCompile>
+      <WarningLevel>Level3</WarningLevel>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <IntrinsicFunctions>true</IntrinsicFunctions>
+      <SDLCheck>false</SDLCheck>
+      <PreprocessorDefinitions>QUERY_DECLSPEC=__declspec(dllexport);_CRT_SECURE_NO_WARNINGS;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <ConformanceMode>true</ConformanceMode>
+      <LanguageStandard>stdcpplatest</LanguageStandard>
+      <LanguageStandard_C>stdc17</LanguageStandard_C>
+      <PrecompiledHeader>Create</PrecompiledHeader>
+      <PrecompiledHeaderFile>pch_msc.hpp</PrecompiledHeaderFile>
+      <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
+      <Optimization>Full</Optimization>
+      <InlineFunctionExpansion>AnySuitable</InlineFunctionExpansion>
+      <FavorSizeOrSpeed>Speed</FavorSizeOrSpeed>
+      <OmitFramePointers>true</OmitFramePointers>
+      <EnableFiberSafeOptimizations>true</EnableFiberSafeOptimizations>
+      <EnableEnhancedInstructionSet>AdvancedVectorExtensions2</EnableEnhancedInstructionSet>
+      <MultiProcessorCompilation>true</MultiProcessorCompilation>
+      <StringPooling>true</StringPooling>
+      <ExceptionHandling>false</ExceptionHandling>
+      <BufferSecurityCheck>false</BufferSecurityCheck>
+      <ControlFlowGuard>false</ControlFlowGuard>
+      <EnableParallelCodeGeneration>true</EnableParallelCodeGeneration>
+      <FloatingPointExceptions>false</FloatingPointExceptions>
+      <AdditionalIncludeDirectories>$(ProjectDir)\..\monetdb\msvc</AdditionalIncludeDirectories>
+      <PrecompiledHeaderOutputFile>$(SolutionDir)..\msc-plugin\pch_msc.pch</PrecompiledHeaderOutputFile>
+      <ProgramDataBaseFileName>$(IntDir)vc$(PlatformToolsetVersion).pdb</ProgramDataBaseFileName>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <EnableCOMDATFolding>true</EnableCOMDATFolding>
+      <OptimizeReferences>true</OptimizeReferences>
+      <GenerateDebugInformation>false</GenerateDebugInformation>
+      <AssemblyDebug>false</AssemblyDebug>
+    </Link>
+    <Lib>
+      <AdditionalDependencies>$(ProjectDir)\..\monetdb\msvc\monetdbe.lib;</AdditionalDependencies>
+    </Lib>
+  </ItemDefinitionGroup>
   <ItemGroup>
     <ClInclude Include="..\csv.h" />
     <ClInclude Include="..\server\aggregations.h" />
@@ -228,6 +352,7 @@
     <ClInclude Include="..\server\priority_vector.hpp" />
     <ClInclude Include="..\server\table.h" />
     <ClInclude Include="..\server\types.h" />
+    <ClInclude Include="..\server\unordered_dense.h" />
     <ClInclude Include="..\server\utils.h" />
     <ClInclude Include="..\server\vector_type.hpp" />
     <ClInclude Include="..\server\winhelper.h" />

msc-plugin/msc-plugin.vcxproj

@@ -1,4 +1,4 @@
-<?xml version="1.0" encoding="utf-8"?>
+<?xml version="1.0" encoding="utf-8"?>
 <Project DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
   <ItemGroup Label="ProjectConfigurations">
     <ProjectConfiguration Include="Debug|Win32">
@@ -17,6 +17,14 @@
       <Configuration>Release</Configuration>
       <Platform>x64</Platform>
     </ProjectConfiguration>
+    <ProjectConfiguration Include="RelWDbgInf|Win32">
+      <Configuration>RelWDbgInf</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="RelWDbgInf|x64">
+      <Configuration>RelWDbgInf</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
   </ItemGroup>
   <PropertyGroup Label="Globals">
     <VCProjectVersion>16.0</VCProjectVersion>
@@ -42,6 +50,14 @@
     <CharacterSet>Unicode</CharacterSet>
     <EnableASAN>false</EnableASAN>
   </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='RelWDbgInf|Win32'" Label="Configuration">
+    <ConfigurationType>DynamicLibrary</ConfigurationType>
+    <UseDebugLibraries>false</UseDebugLibraries>
+    <PlatformToolset>v143</PlatformToolset>
+    <WholeProgramOptimization>true</WholeProgramOptimization>
+    <CharacterSet>Unicode</CharacterSet>
+    <EnableASAN>false</EnableASAN>
+  </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
     <ConfigurationType>DynamicLibrary</ConfigurationType>
     <UseDebugLibraries>true</UseDebugLibraries>
@@ -58,6 +74,14 @@
     <CharacterSet>Unicode</CharacterSet>
     <EnableASAN>false</EnableASAN>
   </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='RelWDbgInf|x64'" Label="Configuration">
+    <ConfigurationType>DynamicLibrary</ConfigurationType>
+    <UseDebugLibraries>false</UseDebugLibraries>
+    <PlatformToolset>v143</PlatformToolset>
+    <WholeProgramOptimization>true</WholeProgramOptimization>
+    <CharacterSet>Unicode</CharacterSet>
+    <EnableASAN>false</EnableASAN>
+  </PropertyGroup>
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
   <ImportGroup Label="ExtensionSettings">
   </ImportGroup>
@@ -69,12 +93,18 @@
   <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
     <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
   </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='RelWDbgInf|Win32'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
   <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
     <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
   </ImportGroup>
   <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
     <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
   </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='RelWDbgInf|x64'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
   <PropertyGroup Label="UserMacros" />
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
     <TargetExt />
@@ -84,6 +114,10 @@
     <TargetExt />
     <TargetName>dll.so</TargetName>
   </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='RelWDbgInf|Win32'">
+    <TargetExt />
+    <TargetName>dll.so</TargetName>
+  </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
     <TargetExt />
     <TargetName>dll.so</TargetName>
@@ -92,13 +126,17 @@
     <TargetExt />
     <TargetName>dll.so</TargetName>
   </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='RelWDbgInf|x64'">
+    <TargetExt />
+    <TargetName>dll.so</TargetName>
+  </PropertyGroup>
   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
     <ClCompile>
       <WarningLevel>Level3</WarningLevel>
       <SDLCheck>true</SDLCheck>
-      <PreprocessorDefinitions>_CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>QUERY_DECLSPEC=__declspec(dllimport);_CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <ConformanceMode>true</ConformanceMode>
-      <LanguageStandard>stdcpp17</LanguageStandard>
+      <LanguageStandard>stdcpplatest</LanguageStandard>
       <ControlFlowGuard>Guard</ControlFlowGuard>
       <EnableModules>false</EnableModules>
       <AdditionalIncludeDirectories>$(ProjectDir)\..\monetdb\msvc</AdditionalIncludeDirectories>
@@ -112,6 +150,7 @@
       <EnableParallelCodeGeneration>true</EnableParallelCodeGeneration>
       <ProgramDataBaseFileName>$(IntDir)vc$(PlatformToolsetVersion).pdb</ProgramDataBaseFileName>
       <ForcedIncludeFiles>pch_msc.hpp</ForcedIncludeFiles>
+      <Optimization>MaxSpeed</Optimization>
     </ClCompile>
     <Link>
       <SubSystem>Console</SubSystem>
@@ -127,9 +166,9 @@
       <FunctionLevelLinking>true</FunctionLevelLinking>
       <IntrinsicFunctions>true</IntrinsicFunctions>
       <SDLCheck>false</SDLCheck>
-      <PreprocessorDefinitions>_CRT_SECURE_NO_WARNINGS;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>QUERY_DECLSPEC=__declspec(dllimport);_CRT_SECURE_NO_WARNINGS;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <ConformanceMode>true</ConformanceMode>
-      <LanguageStandard>stdcpp17</LanguageStandard>
+      <LanguageStandard>stdcpplatest</LanguageStandard>
       <AdditionalIncludeDirectories>$(ProjectDir)\..\monetdb\msvc</AdditionalIncludeDirectories>
       <LanguageStandard_C>stdc17</LanguageStandard_C>
       <PrecompiledHeader>Use</PrecompiledHeader>
@@ -164,31 +203,85 @@
       <AssemblyDebug>false</AssemblyDebug>
     </Link>
   </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='RelWDbgInf|Win32'">
+    <ClCompile>
+      <WarningLevel>Level3</WarningLevel>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <IntrinsicFunctions>true</IntrinsicFunctions>
+      <SDLCheck>false</SDLCheck>
+      <PreprocessorDefinitions>QUERY_DECLSPEC=__declspec(dllimport);_CRT_SECURE_NO_WARNINGS;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <ConformanceMode>true</ConformanceMode>
+      <LanguageStandard>stdcpplatest</LanguageStandard>
+      <AdditionalIncludeDirectories>$(ProjectDir)\..\monetdb\msvc</AdditionalIncludeDirectories>
+      <LanguageStandard_C>stdc17</LanguageStandard_C>
+      <PrecompiledHeader>Use</PrecompiledHeader>
+      <PrecompiledHeaderFile>pch_msc.hpp</PrecompiledHeaderFile>
+      <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
+      <Optimization>Full</Optimization>
+      <InlineFunctionExpansion>AnySuitable</InlineFunctionExpansion>
+      <FavorSizeOrSpeed>Speed</FavorSizeOrSpeed>
+      <OmitFramePointers>true</OmitFramePointers>
+      <EnableFiberSafeOptimizations>true</EnableFiberSafeOptimizations>
+      <EnableEnhancedInstructionSet>AdvancedVectorExtensions2</EnableEnhancedInstructionSet>
+      <MultiProcessorCompilation>true</MultiProcessorCompilation>
+      <StringPooling>true</StringPooling>
+      <ExceptionHandling>false</ExceptionHandling>
+      <BufferSecurityCheck>false</BufferSecurityCheck>
+      <ControlFlowGuard>false</ControlFlowGuard>
+      <EnableParallelCodeGeneration>true</EnableParallelCodeGeneration>
+      <FloatingPointExceptions>false</FloatingPointExceptions>
+      <PrecompiledHeaderOutputFile>
+      </PrecompiledHeaderOutputFile>
+      <EnableModules>false</EnableModules>
+      <ProgramDataBaseFileName>$(IntDir)vc$(PlatformToolsetVersion).pdb</ProgramDataBaseFileName>
+      <ForcedIncludeFiles>pch_msc.hpp</ForcedIncludeFiles>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <EnableCOMDATFolding>true</EnableCOMDATFolding>
+      <OptimizeReferences>true</OptimizeReferences>
+      <GenerateDebugInformation>false</GenerateDebugInformation>
+      <OutputFile>$(ProjectDir)\..\dll.so</OutputFile>
+      <AdditionalDependencies>$(ProjectDir)\..\libaquery.lib;$(ProjectDir)\..\monetdb\msvc\monetdbe.lib;$(CoreLibraryDependencies);%(AdditionalDependencies)</AdditionalDependencies>
+      <AssemblyDebug>false</AssemblyDebug>
+    </Link>
+  </ItemDefinitionGroup>
   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
     <ClCompile>
       <WarningLevel>Level3</WarningLevel>
       <SDLCheck>true</SDLCheck>
-      <PreprocessorDefinitions>_ALLOW_RTCc_IN_STL;_CRT_SECURE_NO_WARNINGS;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>QUERY_DECLSPEC=__declspec(dllimport);_ALLOW_RTCc_IN_STL;_CRT_SECURE_NO_WARNINGS;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <ConformanceMode>true</ConformanceMode>
-      <LanguageStandard>stdcpp17</LanguageStandard>
-      <ControlFlowGuard>Guard</ControlFlowGuard>
+      <LanguageStandard>stdcpplatest</LanguageStandard>
+      <ControlFlowGuard>false</ControlFlowGuard>
       <EnableModules>false</EnableModules>
       <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
-      <InlineFunctionExpansion>Disabled</InlineFunctionExpansion>
+      <InlineFunctionExpansion>Default</InlineFunctionExpansion>
       <OmitFramePointers>false</OmitFramePointers>
-      <SmallerTypeCheck>true</SmallerTypeCheck>
+      <SmallerTypeCheck>false</SmallerTypeCheck>
       <SupportJustMyCode>false</SupportJustMyCode>
       <AdditionalIncludeDirectories>$(ProjectDir)\..\monetdb\msvc</AdditionalIncludeDirectories>
       <AdditionalOptions>/WL %(AdditionalOptions)</AdditionalOptions>
       <LanguageStandard_C>stdc17</LanguageStandard_C>
-      <PrecompiledHeader>Use</PrecompiledHeader>
+      <PrecompiledHeader>NotUsing</PrecompiledHeader>
       <PrecompiledHeaderFile>pch_msc.hpp</PrecompiledHeaderFile>
       <PrecompiledHeaderOutputFile>
       </PrecompiledHeaderOutputFile>
       <MultiProcessorCompilation>true</MultiProcessorCompilation>
       <EnableParallelCodeGeneration>true</EnableParallelCodeGeneration>
       <ProgramDataBaseFileName>$(IntDir)vc$(PlatformToolsetVersion).pdb</ProgramDataBaseFileName>
-      <ForcedIncludeFiles>pch_msc.hpp</ForcedIncludeFiles>
+      <ForcedIncludeFiles>
+      </ForcedIncludeFiles>
+      <BrowseInformation>true</BrowseInformation>
+      <Optimization>Disabled</Optimization>
+      <BasicRuntimeChecks>Default</BasicRuntimeChecks>
+      <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
+      <EnableEnhancedInstructionSet>AdvancedVectorExtensions2</EnableEnhancedInstructionSet>
+      <IntrinsicFunctions>false</IntrinsicFunctions>
+      <FavorSizeOrSpeed>Neither</FavorSizeOrSpeed>
+      <EnableFiberSafeOptimizations>false</EnableFiberSafeOptimizations>
+      <WholeProgramOptimization>false</WholeProgramOptimization>
+      <BufferSecurityCheck>true</BufferSecurityCheck>
     </ClCompile>
     <Link>
       <SubSystem>Console</SubSystem>
@@ -197,6 +290,9 @@
       <AssemblyDebug>true</AssemblyDebug>
       <AdditionalDependencies>$(ProjectDir)\..\libaquery.lib;$(ProjectDir)\..\monetdb\msvc\monetdbe.lib;$(CoreLibraryDependencies);%(AdditionalDependencies)</AdditionalDependencies>
     </Link>
+    <Bscmake>
+      <PreserveSbr>true</PreserveSbr>
+    </Bscmake>
   </ItemDefinitionGroup>
   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
     <ClCompile>
@@ -204,9 +300,9 @@
       <FunctionLevelLinking>true</FunctionLevelLinking>
       <IntrinsicFunctions>true</IntrinsicFunctions>
       <SDLCheck>false</SDLCheck>
-      <PreprocessorDefinitions>_CRT_SECURE_NO_WARNINGS;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>QUERY_DECLSPEC=__declspec(dllimport);_CRT_SECURE_NO_WARNINGS;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <ConformanceMode>true</ConformanceMode>
-      <LanguageStandard>stdcpp17</LanguageStandard>
+      <LanguageStandard>stdcpplatest</LanguageStandard>
       <AdditionalIncludeDirectories>$(ProjectDir)\..\monetdb\msvc</AdditionalIncludeDirectories>
       <LanguageStandard_C>stdc17</LanguageStandard_C>
       <PrecompiledHeader>Use</PrecompiledHeader>
@@ -241,6 +337,49 @@
       <AssemblyDebug>false</AssemblyDebug>
     </Link>
   </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='RelWDbgInf|x64'">
+    <ClCompile>
+      <WarningLevel>Level3</WarningLevel>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <IntrinsicFunctions>true</IntrinsicFunctions>
+      <SDLCheck>false</SDLCheck>
+      <PreprocessorDefinitions>QUERY_DECLSPEC=__declspec(dllimport);_CRT_SECURE_NO_WARNINGS;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <ConformanceMode>true</ConformanceMode>
+      <LanguageStandard>stdcpplatest</LanguageStandard>
+      <AdditionalIncludeDirectories>$(ProjectDir)\..\monetdb\msvc</AdditionalIncludeDirectories>
+      <LanguageStandard_C>stdc17</LanguageStandard_C>
+      <PrecompiledHeader>Use</PrecompiledHeader>
+      <PrecompiledHeaderFile>pch_msc.hpp</PrecompiledHeaderFile>
+      <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
+      <Optimization>Full</Optimization>
+      <InlineFunctionExpansion>AnySuitable</InlineFunctionExpansion>
+      <FavorSizeOrSpeed>Speed</FavorSizeOrSpeed>
+      <OmitFramePointers>true</OmitFramePointers>
+      <EnableFiberSafeOptimizations>true</EnableFiberSafeOptimizations>
+      <EnableEnhancedInstructionSet>AdvancedVectorExtensions2</EnableEnhancedInstructionSet>
+      <MultiProcessorCompilation>true</MultiProcessorCompilation>
+      <StringPooling>true</StringPooling>
+      <ExceptionHandling>false</ExceptionHandling>
+      <BufferSecurityCheck>false</BufferSecurityCheck>
+      <ControlFlowGuard>false</ControlFlowGuard>
+      <EnableParallelCodeGeneration>true</EnableParallelCodeGeneration>
+      <FloatingPointExceptions>false</FloatingPointExceptions>
+      <PrecompiledHeaderOutputFile>
+      </PrecompiledHeaderOutputFile>
+      <EnableModules>false</EnableModules>
+      <ProgramDataBaseFileName>$(IntDir)vc$(PlatformToolsetVersion).pdb</ProgramDataBaseFileName>
+      <ForcedIncludeFiles>pch_msc.hpp</ForcedIncludeFiles>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <EnableCOMDATFolding>true</EnableCOMDATFolding>
+      <OptimizeReferences>true</OptimizeReferences>
+      <GenerateDebugInformation>false</GenerateDebugInformation>
+      <OutputFile>$(ProjectDir)\..\dll.so</OutputFile>
+      <AdditionalDependencies>$(ProjectDir)\..\libaquery.lib;$(ProjectDir)\..\monetdb\msvc\monetdbe.lib;$(CoreLibraryDependencies);%(AdditionalDependencies)</AdditionalDependencies>
+      <AssemblyDebug>false</AssemblyDebug>
+    </Link>
+  </ItemDefinitionGroup>
   <ItemGroup>
     <ClCompile Include="..\out.cpp" />
   </ItemGroup>

msc-plugin/sdk_example.vcxproj

@@ -17,6 +17,14 @@
       <Configuration>Release</Configuration>
       <Platform>x64</Platform>
     </ProjectConfiguration>
+    <ProjectConfiguration Include="RelWDbgInf|Win32">
+      <Configuration>RelWDbgInf</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="RelWDbgInf|x64">
+      <Configuration>RelWDbgInf</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
   </ItemGroup>
   <PropertyGroup Label="Globals">
     <VCProjectVersion>16.0</VCProjectVersion>
@@ -39,6 +47,13 @@
     <WholeProgramOptimization>true</WholeProgramOptimization>
     <CharacterSet>Unicode</CharacterSet>
   </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='RelWDbgInf|Win32'" Label="Configuration">
+    <ConfigurationType>DynamicLibrary</ConfigurationType>
+    <UseDebugLibraries>false</UseDebugLibraries>
+    <PlatformToolset>v143</PlatformToolset>
+    <WholeProgramOptimization>true</WholeProgramOptimization>
+    <CharacterSet>Unicode</CharacterSet>
+  </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
     <ConfigurationType>DynamicLibrary</ConfigurationType>
     <UseDebugLibraries>true</UseDebugLibraries>
@@ -52,6 +67,13 @@
     <WholeProgramOptimization>true</WholeProgramOptimization>
     <CharacterSet>Unicode</CharacterSet>
   </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='RelWDbgInf|x64'" Label="Configuration">
+    <ConfigurationType>DynamicLibrary</ConfigurationType>
+    <UseDebugLibraries>false</UseDebugLibraries>
+    <PlatformToolset>v143</PlatformToolset>
+    <WholeProgramOptimization>true</WholeProgramOptimization>
+    <CharacterSet>Unicode</CharacterSet>
+  </PropertyGroup>
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
   <ImportGroup Label="ExtensionSettings">
   </ImportGroup>
@@ -63,12 +85,18 @@
   <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
     <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
   </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='RelWDbgInf|Win32'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
   <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
     <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
   </ImportGroup>
   <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
     <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
   </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='RelWDbgInf|x64'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
   <PropertyGroup Label="UserMacros" />
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
     <OutDir>$(SolutionDir)..\</OutDir>
@@ -80,6 +108,11 @@
     <TargetName>test</TargetName>
     <TargetExt>.so</TargetExt>
   </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='RelWDbgInf|Win32'">
+    <OutDir>$(SolutionDir)..\</OutDir>
+    <TargetName>test</TargetName>
+    <TargetExt>.so</TargetExt>
+  </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
     <OutDir>$(SolutionDir)..\</OutDir>
     <TargetName>test</TargetName>
@@ -90,6 +123,11 @@
     <TargetName>test</TargetName>
     <TargetExt>.so</TargetExt>
   </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='RelWDbgInf|x64'">
+    <OutDir>$(SolutionDir)..\</OutDir>
+    <TargetName>test</TargetName>
+    <TargetExt>.so</TargetExt>
+  </PropertyGroup>
   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
     <ClCompile>
       <WarningLevel>Level3</WarningLevel>
@@ -124,6 +162,24 @@
       <GenerateDebugInformation>true</GenerateDebugInformation>
     </Link>
   </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='RelWDbgInf|Win32'">
+    <ClCompile>
+      <WarningLevel>Level3</WarningLevel>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <IntrinsicFunctions>true</IntrinsicFunctions>
+      <SDLCheck>true</SDLCheck>
+      <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <ConformanceMode>true</ConformanceMode>
+      <LanguageStandard>stdcpplatest</LanguageStandard>
+      <LanguageStandard_C>stdc17</LanguageStandard_C>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <EnableCOMDATFolding>true</EnableCOMDATFolding>
+      <OptimizeReferences>true</OptimizeReferences>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+    </Link>
+  </ItemDefinitionGroup>
   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
     <ClCompile>
       <WarningLevel>Level3</WarningLevel>
@@ -134,11 +190,15 @@
       <LanguageStandard_C>stdc17</LanguageStandard_C>
       <MultiProcessorCompilation>true</MultiProcessorCompilation>
       <EnableParallelCodeGeneration>true</EnableParallelCodeGeneration>
+      <BrowseInformation>true</BrowseInformation>
     </ClCompile>
     <Link>
       <SubSystem>Console</SubSystem>
       <GenerateDebugInformation>true</GenerateDebugInformation>
     </Link>
+    <Bscmake>
+      <PreserveSbr>true</PreserveSbr>
+    </Bscmake>
   </ItemDefinitionGroup>
   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
     <ClCompile>
@@ -158,6 +218,24 @@
       <GenerateDebugInformation>true</GenerateDebugInformation>
     </Link>
   </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='RelWDbgInf|x64'">
+    <ClCompile>
+      <WarningLevel>Level3</WarningLevel>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <IntrinsicFunctions>true</IntrinsicFunctions>
+      <SDLCheck>true</SDLCheck>
+      <PreprocessorDefinitions>NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <ConformanceMode>true</ConformanceMode>
+      <LanguageStandard>stdcpplatest</LanguageStandard>
+      <LanguageStandard_C>stdc17</LanguageStandard_C>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <EnableCOMDATFolding>true</EnableCOMDATFolding>
+      <OptimizeReferences>true</OptimizeReferences>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+    </Link>
+  </ItemDefinitionGroup>
   <ItemGroup>
     <ClInclude Include="..\sdk\aquery.h" />
   </ItemGroup>

msc-plugin/server.sln

@@ -9,6 +9,9 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "server", "server.vcxproj",
 	EndProjectSection
 EndProject
 Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "msc-plugin", "msc-plugin.vcxproj", "{8081FDAA-4D13-4B7A-ADB2-8224AF7F1C81}"
+	ProjectSection(ProjectDependencies) = postProject
+		{B52AACF7-16A6-4FCA-90AD-867D367BDA4F} = {B52AACF7-16A6-4FCA-90AD-867D367BDA4F}
+	EndProjectSection
 EndProject
 Project("{888888A0-9F3D-457C-B088-3A5042F75D52}") = "msvs-py", "..\msvs-py\msvs-py.pyproj", "{CCC243F5-663E-45B7-A6DE-B2468C58B3A7}"
 EndProject
@@ -21,6 +24,10 @@ EndProject
 Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "testmain", "testmain.vcxproj", "{73BD0DDA-7E66-4318-B46B-B8207AFA6355}"
 	ProjectSection(ProjectDependencies) = postProject
 		{8081FDAA-4D13-4B7A-ADB2-8224AF7F1C81} = {8081FDAA-4D13-4B7A-ADB2-8224AF7F1C81}
+	EndProjectSection
+EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "datagen", "datagen\datagen.vcxproj", "{5AA4E63C-CFF8-467C-8872-84206CEDE7A9}"
+	ProjectSection(ProjectDependencies) = postProject
 		{B52AACF7-16A6-4FCA-90AD-867D367BDA4F} = {B52AACF7-16A6-4FCA-90AD-867D367BDA4F}
 	EndProjectSection
 EndProject
@@ -30,6 +37,8 @@ Global
 		Debug|x86 = Debug|x86
 		Release|x64 = Release|x64
 		Release|x86 = Release|x86
+		RelWDbgInf|x64 = RelWDbgInf|x64
+		RelWDbgInf|x86 = RelWDbgInf|x86
 		shared|x64 = shared|x64
 		shared|x86 = shared|x86
 		sharedlib|x64 = sharedlib|x64
@@ -44,6 +53,10 @@ Global
 		{031352C2-AFBB-45AA-9518-DBC1F9EF2AF3}.Release|x64.Build.0 = Release|x64
 		{031352C2-AFBB-45AA-9518-DBC1F9EF2AF3}.Release|x86.ActiveCfg = Release|Win32
 		{031352C2-AFBB-45AA-9518-DBC1F9EF2AF3}.Release|x86.Build.0 = Release|Win32
+		{031352C2-AFBB-45AA-9518-DBC1F9EF2AF3}.RelWDbgInf|x64.ActiveCfg = RelWDbgInf|x64
+		{031352C2-AFBB-45AA-9518-DBC1F9EF2AF3}.RelWDbgInf|x64.Build.0 = RelWDbgInf|x64
+		{031352C2-AFBB-45AA-9518-DBC1F9EF2AF3}.RelWDbgInf|x86.ActiveCfg = RelWDbgInf|Win32
+		{031352C2-AFBB-45AA-9518-DBC1F9EF2AF3}.RelWDbgInf|x86.Build.0 = RelWDbgInf|Win32
 		{031352C2-AFBB-45AA-9518-DBC1F9EF2AF3}.shared|x64.ActiveCfg = sharedlib|x64
 		{031352C2-AFBB-45AA-9518-DBC1F9EF2AF3}.shared|x86.ActiveCfg = sharedlib|Win32
 		{031352C2-AFBB-45AA-9518-DBC1F9EF2AF3}.sharedlib|x64.ActiveCfg = sharedlib|x64
@@ -58,6 +71,10 @@ Global
 		{8081FDAA-4D13-4B7A-ADB2-8224AF7F1C81}.Release|x64.Build.0 = Release|x64
 		{8081FDAA-4D13-4B7A-ADB2-8224AF7F1C81}.Release|x86.ActiveCfg = Release|Win32
 		{8081FDAA-4D13-4B7A-ADB2-8224AF7F1C81}.Release|x86.Build.0 = Release|Win32
+		{8081FDAA-4D13-4B7A-ADB2-8224AF7F1C81}.RelWDbgInf|x64.ActiveCfg = RelWDbgInf|x64
+		{8081FDAA-4D13-4B7A-ADB2-8224AF7F1C81}.RelWDbgInf|x64.Build.0 = RelWDbgInf|x64
+		{8081FDAA-4D13-4B7A-ADB2-8224AF7F1C81}.RelWDbgInf|x86.ActiveCfg = RelWDbgInf|Win32
+		{8081FDAA-4D13-4B7A-ADB2-8224AF7F1C81}.RelWDbgInf|x86.Build.0 = RelWDbgInf|Win32
 		{8081FDAA-4D13-4B7A-ADB2-8224AF7F1C81}.shared|x64.ActiveCfg = Release|x64
 		{8081FDAA-4D13-4B7A-ADB2-8224AF7F1C81}.shared|x64.Build.0 = Release|x64
 		{8081FDAA-4D13-4B7A-ADB2-8224AF7F1C81}.shared|x86.ActiveCfg = Release|Win32
@@ -70,6 +87,8 @@ Global
 		{CCC243F5-663E-45B7-A6DE-B2468C58B3A7}.Debug|x86.ActiveCfg = Debug|Any CPU
 		{CCC243F5-663E-45B7-A6DE-B2468C58B3A7}.Release|x64.ActiveCfg = Release|Any CPU
 		{CCC243F5-663E-45B7-A6DE-B2468C58B3A7}.Release|x86.ActiveCfg = Release|Any CPU
+		{CCC243F5-663E-45B7-A6DE-B2468C58B3A7}.RelWDbgInf|x64.ActiveCfg = RelWDbgInf|Any CPU
+		{CCC243F5-663E-45B7-A6DE-B2468C58B3A7}.RelWDbgInf|x86.ActiveCfg = RelWDbgInf|Any CPU
 		{CCC243F5-663E-45B7-A6DE-B2468C58B3A7}.shared|x64.ActiveCfg = Release|Any CPU
 		{CCC243F5-663E-45B7-A6DE-B2468C58B3A7}.shared|x86.ActiveCfg = Release|Any CPU
 		{CCC243F5-663E-45B7-A6DE-B2468C58B3A7}.sharedlib|x64.ActiveCfg = Release|Any CPU
@@ -82,6 +101,10 @@ Global
 		{B52AACF7-16A6-4FCA-90AD-867D367BDA4F}.Release|x64.Build.0 = Release|x64
 		{B52AACF7-16A6-4FCA-90AD-867D367BDA4F}.Release|x86.ActiveCfg = Release|Win32
 		{B52AACF7-16A6-4FCA-90AD-867D367BDA4F}.Release|x86.Build.0 = Release|Win32
+		{B52AACF7-16A6-4FCA-90AD-867D367BDA4F}.RelWDbgInf|x64.ActiveCfg = RelWDbgInf|x64
+		{B52AACF7-16A6-4FCA-90AD-867D367BDA4F}.RelWDbgInf|x64.Build.0 = RelWDbgInf|x64
+		{B52AACF7-16A6-4FCA-90AD-867D367BDA4F}.RelWDbgInf|x86.ActiveCfg = RelWDbgInf|Win32
+		{B52AACF7-16A6-4FCA-90AD-867D367BDA4F}.RelWDbgInf|x86.Build.0 = RelWDbgInf|Win32
 		{B52AACF7-16A6-4FCA-90AD-867D367BDA4F}.shared|x64.ActiveCfg = Debug|x64
 		{B52AACF7-16A6-4FCA-90AD-867D367BDA4F}.shared|x64.Build.0 = Debug|x64
 		{B52AACF7-16A6-4FCA-90AD-867D367BDA4F}.shared|x86.ActiveCfg = Debug|Win32
@@ -98,6 +121,10 @@ Global
 		{C8E25628-0B46-4CBE-90DF-5228F79A5A64}.Release|x64.Build.0 = Release|x64
 		{C8E25628-0B46-4CBE-90DF-5228F79A5A64}.Release|x86.ActiveCfg = Release|Win32
 		{C8E25628-0B46-4CBE-90DF-5228F79A5A64}.Release|x86.Build.0 = Release|Win32
+		{C8E25628-0B46-4CBE-90DF-5228F79A5A64}.RelWDbgInf|x64.ActiveCfg = RelWDbgInf|x64
+		{C8E25628-0B46-4CBE-90DF-5228F79A5A64}.RelWDbgInf|x64.Build.0 = RelWDbgInf|x64
+		{C8E25628-0B46-4CBE-90DF-5228F79A5A64}.RelWDbgInf|x86.ActiveCfg = RelWDbgInf|Win32
+		{C8E25628-0B46-4CBE-90DF-5228F79A5A64}.RelWDbgInf|x86.Build.0 = RelWDbgInf|Win32
 		{C8E25628-0B46-4CBE-90DF-5228F79A5A64}.shared|x64.ActiveCfg = sharedlib|x64
 		{C8E25628-0B46-4CBE-90DF-5228F79A5A64}.shared|x64.Build.0 = sharedlib|x64
 		{C8E25628-0B46-4CBE-90DF-5228F79A5A64}.shared|x86.ActiveCfg = sharedlib|Win32
@@ -114,6 +141,10 @@ Global
 		{F954797B-C148-4CBF-9FB4-A9A450EFEC38}.Release|x64.Build.0 = Release|x64
 		{F954797B-C148-4CBF-9FB4-A9A450EFEC38}.Release|x86.ActiveCfg = Release|Win32
 		{F954797B-C148-4CBF-9FB4-A9A450EFEC38}.Release|x86.Build.0 = Release|Win32
+		{F954797B-C148-4CBF-9FB4-A9A450EFEC38}.RelWDbgInf|x64.ActiveCfg = RelWDbgInf|x64
+		{F954797B-C148-4CBF-9FB4-A9A450EFEC38}.RelWDbgInf|x64.Build.0 = RelWDbgInf|x64
+		{F954797B-C148-4CBF-9FB4-A9A450EFEC38}.RelWDbgInf|x86.ActiveCfg = RelWDbgInf|Win32
+		{F954797B-C148-4CBF-9FB4-A9A450EFEC38}.RelWDbgInf|x86.Build.0 = RelWDbgInf|Win32
 		{F954797B-C148-4CBF-9FB4-A9A450EFEC38}.shared|x64.ActiveCfg = Debug|x64
 		{F954797B-C148-4CBF-9FB4-A9A450EFEC38}.shared|x64.Build.0 = Debug|x64
 		{F954797B-C148-4CBF-9FB4-A9A450EFEC38}.shared|x86.ActiveCfg = Debug|Win32
@@ -130,6 +161,10 @@ Global
 		{73BD0DDA-7E66-4318-B46B-B8207AFA6355}.Release|x64.Build.0 = Release|x64
 		{73BD0DDA-7E66-4318-B46B-B8207AFA6355}.Release|x86.ActiveCfg = Release|Win32
 		{73BD0DDA-7E66-4318-B46B-B8207AFA6355}.Release|x86.Build.0 = Release|Win32
+		{73BD0DDA-7E66-4318-B46B-B8207AFA6355}.RelWDbgInf|x64.ActiveCfg = RelWDbgInf|x64
+		{73BD0DDA-7E66-4318-B46B-B8207AFA6355}.RelWDbgInf|x64.Build.0 = RelWDbgInf|x64
+		{73BD0DDA-7E66-4318-B46B-B8207AFA6355}.RelWDbgInf|x86.ActiveCfg = RelWDbgInf|Win32
+		{73BD0DDA-7E66-4318-B46B-B8207AFA6355}.RelWDbgInf|x86.Build.0 = RelWDbgInf|Win32
 		{73BD0DDA-7E66-4318-B46B-B8207AFA6355}.shared|x64.ActiveCfg = sharedlib|x64
 		{73BD0DDA-7E66-4318-B46B-B8207AFA6355}.shared|x64.Build.0 = sharedlib|x64
 		{73BD0DDA-7E66-4318-B46B-B8207AFA6355}.shared|x86.ActiveCfg = sharedlib|Win32
@@ -138,6 +173,26 @@ Global
 		{73BD0DDA-7E66-4318-B46B-B8207AFA6355}.sharedlib|x64.Build.0 = sharedlib|x64
 		{73BD0DDA-7E66-4318-B46B-B8207AFA6355}.sharedlib|x86.ActiveCfg = sharedlib|Win32
 		{73BD0DDA-7E66-4318-B46B-B8207AFA6355}.sharedlib|x86.Build.0 = sharedlib|Win32
+		{5AA4E63C-CFF8-467C-8872-84206CEDE7A9}.Debug|x64.ActiveCfg = Debug|x64
+		{5AA4E63C-CFF8-467C-8872-84206CEDE7A9}.Debug|x64.Build.0 = Debug|x64
+		{5AA4E63C-CFF8-467C-8872-84206CEDE7A9}.Debug|x86.ActiveCfg = Debug|Win32
+		{5AA4E63C-CFF8-467C-8872-84206CEDE7A9}.Debug|x86.Build.0 = Debug|Win32
+		{5AA4E63C-CFF8-467C-8872-84206CEDE7A9}.Release|x64.ActiveCfg = Release|x64
+		{5AA4E63C-CFF8-467C-8872-84206CEDE7A9}.Release|x64.Build.0 = Release|x64
+		{5AA4E63C-CFF8-467C-8872-84206CEDE7A9}.Release|x86.ActiveCfg = Release|Win32
+		{5AA4E63C-CFF8-467C-8872-84206CEDE7A9}.Release|x86.Build.0 = Release|Win32
+		{5AA4E63C-CFF8-467C-8872-84206CEDE7A9}.RelWDbgInf|x64.ActiveCfg = Release|x64
+		{5AA4E63C-CFF8-467C-8872-84206CEDE7A9}.RelWDbgInf|x64.Build.0 = Release|x64
+		{5AA4E63C-CFF8-467C-8872-84206CEDE7A9}.RelWDbgInf|x86.ActiveCfg = Release|Win32
+		{5AA4E63C-CFF8-467C-8872-84206CEDE7A9}.RelWDbgInf|x86.Build.0 = Release|Win32
+		{5AA4E63C-CFF8-467C-8872-84206CEDE7A9}.shared|x64.ActiveCfg = Debug|x64
+		{5AA4E63C-CFF8-467C-8872-84206CEDE7A9}.shared|x64.Build.0 = Debug|x64
+		{5AA4E63C-CFF8-467C-8872-84206CEDE7A9}.shared|x86.ActiveCfg = Debug|Win32
+		{5AA4E63C-CFF8-467C-8872-84206CEDE7A9}.shared|x86.Build.0 = Debug|Win32
+		{5AA4E63C-CFF8-467C-8872-84206CEDE7A9}.sharedlib|x64.ActiveCfg = Debug|x64
+		{5AA4E63C-CFF8-467C-8872-84206CEDE7A9}.sharedlib|x64.Build.0 = Debug|x64
+		{5AA4E63C-CFF8-467C-8872-84206CEDE7A9}.sharedlib|x86.ActiveCfg = Debug|Win32
+		{5AA4E63C-CFF8-467C-8872-84206CEDE7A9}.sharedlib|x86.Build.0 = Debug|Win32
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE

msc-plugin/server.vcxproj

@@ -17,6 +17,14 @@
       <Configuration>Release</Configuration>
       <Platform>x64</Platform>
     </ProjectConfiguration>
+    <ProjectConfiguration Include="RelWDbgInf|Win32">
+      <Configuration>RelWDbgInf</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="RelWDbgInf|x64">
+      <Configuration>RelWDbgInf</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
     <ProjectConfiguration Include="sharedlib|Win32">
       <Configuration>sharedlib</Configuration>
       <Platform>Win32</Platform>
@@ -52,6 +60,14 @@
     <CharacterSet>Unicode</CharacterSet>
     <EnableASAN>false</EnableASAN>
   </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='RelWDbgInf|Win32'" Label="Configuration">
+    <ConfigurationType>DynamicLibrary</ConfigurationType>
+    <UseDebugLibraries>false</UseDebugLibraries>
+    <PlatformToolset>v143</PlatformToolset>
+    <WholeProgramOptimization>true</WholeProgramOptimization>
+    <CharacterSet>Unicode</CharacterSet>
+    <EnableASAN>false</EnableASAN>
+  </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='sharedlib|Win32'" Label="Configuration">
     <ConfigurationType>DynamicLibrary</ConfigurationType>
     <UseDebugLibraries>false</UseDebugLibraries>
@@ -75,6 +91,14 @@
     <CharacterSet>Unicode</CharacterSet>
     <EnableASAN>false</EnableASAN>
   </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='RelWDbgInf|x64'" Label="Configuration">
+    <ConfigurationType>DynamicLibrary</ConfigurationType>
+    <UseDebugLibraries>false</UseDebugLibraries>
+    <PlatformToolset>v143</PlatformToolset>
+    <WholeProgramOptimization>true</WholeProgramOptimization>
+    <CharacterSet>Unicode</CharacterSet>
+    <EnableASAN>false</EnableASAN>
+  </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='sharedlib|x64'" Label="Configuration">
     <ConfigurationType>DynamicLibrary</ConfigurationType>
     <UseDebugLibraries>false</UseDebugLibraries>
@@ -94,6 +118,9 @@
   <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
     <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
   </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='RelWDbgInf|Win32'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
   <ImportGroup Condition="'$(Configuration)|$(Platform)'=='sharedlib|Win32'" Label="PropertySheets">
     <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
   </ImportGroup>
@@ -103,6 +130,9 @@
   <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
     <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
   </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='RelWDbgInf|x64'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
   <ImportGroup Condition="'$(Configuration)|$(Platform)'=='sharedlib|x64'" Label="PropertySheets">
     <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
   </ImportGroup>
@@ -117,6 +147,11 @@
     <TargetExt>.so</TargetExt>
     <OutDir>$(SolutionDir)..\</OutDir>
   </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='RelWDbgInf|Win32'">
+    <LinkIncremental>false</LinkIncremental>
+    <TargetExt>.so</TargetExt>
+    <OutDir>$(SolutionDir)..\</OutDir>
+  </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='sharedlib|Win32'">
     <LinkIncremental>false</LinkIncremental>
     <TargetExt>.so</TargetExt>
@@ -132,6 +167,11 @@
     <TargetExt>.so</TargetExt>
     <OutDir>$(SolutionDir)..\</OutDir>
   </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='RelWDbgInf|x64'">
+    <LinkIncremental>false</LinkIncremental>
+    <TargetExt>.so</TargetExt>
+    <OutDir>$(SolutionDir)..\</OutDir>
+  </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='sharedlib|x64'">
     <LinkIncremental>false</LinkIncremental>
     <TargetExt>.so</TargetExt>
@@ -145,7 +185,7 @@
       <ConformanceMode>true</ConformanceMode>
       <MultiProcessorCompilation>true</MultiProcessorCompilation>
       <EnableParallelCodeGeneration>true</EnableParallelCodeGeneration>
-      <LanguageStandard>stdcpp17</LanguageStandard>
+      <LanguageStandard>stdcpplatest</LanguageStandard>
       <LanguageStandard_C>stdc17</LanguageStandard_C>
       <AdditionalIncludeDirectories>$(ProjectDir)\..\monetdb\msvc</AdditionalIncludeDirectories>
     </ClCompile>
@@ -166,7 +206,32 @@
       <ConformanceMode>true</ConformanceMode>
       <MultiProcessorCompilation>true</MultiProcessorCompilation>
       <EnableParallelCodeGeneration>true</EnableParallelCodeGeneration>
-      <LanguageStandard>stdcpp17</LanguageStandard>
+      <LanguageStandard>stdcpplatest</LanguageStandard>
+      <LanguageStandard_C>stdc17</LanguageStandard_C>
+      <AdditionalIncludeDirectories>$(ProjectDir)\..\monetdb\msvc</AdditionalIncludeDirectories>
+      <DebugInformationFormat>None</DebugInformationFormat>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <EnableCOMDATFolding>true</EnableCOMDATFolding>
+      <OptimizeReferences>true</OptimizeReferences>
+      <GenerateDebugInformation>false</GenerateDebugInformation>
+      <AdditionalDependencies>..\libaquery.lib;$(ProjectDir)\..\monetdb\msvc\monetdbe.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <DataExecutionPrevention>false</DataExecutionPrevention>
+      <AdditionalOptions>/WHOLEARCHIVE:libaquery.lib %(AdditionalOptions)</AdditionalOptions>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='RelWDbgInf|Win32'">
+    <ClCompile>
+      <WarningLevel>Level3</WarningLevel>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <IntrinsicFunctions>true</IntrinsicFunctions>
+      <SDLCheck>true</SDLCheck>
+      <PreprocessorDefinitions>_CRT_SECURE_NO_WARNINGS;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <ConformanceMode>true</ConformanceMode>
+      <MultiProcessorCompilation>true</MultiProcessorCompilation>
+      <EnableParallelCodeGeneration>true</EnableParallelCodeGeneration>
+      <LanguageStandard>stdcpplatest</LanguageStandard>
       <LanguageStandard_C>stdc17</LanguageStandard_C>
       <AdditionalIncludeDirectories>$(ProjectDir)\..\monetdb\msvc</AdditionalIncludeDirectories>
       <DebugInformationFormat>None</DebugInformationFormat>
@@ -215,9 +280,10 @@
       <ConformanceMode>true</ConformanceMode>
       <MultiProcessorCompilation>true</MultiProcessorCompilation>
       <EnableParallelCodeGeneration>true</EnableParallelCodeGeneration>
-      <LanguageStandard>stdcpp17</LanguageStandard>
+      <LanguageStandard>stdcpplatest</LanguageStandard>
       <LanguageStandard_C>stdc17</LanguageStandard_C>
       <AdditionalIncludeDirectories>$(ProjectDir)\..\monetdb\msvc</AdditionalIncludeDirectories>
+      <BrowseInformation>true</BrowseInformation>
     </ClCompile>
     <Link>
       <SubSystem>Console</SubSystem>
@@ -225,6 +291,9 @@
       <AdditionalDependencies>..\libaquery.lib;$(ProjectDir)\..\monetdb\msvc\monetdbe.lib;%(AdditionalDependencies)</AdditionalDependencies>
       <AdditionalOptions>/WHOLEARCHIVE:libaquery.lib %(AdditionalOptions)</AdditionalOptions>
     </Link>
+    <Bscmake>
+      <PreserveSbr>true</PreserveSbr>
+    </Bscmake>
   </ItemDefinitionGroup>
   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
     <ClCompile>
@@ -236,7 +305,32 @@
       <ConformanceMode>true</ConformanceMode>
       <MultiProcessorCompilation>true</MultiProcessorCompilation>
       <EnableParallelCodeGeneration>true</EnableParallelCodeGeneration>
-      <LanguageStandard>stdcpp17</LanguageStandard>
+      <LanguageStandard>stdcpplatest</LanguageStandard>
+      <LanguageStandard_C>stdc17</LanguageStandard_C>
+      <AdditionalIncludeDirectories>$(ProjectDir)\..\monetdb\msvc</AdditionalIncludeDirectories>
+      <DebugInformationFormat>None</DebugInformationFormat>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <EnableCOMDATFolding>true</EnableCOMDATFolding>
+      <OptimizeReferences>true</OptimizeReferences>
+      <GenerateDebugInformation>false</GenerateDebugInformation>
+      <AdditionalDependencies>..\libaquery.lib;$(ProjectDir)\..\monetdb\msvc\monetdbe.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <DataExecutionPrevention>false</DataExecutionPrevention>
+      <AdditionalOptions>/WHOLEARCHIVE:libaquery.lib %(AdditionalOptions)</AdditionalOptions>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='RelWDbgInf|x64'">
+    <ClCompile>
+      <WarningLevel>Level3</WarningLevel>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <IntrinsicFunctions>true</IntrinsicFunctions>
+      <SDLCheck>true</SDLCheck>
+      <PreprocessorDefinitions>_CRT_SECURE_NO_WARNINGS;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <ConformanceMode>true</ConformanceMode>
+      <MultiProcessorCompilation>true</MultiProcessorCompilation>
+      <EnableParallelCodeGeneration>true</EnableParallelCodeGeneration>
+      <LanguageStandard>stdcpplatest</LanguageStandard>
       <LanguageStandard_C>stdc17</LanguageStandard_C>
       <AdditionalIncludeDirectories>$(ProjectDir)\..\monetdb\msvc</AdditionalIncludeDirectories>
       <DebugInformationFormat>None</DebugInformationFormat>

msc-plugin/testmain.vcxproj

@@ -17,6 +17,14 @@
       <Configuration>Release</Configuration>
       <Platform>x64</Platform>
     </ProjectConfiguration>
+    <ProjectConfiguration Include="RelWDbgInf|Win32">
+      <Configuration>RelWDbgInf</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="RelWDbgInf|x64">
+      <Configuration>RelWDbgInf</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
     <ProjectConfiguration Include="sharedlib|Win32">
       <Configuration>sharedlib</Configuration>
       <Platform>Win32</Platform>
@@ -27,7 +35,7 @@
     </ProjectConfiguration>
   </ItemGroup>
   <ItemGroup>
-    <ClCompile Include="dummy.cpp" />
+    <ClCompile Include="..\server\tests\testmain.cpp" />
   </ItemGroup>
   <PropertyGroup Label="Globals">
     <VCProjectVersion>16.0</VCProjectVersion>
@@ -52,6 +60,14 @@
     <CharacterSet>Unicode</CharacterSet>
     <EnableASAN>false</EnableASAN>
   </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='RelWDbgInf|Win32'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <UseDebugLibraries>false</UseDebugLibraries>
+    <PlatformToolset>v143</PlatformToolset>
+    <WholeProgramOptimization>true</WholeProgramOptimization>
+    <CharacterSet>Unicode</CharacterSet>
+    <EnableASAN>false</EnableASAN>
+  </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='sharedlib|Win32'" Label="Configuration">
     <ConfigurationType>Application</ConfigurationType>
     <UseDebugLibraries>false</UseDebugLibraries>
@@ -75,6 +91,14 @@
     <CharacterSet>Unicode</CharacterSet>
     <EnableASAN>false</EnableASAN>
   </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='RelWDbgInf|x64'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <UseDebugLibraries>false</UseDebugLibraries>
+    <PlatformToolset>v143</PlatformToolset>
+    <WholeProgramOptimization>true</WholeProgramOptimization>
+    <CharacterSet>Unicode</CharacterSet>
+    <EnableASAN>false</EnableASAN>
+  </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='sharedlib|x64'" Label="Configuration">
     <ConfigurationType>Application</ConfigurationType>
     <UseDebugLibraries>false</UseDebugLibraries>
@@ -94,6 +118,9 @@
   <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
     <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
   </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='RelWDbgInf|Win32'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
   <ImportGroup Condition="'$(Configuration)|$(Platform)'=='sharedlib|Win32'" Label="PropertySheets">
     <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
   </ImportGroup>
@@ -103,6 +130,9 @@
   <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
     <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
   </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='RelWDbgInf|x64'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
   <ImportGroup Condition="'$(Configuration)|$(Platform)'=='sharedlib|x64'" Label="PropertySheets">
     <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
   </ImportGroup>
@@ -117,6 +147,11 @@
     <TargetExt>.exe</TargetExt>
     <OutDir>$(SolutionDir)..\</OutDir>
   </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='RelWDbgInf|Win32'">
+    <LinkIncremental>false</LinkIncremental>
+    <TargetExt>.exe</TargetExt>
+    <OutDir>$(SolutionDir)..\</OutDir>
+  </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='sharedlib|Win32'">
     <LinkIncremental>false</LinkIncremental>
     <TargetExt>.exe</TargetExt>
@@ -132,6 +167,11 @@
     <TargetExt>.exe</TargetExt>
     <OutDir>$(SolutionDir)..\</OutDir>
   </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='RelWDbgInf|x64'">
+    <LinkIncremental>false</LinkIncremental>
+    <TargetExt>.exe</TargetExt>
+    <OutDir>$(SolutionDir)..\</OutDir>
+  </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='sharedlib|x64'">
     <LinkIncremental>false</LinkIncremental>
     <TargetExt>.exe</TargetExt>
@@ -145,7 +185,7 @@
       <ConformanceMode>true</ConformanceMode>
       <MultiProcessorCompilation>true</MultiProcessorCompilation>
       <EnableParallelCodeGeneration>true</EnableParallelCodeGeneration>
-      <LanguageStandard>stdcpp17</LanguageStandard>
+      <LanguageStandard>stdcpplatest</LanguageStandard>
       <LanguageStandard_C>stdc17</LanguageStandard_C>
       <AdditionalIncludeDirectories>$(ProjectDir)\..\monetdb\msvc</AdditionalIncludeDirectories>
     </ClCompile>
@@ -166,7 +206,7 @@
       <ConformanceMode>true</ConformanceMode>
       <MultiProcessorCompilation>true</MultiProcessorCompilation>
       <EnableParallelCodeGeneration>true</EnableParallelCodeGeneration>
-      <LanguageStandard>stdcpp17</LanguageStandard>
+      <LanguageStandard>stdcpplatest</LanguageStandard>
       <LanguageStandard_C>stdc17</LanguageStandard_C>
       <AdditionalIncludeDirectories>$(ProjectDir)\..\monetdb\msvc</AdditionalIncludeDirectories>
       <DebugInformationFormat>None</DebugInformationFormat>
@@ -179,6 +219,33 @@
       <AdditionalDependencies>..\libaquery.lib;$(ProjectDir)\..\monetdb\msvc\monetdbe.lib;%(AdditionalDependencies)</AdditionalDependencies>
       <DataExecutionPrevention>false</DataExecutionPrevention>
       <AdditionalOptions>/WHOLEARCHIVE:libaquery.lib %(AdditionalOptions)</AdditionalOptions>
+      <Profile>true</Profile>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='RelWDbgInf|Win32'">
+    <ClCompile>
+      <WarningLevel>Level3</WarningLevel>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <IntrinsicFunctions>true</IntrinsicFunctions>
+      <SDLCheck>true</SDLCheck>
+      <PreprocessorDefinitions>_CRT_SECURE_NO_WARNINGS;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <ConformanceMode>true</ConformanceMode>
+      <MultiProcessorCompilation>true</MultiProcessorCompilation>
+      <EnableParallelCodeGeneration>true</EnableParallelCodeGeneration>
+      <LanguageStandard>stdcpplatest</LanguageStandard>
+      <LanguageStandard_C>stdc17</LanguageStandard_C>
+      <AdditionalIncludeDirectories>$(ProjectDir)\..\monetdb\msvc</AdditionalIncludeDirectories>
+      <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <EnableCOMDATFolding>true</EnableCOMDATFolding>
+      <OptimizeReferences>true</OptimizeReferences>
+      <GenerateDebugInformation>DebugFull</GenerateDebugInformation>
+      <AdditionalDependencies>..\libaquery.lib;$(ProjectDir)\..\monetdb\msvc\monetdbe.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <DataExecutionPrevention>false</DataExecutionPrevention>
+      <AdditionalOptions>/WHOLEARCHIVE:libaquery.lib %(AdditionalOptions)</AdditionalOptions>
+      <AssemblyDebug>true</AssemblyDebug>
     </Link>
   </ItemDefinitionGroup>
   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='sharedlib|Win32'">
@@ -215,9 +282,10 @@
       <ConformanceMode>true</ConformanceMode>
       <MultiProcessorCompilation>true</MultiProcessorCompilation>
       <EnableParallelCodeGeneration>true</EnableParallelCodeGeneration>
-      <LanguageStandard>stdcpp17</LanguageStandard>
+      <LanguageStandard>stdcpplatest</LanguageStandard>
       <LanguageStandard_C>stdc17</LanguageStandard_C>
       <AdditionalIncludeDirectories>$(ProjectDir)\..\monetdb\msvc</AdditionalIncludeDirectories>
+      <BrowseInformation>true</BrowseInformation>
     </ClCompile>
     <Link>
       <SubSystem>Console</SubSystem>
@@ -225,6 +293,9 @@
       <AdditionalDependencies>..\libaquery.lib;$(ProjectDir)\..\monetdb\msvc\monetdbe.lib;%(AdditionalDependencies)</AdditionalDependencies>
       <AdditionalOptions>/WHOLEARCHIVE:libaquery.lib %(AdditionalOptions)</AdditionalOptions>
     </Link>
+    <Bscmake>
+      <PreserveSbr>true</PreserveSbr>
+    </Bscmake>
   </ItemDefinitionGroup>
   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
     <ClCompile>
@@ -236,7 +307,7 @@
       <ConformanceMode>true</ConformanceMode>
       <MultiProcessorCompilation>true</MultiProcessorCompilation>
       <EnableParallelCodeGeneration>true</EnableParallelCodeGeneration>
-      <LanguageStandard>stdcpp17</LanguageStandard>
+      <LanguageStandard>stdcpplatest</LanguageStandard>
       <LanguageStandard_C>stdc17</LanguageStandard_C>
       <AdditionalIncludeDirectories>$(ProjectDir)\..\monetdb\msvc</AdditionalIncludeDirectories>
       <DebugInformationFormat>None</DebugInformationFormat>
@@ -249,6 +320,33 @@
       <AdditionalDependencies>..\libaquery.lib;$(ProjectDir)\..\monetdb\msvc\monetdbe.lib;%(AdditionalDependencies)</AdditionalDependencies>
       <DataExecutionPrevention>false</DataExecutionPrevention>
       <AdditionalOptions>/WHOLEARCHIVE:libaquery.lib %(AdditionalOptions)</AdditionalOptions>
+      <Profile>true</Profile>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='RelWDbgInf|x64'">
+    <ClCompile>
+      <WarningLevel>Level3</WarningLevel>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <IntrinsicFunctions>true</IntrinsicFunctions>
+      <SDLCheck>true</SDLCheck>
+      <PreprocessorDefinitions>_CRT_SECURE_NO_WARNINGS;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <ConformanceMode>true</ConformanceMode>
+      <MultiProcessorCompilation>true</MultiProcessorCompilation>
+      <EnableParallelCodeGeneration>true</EnableParallelCodeGeneration>
+      <LanguageStandard>stdcpplatest</LanguageStandard>
+      <LanguageStandard_C>stdc17</LanguageStandard_C>
+      <AdditionalIncludeDirectories>$(ProjectDir)\..\monetdb\msvc</AdditionalIncludeDirectories>
+      <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <EnableCOMDATFolding>true</EnableCOMDATFolding>
+      <OptimizeReferences>true</OptimizeReferences>
+      <GenerateDebugInformation>DebugFull</GenerateDebugInformation>
+      <AdditionalDependencies>..\libaquery.lib;$(ProjectDir)\..\monetdb\msvc\monetdbe.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <DataExecutionPrevention>false</DataExecutionPrevention>
+      <AdditionalOptions>/WHOLEARCHIVE:libaquery.lib %(AdditionalOptions)</AdditionalOptions>
+      <AssemblyDebug>true</AssemblyDebug>
     </Link>
   </ItemDefinitionGroup>
   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='sharedlib|x64'">

msvs-py/msvs-py.pyproj

@@ -11,6 +11,9 @@
     <Name>msvs-py</Name>
     <RootNamespace>msvs-py</RootNamespace>
     <IsWindowsApplication>False</IsWindowsApplication>
+    <InterpreterId>Global|PythonCore|3.9</InterpreterId>
+    <LaunchProvider>Standard Python launcher</LaunchProvider>
+    <EnableNativeCodeDebugging>True</EnableNativeCodeDebugging>
   </PropertyGroup>
   <PropertyGroup Condition=" '$(Configuration)' == 'Debug' ">
     <DebugSymbols>true</DebugSymbols>
@@ -20,6 +23,11 @@
     <DebugSymbols>true</DebugSymbols>
     <EnableUnmanagedDebugging>false</EnableUnmanagedDebugging>
   </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)' == 'RelWDbgInf' ">
+    <DebugSymbols>true</DebugSymbols>
+    <EnableUnmanagedDebugging>false</EnableUnmanagedDebugging>
+    <OutputPath>bin\RelWDbgInf\</OutputPath>
+  </PropertyGroup>
   <ItemGroup>
     <Folder Include="..\aquery_parser\" />
     <Folder Include="..\engine\" />
@@ -41,16 +49,18 @@
     <Compile Include="..\engine\projection.py" />
     <Compile Include="..\engine\scan.py" />
     <Compile Include="..\engine\types.py" />
-	<Compile Include="..\reconstruct\storage.py" />
-	<Compile Include="..\reconstruct\expr.py" />
-	<Compile Include="..\reconstruct\__init__.py" />
-	<Compile Include="..\reconstruct\ast.py" />
+    <Compile Include="..\reconstruct\storage.py" />
+    <Compile Include="..\reconstruct\expr.py" />
+    <Compile Include="..\reconstruct\__init__.py" />
+    <Compile Include="..\reconstruct\ast.py" />
     <Compile Include="..\engine\__init__.py" />
-	<Compile Include="..\prompt.py" />
-	<Compile Include="..\build.py" />
-	<Compile Include="..\aquery_config.py" />
+    <Compile Include="..\prompt.py" />
+    <Compile Include="..\build.py" />
+    <Compile Include="..\aquery_config.py" />
+  </ItemGroup>
+  <ItemGroup>
+    <InterpreterReference Include="Global|PythonCore|3.9" />
   </ItemGroup>
-
   <Import Project="$(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion)\Python Tools\Microsoft.PythonTools.targets" />
   <!-- Uncomment the CoreCompile target to enable the Build command in
        Visual Studio and specify your pre- and post-build commands in

proctool.py

@@ -0,0 +1,59 @@
+import struct
+import readline
+from typing import List
+
+name : str = input('Filename (in path ./procedures/<filename>.aqp):')
+
+def write():
+    s : str = input('Enter queries: empty line to stop. \n')
+    qs : List[str] = []
+
+    while(len(s) and not s.startswith('S')):
+        qs.append(s)
+        s = input()
+
+    ms : int = int(input('number of modules:'))
+
+    with open(f'./procedures/{name}.aqp', 'wb') as fp:
+        fp.write(struct.pack("I", len(qs) + (ms > 0)))
+        fp.write(struct.pack("I", ms))
+        if (ms > 0):
+            fp.write(b'N\x00')
+            
+        for q in qs:
+            fp.write(q.encode('utf-8'))
+            if q.startswith('Q'):
+                fp.write(b'\n ')
+            fp.write(b'\x00')
+        
+
+def read(cmd : str):
+    rc = len(cmd) > 1 and cmd[1] == 'c'
+    clip = ''
+    with open(f'./procedures/{name}.aqp', 'rb') as fp:
+        nq = struct.unpack("I", fp.read(4))[0]
+        ms = struct.unpack("I", fp.read(4))[0]
+        qs = fp.read().split(b'\x00')
+        print(f'Procedure {name}, {nq} queries, {ms} modules:')
+        for q in qs:
+            q = q.decode('utf-8').strip()
+            if q:
+                q = f'"{q}",' if rc else f'\t{q}'
+                print(q)
+                clip += q + '\n'
+    if rc and not input('copy to clipboard?').lower().startswith('n'):
+        import pyperclip
+        pyperclip.copy(clip)    
+        
+if __name__ == '__main__':
+    while True:
+        cmd = input("r for read, rc to read c_str, w for write: ")
+        if cmd.lower().startswith('r'):
+            read(cmd.lower())
+            break
+        elif cmd.lower().startswith('w'):
+            write()    
+            break
+        elif cmd.lower().startswith('q'):
+            break
+        

prompt.py

@@ -119,6 +119,15 @@ class Backend_Type(enum.Enum):
 	BACKEND_MonetDB = 1
 	BACKEND_MariaDB = 2
 
+class StoredProcedure(ctypes.Structure):
+    _fields_ = [
+            ('cnt', ctypes.c_uint32), 
+            ('postproc_modules', ctypes.c_uint32), 
+            ('queries', ctypes.POINTER(ctypes.c_char_p)), 
+            ('name', ctypes.c_char_p), 
+            ('__rt_loaded_modules', ctypes.POINTER(ctypes.c_void_p)), 
+        ]
+
 @dataclass 
 class QueryStats:
     last_time : int = time.time()
@@ -229,6 +238,7 @@ class PromptState():
     server_bin = 'server.bin' if server_mode == RunType.IPC else 'server.so'
     wait_engine = lambda: None
     wake_engine = lambda: None
+    get_storedproc = lambda : StoredProcedure()
     set_ready = lambda: None
     get_ready = lambda: None
     server_status = lambda: False
@@ -246,6 +256,7 @@ class PromptState():
     stats : Optional[QueryStats] = None
     currstats : Optional[QueryStats] = None
     buildmgr : Optional[build_manager]= None
+    current_procedure : Optional[str] = None
 ## CLASSES END
 
 ## FUNCTIONS BEGIN
@@ -321,6 +332,8 @@ def init_threaded(state : PromptState):
         state.send = server_so['receive_args']
         state.wait_engine = server_so['wait_engine']
         state.wake_engine = server_so['wake_engine']
+        state.get_storedproc = server_so['get_procedure']
+        state.get_storedproc.restype = StoredProcedure
         aquery_config.have_hge = server_so['have_hge']()
         if aquery_config.have_hge != 0:
             from engine.types import get_int128_support
@@ -329,9 +342,11 @@ def init_threaded(state : PromptState):
         state.th.start() 
 
 def init_prompt() -> PromptState:
+    from engine.utils import session_context
     aquery_config.init_config()
     
     state = PromptState()
+    session_context = state
     # if aquery_config.rebuild_backend:
     #     try:
     #         os.remove(state.server_bin) 
@@ -392,7 +407,7 @@ def save(q:str, cxt: xengine.Context):
         savefile('udf', 'udf', '.hpp')
         savefile('sql', 'sql')
 
-def prompt(running = lambda:True, next = lambda:input('> '), state = None):
+def prompt(running = lambda:True, next = lambda:input('> '), state : Optional[PromptState] = None):
     if state is None:
         state = init_prompt()
     q = ''
@@ -453,7 +468,7 @@ def prompt(running = lambda:True, next = lambda:input('> '), state = None):
                 continue
             elif q.startswith('xexec') or q.startswith('exec'): # generate build and run (MonetDB Engine)
                 state.cfg.backend_type = Backend_Type.BACKEND_MonetDB.value
-                cxt = xengine.exec(state.stmts, cxt, keep)
+                cxt = xengine.exec(state.stmts, cxt, keep, parser.parse)
                 
                 this_udf = cxt.finalize_udf()
                 if this_udf:
@@ -609,6 +624,39 @@ def prompt(running = lambda:True, next = lambda:input('> '), state = None):
                 state.stats.need_print = True
                 state.stats.print(clear = False)
                 continue
+            elif q.startswith('procedure'):
+                qs = re.split(r'[ \t\r\n]', q)
+                procedure_help = '''Usage: procedure <procedure_name> [record|stop|run|remove|save|load]'''
+                from engine.utils import send_to_server
+                if len(qs) > 2:
+                    if qs[2].lower() =='record':
+                        if state.current_procedure is not None and state.current_procedure != qs[1]:             
+                            print(f'Cannot record 2 procedures at the same time. Stop recording {state.current_procedure} first.')
+                        elif state.current_procedure is None:
+                            state.current_procedure = qs[1]
+                            send_to_server(f'R\0{qs[1]}')
+                    elif qs[2].lower() == 'stop':
+                        send_to_server(f'RT\0{qs[1]}')
+                        state.current_procedure = None
+                    else:
+                        if state.current_procedure:
+                            print(f'Procedure manipulation commands are disallowed during procedure recording.')
+                            continue
+                        if qs[2].lower() == 'run':
+                            send_to_server(f'RE\0{qs[1]}')
+                        elif qs[2].lower() == 'remove':
+                            send_to_server(f'RD\0{qs[1]}')
+                        elif qs[2].lower() == 'save':
+                            send_to_server(f'RS\0{qs[1]}')
+                        elif qs[2].lower() == 'load':
+                            send_to_server(f'RL\0{qs[1]}')
+                     
+                elif len(qs) > 1:
+                    if qs[1].lower() == 'display':
+                        send_to_server(f'Rd\0')
+                else:
+                    print(procedure_help)
+                continue
             trimed = ws.sub(' ', og_q).split(' ') 
             if len(trimed) > 1 and trimed[0].lower().startswith('fi') or trimed[0].lower() == 'f':
                 fn = 'stock.a' if len(trimed) <= 1 or len(trimed[1]) == 0 \

reconstruct/__init__.py

@@ -18,10 +18,11 @@ def generate(ast, cxt):
         if k in ast_node.types.keys():
             ast_node.types[k](None, ast, cxt)
 
-def exec(stmts, cxt = None, keep = False):
+def exec(stmts, cxt = None, keep = False, parser = None):
     if 'stmts' not in stmts:
         return
     cxt = initialize(cxt, keep)
+    cxt.parser = parser
     stmts_stmts = stmts['stmts']
     if type(stmts_stmts) is list:
         for s in stmts_stmts:
@@ -29,7 +30,8 @@ def exec(stmts, cxt = None, keep = False):
     else:
         generate(stmts_stmts, cxt)
     for q in cxt.queries:
-        cxt.print(q.strip())
+        if not q.startswith('O'):
+            cxt.print(q.strip())
     return cxt
     
 __all__ = ["initialize", "generate", "exec", "saved_cxt"]

reconstruct/ast.py

@@ -4,8 +4,8 @@ from enum import Enum, auto
 from typing import Dict, List, Optional, Set, Tuple, Union
 
 from engine.types import *
-from engine.utils import (base62alp, base62uuid, enlist, get_innermost,
-                          get_legal_name)
+from engine.utils import (base62alp, base62uuid, enlist, 
+                          get_innermost, get_legal_name)
 from reconstruct.storage import ColRef, Context, TableInfo
 
 class ast_node:
@@ -246,7 +246,7 @@ class projection(ast_node):
             self.datasource.rec = None
             # TODO: Type deduction in Python
             for t, n, c in zip(this_type, disp_name, compound):
-                cols.append(ColRef(t, self.out_table, None, n, len(cols), compound=c))
+                cols.append(ColRef(t, None, self.out_table, n, len(cols), compound=c))
         
         self.out_table.add_cols(cols, new = False)
         
@@ -281,11 +281,11 @@ class projection(ast_node):
         #     self.into_stub = f'{{INTOSTUB{base62uuid(20)}}}'
         #     self.add(self.into_stub, '')
             
-        def finialize(astnode:ast_node):
+        def finalize(astnode:ast_node):
             if(astnode is not None):
                 self.add(astnode.sql)
-        finialize(self.datasource)                
-        finialize(self.where)
+        finalize(self.datasource)                
+        finalize(self.where)
         if self.group_node and not self.group_node.use_sp_gb:
             self.add(self.group_node.sql)
 
@@ -339,8 +339,8 @@ class projection(ast_node):
                 return ', '.join([self.pyname2cname[n.name] for n in lst_names])
             else:
                 return self.pyname2cname[proj_name]
-        
-        for key, val in proj_map.items():
+        gb_tovec = [False] * len(proj_map)
+        for i, (key, val) in enumerate(proj_map.items()):
             if type(val[1]) is str:
                 x = True
                 y = get_proj_name
@@ -357,22 +357,27 @@ class projection(ast_node):
                 out_typenames[key] = decltypestring
             else:
                 out_typenames[key] = val[0].cname
-            if (type(val[2].udf_called) is udf and # should bulkret also be colref?
+            elemental_ret_udf = (
+                    type(val[2].udf_called) is udf and # should bulkret also be colref?
                     val[2].udf_called.return_pattern == udf.ReturnPattern.elemental_return
-                    or 
-                    self.group_node and 
-                    (self.group_node.use_sp_gb and
+            )
+            folding_vector_groups = (
+                self.group_node and 
+                (
+                    self.group_node.use_sp_gb and
                     val[2].cols_mentioned.intersection(
                         self.datasource.all_cols().difference(
                             self.datasource.get_joint_cols(self.group_node.refs)
-                        ))
-                    ) and val[2].is_compound # compound val not in key
-                    # or 
-                    # val[2].is_compound > 1
-                    # (not self.group_node and val[2].is_compound)
-                    ):
-                    out_typenames[key] = f'vector_type<{out_typenames[key]}>'
-                    self.out_table.columns[key].compound = True
+                        )
+                    )
+                ) and 
+                val[2].is_compound # compound val not in key
+            )
+            if (elemental_ret_udf or folding_vector_groups):
+                out_typenames[key] = f'vector_type<{out_typenames[key]}>'
+                self.out_table.columns[key].compound = True
+                if self.group_node is not None and self.group_node.use_sp_gb:
+                    gb_tovec[i] = True
         outtable_col_nameslist = ', '.join([f'"{c.name}"' for c in self.out_table.columns])
         self.outtable_col_names = 'names_' + base62uuid(4)
         self.context.emitc(f'const char* {self.outtable_col_names}[] = {{{outtable_col_nameslist}}};')
@@ -383,12 +388,15 @@ class projection(ast_node):
         if self.group_node is not None and self.group_node.use_sp_gb:
             gb_vartable : Dict[str, Union[str, int]] = deepcopy(self.pyname2cname)
             gb_cexprs : List[str] = []
-            
+            gb_colnames : List[str] = []
+            gb_types : List[Types] = []
             for key, val in proj_map.items():
                 col_name = 'col_' + base62uuid(6)
                 self.context.emitc(f'decltype(auto) {col_name} = {self.out_table.contextname_cpp}->get_col<{key}>();')
                 gb_cexprs.append((col_name, val[2]))
-            self.group_node.finalize(gb_cexprs, gb_vartable)
+                gb_colnames.append(col_name)
+                gb_types.append(val[0])
+            self.group_node.finalize(gb_cexprs, gb_vartable, gb_colnames, gb_types, gb_tovec)
         else:
             for i, (key, val) in enumerate(proj_map.items()):
                 if type(val[1]) is int:
@@ -409,13 +417,14 @@ class projection(ast_node):
         if self.outfile and self.has_postproc:
                 self.outfile.finalize()
 
-        if 'into' in node: 
-            self.context.emitc(select_into(self, node['into']).ccode)
-            self.has_postproc = True
         if not self.distinct:
-            self.finalize()
-                    
-    def finalize(self):      
+            self.finalize(node)
+        
+    def deal_with_into(self, node):
+        if 'into' in node: 
+            self.context.emitc(select_into(self, node['into']).ccode)            
+    def finalize(self, node):
+        self.deal_with_into(node)
         self.context.emitc(f'puts("done.");')
 
         if self.parent is None:
@@ -436,7 +445,7 @@ class select_distinct(projection):
             self.context.emitc(
                 f'{self.out_table.contextname_cpp}->distinct();'
             )
-        self.finalize()
+        self.finalize(node)
         
 class select_into(ast_node):
     def init(self, _):
@@ -531,11 +540,12 @@ class groupby_c(ast_node):
     def init(self, node : List[Tuple[expr, Set[ColRef]]]):
         self.proj : projection = self.parent
         self.glist : List[Tuple[expr, Set[ColRef]]] = node
+        self.vecs : str = 'vecs_' + base62uuid(3)
         return super().init(node)
     
     def produce(self, node : List[Tuple[expr, Set[ColRef]]]):
         self.context.headers.add('"./server/hasher.h"')
-        self.context.headers.add('unordered_map')
+        # self.context.headers.add('unordered_map')
         self.group = 'g' + base62uuid(7)
         self.group_type = 'record_type' + base62uuid(7)
         self.datasource = self.proj.datasource
@@ -559,20 +569,22 @@ class groupby_c(ast_node):
                 e = g_str
             g_contents_list.append(e)
         first_col = g_contents_list[0]
+        self.total_sz = 'len_' + base62uuid(4)
+        self.context.emitc(f'uint32_t {self.total_sz} = {first_col}.size;')
         g_contents_decltype = [f'decays<decltype({c})::value_t>' for c in g_contents_list]
         g_contents = ', '.join(
             [f'{c}[{scanner_itname}]' for c in g_contents_list]
         )
         self.context.emitc(f'typedef record<{",".join(g_contents_decltype)}> {self.group_type};')
-        self.context.emitc(f'unordered_map<{self.group_type}, vector_type<uint32_t>, '
-            f'transTypes<{self.group_type}, hasher>> {self.group};')
+        self.context.emitc(f'AQHashTable<{self.group_type}, '
+            f'transTypes<{self.group_type}, hasher>> {self.group} {{{self.total_sz}}};')
         self.n_grps = len(self.glist)
-        self.scanner = scan(self, first_col + '.size', it_name=scanner_itname)
-        self.scanner.add(f'{self.group}[forward_as_tuple({g_contents})].emplace_back({self.scanner.it_var});')
+        self.scanner = scan(self, self.total_sz, it_name=scanner_itname)
+        self.scanner.add(f'{self.group}.hashtable_push(forward_as_tuple({g_contents}), {self.scanner.it_var});')
 
     def consume(self, _):
         self.scanner.finalize()
-        
+        self.context.emitc(f'auto {self.vecs} = {self.group}.ht_postproc({self.total_sz});')
     # def deal_with_assumptions(self, assumption:assumption, out:TableInfo):
     #     gscanner = scan(self, self.group)
     #     val_var = 'val_'+base62uuid(7)
@@ -580,13 +592,45 @@ class groupby_c(ast_node):
     #     gscanner.add(f'{self.datasource.cxt_name}->order_by<{assumption.result()}>(&{val_var});')
     #     gscanner.finalize()
         
-    def finalize(self, cexprs : List[Tuple[str, expr]], var_table : Dict[str, Union[str, int]]):
-        gscanner = scan(self, self.group, loop_style = 'for_each')
+    def finalize(self, cexprs : List[Tuple[str, expr]], var_table : Dict[str, Union[str, int]], 
+                 col_names : List[str], col_types : List[Types], col_tovec : List[bool]):
+        tovec_columns = set()
+        for i, c in enumerate(col_names):
+            if col_tovec[i]: # and type(col_types[i]) is VectorT:
+                self.context.emitc(f'{c}.resize({self.group}.size());')
+                typename : Types = col_types[i] # .inner_type
+                self.context.emitc(f'auto buf_{c} = static_cast<{typename.cname} *>(calloc({self.total_sz}, sizeof({typename.cname})));')
+                tovec_columns.add(c)
+            else:
+                self.context.emitc(f'{c}.reserve({self.group}.size());')
+                
+        self.arr_len = 'arrlen_' + base62uuid(3)
+        self.arr_values = 'arrvals_' + base62uuid(3)
+        
+        self.context.emitc(f'auto {self.arr_len} = {self.group}.size();')
+        self.context.emitc(f'auto {self.arr_values} = {self.group}.values();')
+        
+        if len(tovec_columns):
+            preproc_scanner = scan(self, self.arr_len)
+            preproc_scanner_it = preproc_scanner.it_var
+            for c in tovec_columns:
+                preproc_scanner.add(f'{c}[{preproc_scanner_it}].init_from'
+                                    f'({self.vecs}[{preproc_scanner_it}].size,'
+                                    f' {"buf_" + c} + {self.group}.ht_base'
+                                    f'[{preproc_scanner_it}]);'
+                )
+            preproc_scanner.finalize()
+        
+        self.context.emitc(f'GC::scratch_space = GC::gc_handle ? &(GC::gc_handle->scratch) : nullptr;')
+        # gscanner = scan(self, self.group, loop_style = 'for_each')
+        gscanner = scan(self, self.arr_len)
         key_var = 'key_'+base62uuid(7)
         val_var = 'val_'+base62uuid(7)
         
-        gscanner.add(f'auto &{key_var} = {gscanner.it_var}.first;', position = 'front')
-        gscanner.add(f'auto &{val_var} = {gscanner.it_var}.second;', position = 'front')
+        # gscanner.add(f'auto &{key_var} = {gscanner.it_var}.first;', position = 'front')
+        # gscanner.add(f'auto &{val_var} = {gscanner.it_var}.second;', position = 'front')
+        gscanner.add(f'auto &{key_var} = {self.arr_values}[{gscanner.it_var}];', position = 'front')
+        gscanner.add(f'auto &{val_var} = {self.vecs}[{gscanner.it_var}];', position = 'front')
         len_var = None
         def define_len_var():
             nonlocal len_var
@@ -621,7 +665,7 @@ class groupby_c(ast_node):
                         materialize_builtin = materialize_builtin, 
                         count=lambda:f'{val_var}.size')
                 
-        for ce in cexprs:
+        for i, ce in enumerate(cexprs):
             ex = ce[1]
             materialize_builtin = {}
             if type(ex.udf_called) is udf:
@@ -634,9 +678,18 @@ class groupby_c(ast_node):
                     materialize_builtin['_builtin_ret'] = f'{ce[0]}.back()'
                     gscanner.add(f'{ex.eval(c_code = True, y=get_var_names, materialize_builtin = materialize_builtin)};\n')
                     continue
-            gscanner.add(f'{ce[0]}.emplace_back({get_var_names_ex(ex)});\n')
+            if col_tovec[i]:
+                if ex.remake_binary(f'{ce[0]}[{gscanner.it_var}]'):
+                    gscanner.add(f'{get_var_names_ex(ex)};\n')
+                else:
+                    gscanner.add(f'{ce[0]}[{gscanner.it_var}] = {get_var_names_ex(ex)};\n')
+            else:
+                gscanner.add(f'{ce[0]}.emplace_back({get_var_names_ex(ex)});\n')
+        
+        gscanner.add(f'GC::scratch_space->release();')
         
         gscanner.finalize()
+        self.context.emitc(f'GC::scratch_space = nullptr;')
         
         self.datasource.groupinfo = None
 
@@ -712,10 +765,11 @@ class groupby(ast_node):
                 #     self.parent.var_table.
                 self.parent.col_ext.update(l[1])    
                 
-    def finalize(self, cexprs : List[Tuple[str, expr]], var_table : Dict[str, Union[str, int]]):
+    def finalize(self, cexprs : List[Tuple[str, expr]], var_table : Dict[str, Union[str, int]], 
+                 col_names : List[str], col_types : List[Types], col_tovec : List[bool]):
         if self.use_sp_gb:
             self.dedicated_gb = groupby_c(self.parent, self.dedicated_glist)
-            self.dedicated_gb.finalize(cexprs, var_table)
+            self.dedicated_gb.finalize(cexprs, var_table, col_names, col_types, col_tovec)
 
 
 class join(ast_node):
@@ -955,19 +1009,31 @@ class union_all(ast_node):
     sql_name = 'UNION ALL'
     def produce(self, node):
         queries = node[self.name]
-        generated_queries : List[Optional[projection]] = [None] * len(queries)
+        self.generated_queries : List[Optional[projection]] = [None] * len(queries)
         is_standard = True
         for i, q in enumerate(queries):
             if 'select' in q:
-                generated_queries[i] = projection(self, q)
-                is_standard &= not generated_queries[i].has_postproc
+                self.generated_queries[i] = projection(self, q)
+                is_standard &= not self.generated_queries[i].has_postproc
         if is_standard:
-            self.sql = f' {self.sql_name} '.join([q.sql for q in generated_queries])
+            self.sql = f' {self.sql_name} '.join([q.sql for q in self.generated_queries])
         else:
             raise NotImplementedError(f"{self.sql_name} only support standard sql for now")
     def consume(self, node):
+        if 'into' in node:
+            outtable = TableInfo(node['into'], [], self.context)
+            lst_cols = [None] * len(self.generated_queries[0].out_table.columns)
+            for i, c in enumerate(self.generated_queries[0].out_table.columns):
+                lst_cols[i] = ColRef(c.type, None, outtable, c.name, i, c.compound)
+            outtable.add_cols(lst_cols, new = False)
+            
+            col_names = [c.name for c in outtable.columns]
+            col_names = '(' + ', '.join(col_names) + ')'
+            self.sql = f'CREATE TABLE {node["into"]} {col_names} AS {self.sql}'
         super().consume(node)
-        self.context.direct_output()
+        if 'into' not in node:
+            self.context.direct_output()
+        
 
 class except_clause(union_all):
     name = 'except'
@@ -1015,7 +1081,63 @@ class create_table(ast_node):
         self.sql += ')'
         if self.context.use_columnstore:
             self.sql += ' engine=ColumnStore'
-                    
+
+class create_trigger(ast_node):
+    name = 'create_trigger'
+    first_order = name
+    class Type (Enum):
+        Interval = auto()
+        Callback = auto()
+        
+    def produce(self, node):
+        from engine.utils import send_to_server, get_storedproc
+        node = node['create_trigger']
+        self.trigger_name = node['name']
+        self.action_name = node['action']
+        self.action = get_storedproc(self.action_name)
+        if self.trigger_name in self.context.triggers:
+            raise ValueError(f'trigger {self.trigger_name} exists')
+        elif self.action:
+            raise ValueError(f'Stored Procedure {self.action_name} do not exist')
+
+        if 'interval' in node: # executed periodically from server
+            self.type = self.Type.Interval
+            self.interval = node['interval']
+            send_to_server(f'TI{self.trigger_name}\0{self.action_name}\0{self.interval}')
+        else: # executed from sql backend
+            self.type = self.Type.Callback
+            self.query_name = node['query']
+            self.table_name = node['table']
+            self.procedure = get_storedproc(self.query_name)
+            if self.procedure and self.table_name in self.context.tables_byname:
+                self.table = self.context.tables_byname[self.table_name]
+                self.table.triggers.add(self)
+            else:
+                return
+        self.context.triggers[self.trigger_name] = self
+
+    # manually execute trigger
+    def register(self): 
+        if self.type != self.Type.Callback:
+            self.context.triggers.pop(self.trigger_name)
+            raise ValueError(f'Trigger {self.trigger_name} is not a callback based trigger')
+        self.context.triggers_active.add(self)
+
+    def execute(self): 
+        from engine.utils import send_to_server
+        send_to_server(f'TC{self.query_name}\0{self.action_name}')
+
+    def remove(self):
+        from engine.utils import send_to_server
+        send_to_server(f'TR{self.trigger_name}')
+
+
+class drop_trigger(ast_node):
+    name = 'drop_trigger'
+    first_order = name
+    def produce(self, node):
+        ...
+
 class drop(ast_node):
     name = 'drop'
     first_order = name
@@ -1045,9 +1167,11 @@ class insert(ast_node):
             complex_query_kw = ['from', 'where', 'groupby', 'having', 'orderby', 'limit']
             if any([kw in values for kw in complex_query_kw]):
                 values['into'] = node['insert']
-                proj_cls = (select_distinct 
-                if 'select_distinct' in values 
-                else projection)
+                proj_cls = (
+                    select_distinct 
+                    if 'select_distinct' in values 
+                    else projection
+                )
                 proj_cls(None, values, self.context)
                 self.produce = lambda*_:None
                 self.spawn = lambda*_:None
@@ -1081,6 +1205,11 @@ class insert(ast_node):
             
         keys = f'({", ".join(keys)})' if keys else ''
         tbl = node['insert']
+        if tbl not in self.context.tables_byname:
+            print('Warning: {tbl} not registered in aquery compiler.')
+        tbl_obj = self.context.tables_byname[tbl]
+        for t in tbl_obj.triggers:
+            t.register()
         self.sql = f'INSERT INTO {tbl}{keys} VALUES'
         # if len(values) != table.n_cols:
         #     raise ValueError("Column Mismatch")
@@ -1095,7 +1224,7 @@ class insert(ast_node):
             list_values.append(f"({', '.join(inner_list_values)})")
             
         self.sql += ', '.join(list_values) 
-        
+
 
 class delete_from(ast_node):
     name = 'delete'
@@ -1282,7 +1411,7 @@ class outfile(ast_node):
         filename = self.node['loc']['literal'] if 'loc' in self.node else self.node['literal']
         sep = ',' if 'term' not in self.node else self.node['term']['literal']
         file_pointer = 'fp_' + base62uuid(6)
-        self.addc(f'FILE* {file_pointer} = fopen("{filename}", "w");')
+        self.addc(f'FILE* {file_pointer} = fopen("{filename}", "wb");')
         self.addc(f'{self.parent.out_table.contextname_cpp}->printall("{sep}", "\\n", nullptr, {file_pointer});')
         self.addc(f'fclose({file_pointer});')  
         self.context.ccode += self.ccode
@@ -1558,6 +1687,11 @@ class passthru_sql(ast_node):
     seprator = re.compile(r'''((?:[^;"']|"[^"]*"|'[^']*')+)''')
     def __init__(self, _, node, context:Context):
         sqls = passthru_sql.seprator.split(node['sql'])
+        try:
+            if callable(context.parser):
+                parsed = context.parser(node['sql'])
+        except BaseException:
+            parsed = None
         for sql in sqls:
             sq = sql.strip(' \t\n\r;')
             if sq:

reconstruct/expr.py

@@ -367,6 +367,19 @@ class expr(ast_node):
             self.curr_code += c.codegen(delegate)
         return self.curr_code
     
+    def remake_binary(self, ret_expr):
+        if self.root:
+            self.oldsql = self.sql
+            if (self.opname in builtin_binary_ops):
+                patched_opname = 'aqop_' + self.opname
+                self.sql = (f'{patched_opname}({self.children[0].sql}, '
+                            f'{self.children[1].sql}, {ret_expr})')
+                return True
+            elif self.opname in builtin_vecfunc:
+                self.sql = self.sql[:self.sql.rindex(')')]
+                self.sql += ', ' + ret_expr + ')'
+                return True
+        return False
     def __str__(self):
         return self.sql
     def __repr__(self):

reconstruct/storage.py

@@ -64,12 +64,14 @@ class ColRef:
 
 class TableInfo:
     def __init__(self, table_name, cols, cxt:'Context'):
+        from reconstruct.ast import create_trigger
         # statics
         self.table_name : str = table_name
         self.contextname_cpp : str = ''
         self.alias : Set[str] = set([table_name])
         self.columns_byname : CaseInsensitiveDict[str, ColRef] = CaseInsensitiveDict() # column_name, type
         self.columns : List[ColRef] = []
+        self.triggers : Set[create_trigger] = set()
         self.cxt = cxt
         # keep track of temp vars
         self.rec = None 
@@ -83,7 +85,7 @@ class TableInfo:
     def add_cols(self, cols, new = True):
         for c in enlist(cols):
             self.add_col(c, new)
-            
+        
     def add_col(self, c, new = True):
         _ty = c['type']
         _ty_args = None
@@ -91,7 +93,7 @@ class TableInfo:
             _ty_val = list(_ty.keys())[0]
             _ty_args = _ty[_ty_val]
             _ty = _ty_val
-        if new:
+        if new or type(c) is not ColRef:
             col_object = ColRef(_ty, c, self, c['name'], len(self.columns), _ty_args = _ty_args)
         else:
             col_object = c
@@ -144,10 +146,9 @@ class TableInfo:
     
 class Context:
     def new(self):
-        self.headers = set(['\"./server/libaquery.h\"', 
-                            '\"./server/monetdb_conn.h\"'])
-
+        self.headers = set(['\"./server/monetdb_conn.h\"'])
         self.ccode = ''
+
         self.sql = ''  
         self.finalized = False
         self.udf = None
@@ -157,9 +158,11 @@ class Context:
         self.module_init_loc = 0
         self.special_gb = False
         self.has_dll = False
-         
+        self.triggers_active.clear()
+        
     def __init__(self):
-        self.tables_byname = dict()
+        from .ast import create_trigger
+        self.tables_byname : Dict[str, TableInfo] = dict()
         self.col_byname = dict()
         self.tables : Set[TableInfo] = set()
         self.cols = []
@@ -175,7 +178,11 @@ class Context:
         self.have_hge = False
         self.Error = lambda *args: print(*args)
         self.Info = lambda *_: None
-        
+        self.triggers : Dict[str, create_trigger] = dict()
+        self.triggers_active = set()
+        self.stored_proceudres = dict()
+        # self.new() called everytime new query batch is started
+
     def get_scan_var(self):
         it_var = 'i' + base62uuid(2)
         scan_vars = set(s.it_var for s in self.scans)
@@ -256,7 +263,19 @@ class Context:
         limit = limit.to_bytes(4, 'little').decode('latin-1')
         self.queries.append(
             'O' + limit + sep + end)
-    
+        
+    def remove_trigger(self, name : str):
+        from reconstruct.ast import create_trigger
+        val = self.triggers.pop(name, None)
+        if val.type == create_trigger.Type.Callback:
+            val.table.triggers.remove(val)
+        val.remove()
+
+    def post_exec_triggers(self):
+        for t in self.triggers_active:
+            t.execute()
+        self.triggers_active.clear()
+
     def abandon_postproc(self):
         self.ccode = ''
         self.finalize_query()
@@ -277,7 +296,8 @@ class Context:
             headers = ''
             # if build_driver == 'MSBuild':
                 # headers ='#include \"./server/pch.hpp\"\n'
-             
+            with open('header.cxx', 'r') as header:
+                headers += header.read() 
             for h in self.headers:
                 if h[0] != '"':
                     headers += '#include <' + h + '>\n'
@@ -287,6 +307,6 @@ class Context:
                 headers += '#undef max\n'
                 headers += '#undef min\n'
 
-            self.ccode = headers + '\n'.join(self.procs)
+            self.ccode += headers + '\n'.join(self.procs)
             self.headers = set()
         return self.ccode

rfdata_preproc.py

@@ -0,0 +1,44 @@
+import os
+
+sep = os.sep
+
+# toggles
+dataset = 'power' # [covtype, electricity, mixed, phishing, power]
+use_threadpool = False # True
+
+# environments
+input_prefix = f'data{sep}{dataset}_orig'
+output_prefix = f'data{sep}{dataset}'
+sep_field = b','
+sep_subfield = b';'
+
+lst_files = os.listdir(input_prefix)
+# lst_files.sort()
+
+try:
+    os.mkdir(output_prefix)
+except FileExistsError:
+    pass
+
+def process(f : str):
+    filename = input_prefix + sep + f
+    ofilename = output_prefix + sep + f[:-3] + 'csv'
+    with open(filename, 'rb') as ifile:
+        icontents = ifile.read()
+        with open(ofilename, 'wb') as ofile:
+            ofile.write(b'\n')
+            for l in icontents.splitlines():
+                fields = l.strip().split(b' ')
+                subfields = fields[:-1]
+                ol = ( # fields[0] + sep_field +
+                      sep_subfield.join(subfields) +
+                      sep_field + fields[-1] + b'\n')
+                ofile.write(ol)
+
+if not use_threadpool:
+    for f in lst_files: 
+        process(f)
+elif __name__ == '__main__':
+    from multiprocessing import Pool
+    with Pool(8) as tp:
+        tp.map(process, lst_files)

sdk/DecisionTree.h

@@ -9,49 +9,52 @@ struct DR;
 
 struct DT;
 
-//enum Evaluation {gini, entropy, logLoss};
-
-class DecisionTree{
+class DecisionTree
+{
 public:
+    DT *DTree = nullptr;
+    double minIG;
+    long maxHeight;
+    long feature;
+    long maxFeature;
+    bool isRF;
+    long classes;
+    int *Sparse;
+    double forgetRate;
+    double increaseRate;
+    double initialIR;
+    Evaluation evalue;
+    long Rebuild;
+    long roundNo;
+    long called;
+    long retain;
+    long lastT;
+    long lastAll;
 
-DT* DTree = nullptr;
-int maxHeight;
-long feature;
-long maxFeature;
-long seed;
-long classes;
-int* Sparse;
-double forgetRate;
-Evaluation evalue;
-long Rebuild;
-long roundNo;
-long called;
-long retain;
+    DecisionTree(long f, int *sparse, double forget, long maxFeature, long noClasses, Evaluation e);
 
-DecisionTree(int hight, long f, int* sparse, double forget, long maxFeature, long noClasses, Evaluation e, long r, long rb);
+    void Stablelize();
 
-void Stablelize();
+    void Free();
 
-void Free();
+    minEval findMinGiniDense(double **data, long *result, long *totalT, long size, long col);
 
-minEval findMinGiniDense(double** data, long* result, long* totalT, long size, long col);
+    minEval findMinGiniSparse(double **data, long *result, long *totalT, long size, long col, DT *current);
 
-minEval findMinGiniSparse(double** data, long* result, long* totalT, long size, long col, DT* current);
+    minEval incrementalMinGiniDense(double **data, long *result, long size, long col, long ***count, double **record, long *max, long newCount, long forgetSize, double **forgottenData, long *forgottenClass);
 
-minEval incrementalMinGiniDense(double** data, long* result, long size, long col, long*** count, double** record, long* max, long newCount, long forgetSize, bool isRoot);
+    minEval incrementalMinGiniSparse(double **dataNew, long *resultNew, long sizeNew, long sizeOld, DT *current, long col, long forgetSize, double **forgottenData, long *forgottenClass);
 
-minEval incrementalMinGiniSparse(double** dataNew, long* resultNew, long sizeNew, long sizeOld, DT* current, long col, long forgetSize, bool isRoot);
+    long *fitThenPredict(double **trainData, long *trainResult, long trainSize, double **testData, long testSize);
 
-long* fitThenPredict(double** trainData, long* trainResult, long trainSize, double** testData, long testSize);
+    void fit(double **data, long *result, long size);
 
-void fit(double** data, long* result, long size);
+    void Update(double **data, long *result, long size, DT *current);
 
-void Update(double** data, long* result, long size, DT* current);
+    void IncrementalUpdate(double **data, long *result, long size, DT *current);
 
-void IncrementalUpdate(double** data, long* result, long size, DT* current);
+    long Test(double *data, DT *root);
 
-long Test(double* data, DT* root);
-
-void print(DT* root);
+    void print(DT *root);
 };
 #endif

sdk/Evaluation.cpp

@@ -26,7 +26,7 @@ minEval giniSparse(double** data, long* result, long* d, long size, long col, lo
         double gini1, gini2;
         double c;
         long l, r;
-        for(i=0; i<size; i++){
+        for(i=0; i<size-1; i++){
                 c = data[d[i]][col];
                 if(c==max)break;
                 count[result[d[i]]]++;
@@ -62,7 +62,7 @@ minEval entropySparse(double** data, long* result, long* d, long size, long col,
         double entropy1, entropy2;
         double c;
         long l, r;
-        for(i=0; i<size; i++){
+        for(i=0; i<size-1; i++){
                 c = data[d[i]][col];
                 if(c==max)break;
                 count[result[d[i]]]++;
@@ -73,8 +73,8 @@ minEval entropySparse(double** data, long* result, long* d, long size, long col,
                 for(j=0;j<classes;j++){
                         l = count[j];
                         r = totalT[j]-l;
-                        entropy1 -= ((double)l/total)*log((double)l/total);
-                        entropy2 -= ((double)r/(size-total))*log((double)r/(size-total));
+                        if(l!=0)entropy1 -= ((double)l/total)*log((double)l/total);
+                        if(r!=0)entropy2 -= ((double)r/(size-total))*log((double)r/(size-total));
                 }
                 entropy1 = entropy1*total/size + entropy2*(size-total)/size;
                 if(ret.eval>entropy1){
@@ -140,8 +140,8 @@ minEval entropySparseIncremental(long sizeTotal, long classes, double* newSorted
                 for(j=0;j<classes;j++){
                         l = count[j];
                         r = T[j]-l;
-                        e1 -= ((double)l/total)*log((double)l/total);
-                        e2 -= ((double)r/(sizeTotal-total))*log((double)r/(sizeTotal-total));
+                        if(l!=0)e1 -= ((double)l/total)*log((double)l/total);
+                        if(r!=0)e2 -= ((double)r/(sizeTotal-total))*log((double)r/(sizeTotal-total));
                 }
                 e1 = e1*total/sizeTotal + e2*(sizeTotal-total)/sizeTotal;
                 if(ret.eval>e1){
@@ -159,9 +159,9 @@ minEval giniDense(long max, long size, long classes, long** rem, long* d, double
 	double gini1, gini2;
 	long *t, *t2, *r, *r2, i, j;
         for(i=0;i<max;i++){
-                t = rem[d[i]];
+                t = rem[i];
                 if(i>0){
-                        t2 = rem[d[i-1]];
+                        t2 = rem[i-1];
                         for(j=0;j<=classes;j++){
                                 t[j]+=t2[j];
                         }
@@ -179,7 +179,7 @@ minEval giniDense(long max, long size, long classes, long** rem, long* d, double
                 gini1 = (gini1*t[classes])/size + (gini2*(size-t[classes]))/size;
                 if(gini1<ret.eval){
                         ret.eval = gini1;
-                        ret.value = record[d[i]];
+                        ret.value = record[i];
                         ret.left = t[classes];
                 }
 	}
@@ -193,9 +193,9 @@ minEval entropyDense(long max, long size, long classes, long** rem, long* d, dou
 	double entropy1, entropy2;
 	long *t, *t2, *r, *r2, i, j;
         for(i=0;i<max;i++){
-                t = rem[d[i]];
+                t = rem[i];
                 if(i>0){
-                        t2 = rem[d[i-1]];
+                        t2 = rem[i-1];
                         for(j=0;j<=classes;j++){
                                 t[j]+=t2[j];
                         }
@@ -207,71 +207,15 @@ minEval entropyDense(long max, long size, long classes, long** rem, long* d, dou
                         long l, r;
                         l = t[j];
                         r = totalT[j]-l;
-                        entropy1 -= ((double)l/t[classes])*log((double)l/t[classes]);
-                        entropy2 -= ((double)r/(size-t[classes]))*log((double)r/(size-t[classes]));
+                        if(l!=0)entropy1 -= ((double)l/t[classes])*log((double)l/t[classes]);
+                        if(r!=0)entropy2 -= ((double)r/(size-t[classes]))*log((double)r/(size-t[classes]));
                 }
                 entropy1 = entropy1*t[classes]/size + entropy2*(size-t[classes])/size;
                 if(entropy1<ret.eval){
                         ret.eval = entropy1;
-                        ret.value = record[d[i]];
+                        ret.value = record[i];
                         ret.left = t[classes];
                 }
         }
 	return ret;
 }
-
-minEval giniDenseIncremental(long max, double* record, long** count, long classes, long newSize, long* T){
-	double gini1, gini2;
-        minEval ret;
-	long i, j;
-
-	ret.eval = DBL_MAX;
-        for(i=0; i<max; i++){
-                if(count[i][classes]==newSize){
-                        continue;
-                }
-                gini1 = 1.0;
-                gini2 = 1.0;
-                for(j=0;j<classes;j++){
-                        long l, r;
-                        l = count[i][j];
-                        r = T[j]-l;
-                        gini1 -= pow((double)l/count[i][classes], 2);
-                        gini2 -= pow((double)r/(newSize-count[i][classes]), 2);
-                }
-                gini1 = gini1*count[i][classes]/newSize + gini2*((newSize-count[i][classes]))/newSize;
-                if(gini1<ret.eval){
-                        ret.eval = gini1;
-                        ret.value = record[i];
-                }
-        }
-	return ret;
-}
-
-minEval entropyDenseIncremental(long max, double* record, long** count, long classes, long newSize, long* T){
-        double entropy1, entropy2;
-        minEval ret;
-        long i, j;
-
-        ret.eval = DBL_MAX;
-        for(i=0; i<max; i++){
-                if(count[i][classes]==newSize or count[i][classes]==0){
-                        continue;
-                }
-                entropy1 = 0;
-                entropy2 = 0;
-                for(j=0;j<classes;j++){
-                        long l, r;
-                        l = count[i][j];
-                        r = T[j]-l;
-                        entropy1 -= ((double)l/count[i][classes])*log((double)l/count[i][classes]);
-                        entropy2 -= (double)r/(newSize-count[i][classes])*log((double)r/(newSize-count[i][classes]));
-                }
-                entropy1 = entropy1*count[i][classes]/newSize + entropy2*((newSize-count[i][classes]))/newSize;
-                if(entropy1<ret.eval){
-                        ret.eval = entropy1;
-                        ret.value = record[i];
-                }
-        }
-        return ret;
-}

sdk/Evaluation.h

@@ -17,8 +17,4 @@ minEval giniDense(long max, long size, long classes, long** rem, long* d, double
 
 minEval entropyDense(long max, long size, long classes, long** rem, long* d, double* record, long* totalT);
 
-minEval giniDenseIncremental(long max, double* record, long** count, long classes, long newSize, long* T);
-
-minEval entropyDenseIncremental(long max, double* record, long** count, long classes, long newSize, long* T);
-
 #endif

sdk/Makefile

@@ -5,7 +5,7 @@ else
 	OPT_FLAGS = -g3 -D_DEBUG -fsanitize=leak -fsanitize=address
 endif
 example:
-	$(CXX) -shared -fPIC example.cpp aquery_mem.cpp -fno-semantic-interposition -Ofast -march=native -flto --std=c++1z -o ../test.so
+	$(CXX) -shared -fPIC example.cpp aquery_mem.cpp -fno-semantic-interposition -Ofast -march=native -flto --std=c++1z -L.. -laquery -o ../test.so
 irf:
 	$(CXX) -shared -fPIC RF.cpp irf.cpp incrementalDecisionTree.cpp aquery_mem.cpp Evaluation.cpp -fno-semantic-interposition $(OPT_FLAGS) --std=c++1z -o ../libirf.so
 all: example

sdk/RF.cpp

@@ -2,7 +2,27 @@
 #include <stdlib.h>
 #include <stdio.h>
 #include <ctime>
+#include <math.h>
+#include <algorithm>
+#include <boost/math/distributions/students_t.hpp>
+#include <random>
 
+long poisson(int Lambda)
+{
+	int  k = 0;
+	long double p = 1.0;
+	long double l = exp(-Lambda);
+	srand((long)clock());
+	
+	while(p>=l)
+	{
+		double u = (double)(rand()%10000)/10000;
+		p *= u;
+		k++;
+	}
+	if (k>11)k=11;
+	return k-1;
+}
 struct DT{
         int height;
         long* featureId;
@@ -30,62 +50,126 @@ struct DT{
         long size = 0;// Size of the dataset
 };
 
-RandomForest::RandomForest(long mTree, long actTree, long rTime, int h, long feature, int* s, double forg, long maxF, long noC, Evaluation eval, long r, long rb){
+RandomForest::RandomForest(long mTree, long feature, int* s, double forg, long noC, Evaluation eval, bool b, double t){
 	srand((long)clock());
-	Rebuild = rb;
-	if(actTree<1)actTree=1;
-	noTree = actTree;
-	activeTree = actTree;
-	treePointer = 0;
-	if(mTree<actTree)mTree=activeTree;
+	bagging = b;
+	activeTree = mTree;
 	maxTree = mTree;
-	if(rTime<=0)rTime=1;
-	rotateTime = rTime;
-	timer = 0;
-	retain = r;	
+	allT = new long[mTree];
 
+	tThresh=t;
+	lastT = -2;
+	lastAll = 0;
 	long i;
-	height = h;
 	f = feature;
 	sparse = new int[f];
 	for(i=0; i<f; i++)sparse[i]=s[i];
 	forget = forg;
-	maxFeature = maxF;
 	noClasses = noC;
 	e = eval;
-
+	minF = floor(sqrt((double)f))+2;
+	if(minF>f)minF=f;
 	DTrees = (DecisionTree**)malloc(mTree*sizeof(DecisionTree*));
-	for(i=0; i<mTree; i++){
-		if(i<actTree){
-			DTrees[i] = new DecisionTree(height, f, sparse, forget, maxFeature, noClasses, e, r, rb);
-		}
-		else{
-			DTrees[i]=nullptr;
-		}
+	for(i=0; i<maxTree; i++){
+		DTrees[i] = new DecisionTree(f, sparse, forget, minF+rand()%(f+1-minF), noClasses, e);
+		DTrees[i]->isRF=true;
 	}
 }
 
 void RandomForest::fit(double** data, long* result, long size){
-	if(timer==rotateTime and maxTree!=activeTree){
-		Rotate();
-		timer=0;
-	}
-	long i, j, k;
+	long i, j, k, l;
 	double** newData;
 	long* newResult;
-	for(i=0; i<activeTree; i++){
-		newData = new double*[size];
-		newResult = new long[size];
-		for(j = 0; j<size; j++){
-			newData[j] = new double[f];
-			for(k=0; k<f; k++){
-				newData[j][k] = data[j][k];
+	long localT = 0;
+	int stale = 0;
+	if(lastT==-2){
+		lastT=-1;
+	}else{
+		for(i=0; i<maxTree; i++)allT[i] = 0;
+		for(i=0; i<size; i++){
+                	if(Test(data[i], result[i])==result[i])localT++;
+        	}
+		long localAll = size;
+		if(lastT>=0){
+			double lastSm = (double)lastT/lastAll;
+                        double localSm = (double)localT/localAll;
+                        double lastSd = sqrt(pow((1.0-lastSm),2)*lastT+pow(lastSm,2)*(lastAll-lastT)/(lastAll-1));
+                        double localSd = sqrt(pow((1.0-localSm),2)*localT+pow(localSm,2)*(localAll-localT)/(localAll-1));
+                        double v = lastAll+localAll-2;
+                        double sp = sqrt(((lastAll-1) * lastSd * lastSd + (localAll-1) * localSd * localSd) / v);
+                        double q;
+			double t = lastSm-localSm;
+			if(sp==0){q = 1;}
+			else{
+				t = t/(sp*sqrt(1.0/lastAll+1.0/localAll));
+                        	boost::math::students_t dist(v);
+                        	double c = cdf(dist, t);
+				q = cdf(complement(dist, fabs(t)));
 			}
-			newResult[j] = result[j];
+                        if(q<=tThresh){
+				lastT += localT;
+				lastAll += localAll;
+			}else if(t<0){
+				lastT = localT;
+				lastAll = localAll;
+			}else{
+				double newAcc = (double)localT/localAll;
+				double lastAcc= (double)lastT/lastAll;
+				stale = floor((newAcc-lastAcc)/(lastAcc)*maxTree);
+				lastT = localT;
+				lastAll = localAll;
+			}
+		}else{
+			lastT = localT;
+			lastAll = localAll;
 		}
-		DTrees[(i+treePointer)%maxTree]->fit(newData, newResult, size);
 	}
-	timer++;
+	Rotate(stale);
+	for(i=0; i<maxTree; i++){
+		long times;
+	       	if(bagging)times = poisson(6);
+		else times=1;
+		if(times==0)continue;
+		newData = (double**)malloc(sizeof(double*)*size*times);
+		newResult = (long*)malloc(sizeof(long)*size*times);
+		long c = 0;
+		for(j = 0; j<size*times; j++){
+			long jj;
+		       	if(bagging) jj = rand()%size;
+			else jj=j;
+			newData[j] = (double*)malloc((f+1)*sizeof(double));
+			for(l=0; l<f; l++){
+				newData[j][l] = data[jj][l];
+			}
+			newData[j][f] = 0;
+			newResult[j] = result[jj];
+		}
+		DTrees[i]->fit(newData, newResult, size*times);
+	}
+	/*for(i=0; i<maxTree; i++){
+		//backupTrees[i]->retain = 10*size;
+		//if(backupTrees[i]==nullptr) continue;
+		long times;
+		//times = poisson(posMean);
+		//if(times==0)continue;
+		times=1;
+		newData = (double**)malloc(sizeof(double*)*size*times);
+		newResult = (long*)malloc(sizeof(long)*size*times);
+		long c = 0;
+		for(j = 0; j<size; j++){
+			long jj = rand()%size;
+			jj=j;
+			for(k=0; k<times; k++){
+				newData[j*times+k] = (double*)malloc((f+1)*sizeof(double));
+				for(l=0; l<f; l++){
+					newData[j*times+k][l] = data[jj][l];
+				}
+				newData[j*times+k][f] = 0;
+				newResult[j*times+k] = result[jj];
+			}
+		}
+		backupTrees[i]->fit(newData, newResult, size*times);
+	}*/
 }
 
 long* RandomForest::fitThenPredict(double** trainData, long* trainResult, long trainSize, double** testData, long testSize){
@@ -97,36 +181,80 @@ long* RandomForest::fitThenPredict(double** trainData, long* trainResult, long t
         return testResult;
 }
 
-void RandomForest::Rotate(){
-	if(noTree==maxTree){
-		DTrees[(treePointer+activeTree)%maxTree]->Free();
-		delete DTrees[(treePointer+activeTree)%maxTree];
-	}else{
-		noTree++;
+void RandomForest::Rotate(long stale){
+	long i, j, k;
+	long minIndex = -1;
+	if(stale>=0)return;
+	else{
+		stale = std::min(stale, maxTree);
+		stale*=-1;
+		while(stale>0){
+			long currentMin = 2147483647;
+			for(i = 0; i<maxTree; i++){
+				if(allT[i]<currentMin){
+					currentMin=allT[i];
+					minIndex=i;
+				}
+			}
+			stale--;
+			if(minIndex<0)break;
+			allT[minIndex] = 2147483647; 
+				double** newData;
+				long* newResult;
+				long size = 0;
+				long lastT2 = 0;
+				size = DTrees[minIndex]->DTree->size;
+				newData = (double**)malloc(sizeof(double*)*size);
+				newResult = (long*)malloc(sizeof(long)*size);
+				for(j = 0; j<size; j++){
+					newData[j] = (double*)malloc(sizeof(double)*(f+1));
+               				for(k=0; k<f; k++){
+                       				newData[j][k] = DTrees[minIndex]->DTree->dataRecord[j][k];
+                			}
+					newData[j][f] = 0;
+                			newResult[j] = DTrees[minIndex]->DTree->resultRecord[j];
+        			}
+				DTrees[minIndex]->Stablelize();
+				DTrees[minIndex]->Free();
+				delete DTrees[minIndex];
+				DTrees[minIndex] = new DecisionTree(f, sparse, forget, minF+rand()%(f+1-minF), noClasses, e);
+				DTrees[minIndex]->isRF=true;
+				DTrees[minIndex]->fit(newData, newResult, size);
+				for(j=0; j<size; j++){
+					if(DTrees[minIndex]->Test(newData[j], DTrees[minIndex]->DTree)==newResult[j])lastT2++;
+				}
+				DTrees[minIndex]->lastAll=size;
+				DTrees[minIndex]->lastT=lastT2;
+		}	
 	}
-	DTrees[(treePointer+activeTree)%maxTree] = new DecisionTree(height, f, sparse, forget, maxFeature, noClasses, e, retain, Rebuild);
-	long size = DTrees[(treePointer+activeTree-1)%maxTree]->DTree->size;
-	double** newData = new double*[size];
-	long* newResult = new long[size];
-	for(long j = 0; j<size; j++){
-		newData[j] = new double[f];
-                for(long k=0; k<f; k++){
-                        newData[j][k] = DTrees[(treePointer+activeTree-1)%maxTree]->DTree->dataRecord[j][k];
-                }
-                newResult[j] = DTrees[(treePointer+activeTree-1)%maxTree]->DTree->resultRecord[j];
-        }
-
-	DTrees[(treePointer+activeTree)%maxTree]->fit(newData, newResult, size);
-	DTrees[treePointer]->Stablelize();
-	if(++treePointer==maxTree)treePointer=0;
 }
 
 
+long RandomForest::Test(double* data, long result){
+	long i;
+	long predict[noClasses];
+	for(i=0; i<noClasses; i++){
+		predict[i]=0;
+	}
+	for(i=0; i<maxTree; i++){
+		long tmp = DTrees[i]->Test(data, DTrees[i]->DTree);
+		predict[tmp]++;
+		if(tmp==result)allT[i]++;
+	}
+	
+	long ret = 0;
+	for(i=1; i<noClasses; i++){
+		if(predict[i]>predict[ret])ret = i;
+	}
+
+	return ret;
+}
+
 long RandomForest::Test(double* data){
 	long i;
 	long predict[noClasses];
 	for(i=0; i<noClasses; i++)predict[i]=0;
-	for(i=0; i<noTree; i++){
+	for(i=0; i<maxTree; i++){
 		predict[DTrees[i]->Test(data, DTrees[i]->DTree)]++;
 	}
 	

sdk/RF.h

@@ -14,33 +14,34 @@ struct DT;
 class RandomForest{
 public:
 
-long noTree;
 long maxTree;
 long activeTree;
-long treePointer;
-long rotateTime;
-long timer;
-long retain;
-DecisionTree** DTrees = nullptr;
+long* allT;
+double tThresh;
+DecisionTree** DTrees;
+DecisionTree** backupTrees;
 
 long height;
-long Rebuild;
+bool bagging;
 long f;
 int* sparse;
 double forget;
-long maxFeature;
 long noClasses;
 Evaluation e;
+long lastT;
+long lastAll;
+int minF;
 
-
-RandomForest(long maxTree, long activeTree, long rotateTime, int height, long f, int* sparse, double forget, long maxFeature=0, long noClasses=2, Evaluation e=Evaluation::gini, long r=-1, long rb=2147483647);
+RandomForest(long maxTree, long f, int* sparse, double forget, long noClasses=2, Evaluation e=Evaluation::entropy, bool b=false, double tThresh=0.05);
 
 void fit(double** data, long* result, long size);
 
 long* fitThenPredict(double** trainData, long* trainResult, long trainSize, double** testData, long testSize);
 
-void Rotate();
+void Rotate(long stale);
 
 long Test(double* data);
+
+long Test(double* data, long result);
 };
 #endif

sdk/aquery.h

@@ -84,7 +84,9 @@ __AQEXPORT__(void) init_session(Context* cxt);
 #ifdef _WIN32
 #include <cstring>
 #else 
-void* memcpy(void*, const void*, unsigned long long);
+namespace std {
+	void* memcpy(void*, const void*, unsigned long long);
+}
 #endif
 
 struct vectortype_storage{
@@ -95,7 +97,7 @@ struct vectortype_storage{
 	vectortype_storage() = default;
 	template <class Ty, template <typename> class VT>
 	vectortype_storage(const VT<Ty>& vt) {
-		memcpy(this, &vt, sizeof(vectortype_storage));
+		std::memcpy(this, &vt, sizeof(vectortype_storage));
 	}
 };
 struct ColRef_storage {
@@ -108,7 +110,7 @@ struct ColRef_storage {
 	ColRef_storage() = default;
 	template <class Ty, template <typename> class VT>
 	ColRef_storage(const VT<Ty>& vt) {
-		memcpy(this, &vt, sizeof(ColRef_storage));
+		std::memcpy(this, &vt, sizeof(ColRef_storage));
 	}
 };
 #endif

sdk/example.cpp

@@ -1,6 +1,6 @@
-#include "aquery.h"
 // __AQ_NO_SESSION__
 #include "../server/table.h"
+#include "aquery.h"
 
 __AQEXPORT__(ColRef_storage) mulvec(int a, ColRef<float> b){
     return a * b;

sdk/irf.cpp

@@ -1,63 +1,93 @@
 #include "DecisionTree.h"
-#include "aquery.h"
+#include "RF.h"
+
 // __AQ_NO_SESSION__
 #include "../server/table.h"
 
-DecisionTree* dt = nullptr;
+#include "aquery.h"
 
-__AQEXPORT__(bool) newtree(int height, long f, ColRef<int> sparse, double forget, long maxf, long noclasses, Evaluation e, long r, long rb){
-	if(sparse.size!=f)return 0;
-	int* issparse = (int*)malloc(f*sizeof(int));
-	for(long i=0; i<f; i++){
-		issparse[i] = sparse.container[i];
-	}
-	if(maxf<0)maxf=f;
-	dt = new DecisionTree(height, f, issparse, forget, maxf, noclasses, e, r, rb);
-	return 1;
+#include "./server/gc.h"
+__AQEXPORT__(void) __AQ_Init_GC__(Context* cxt) {
+    GC::gc_handle = static_cast<GC*>(cxt->gc);
+    GC::scratch_space = nullptr;
 }
 
-__AQEXPORT__(bool) fit(ColRef<ColRef<double>> X, ColRef<int> y){
-	if(X.size != y.size)return 0;
-	double** data = (double**)malloc(X.size*sizeof(double*));
-	long* result = (long*)malloc(y.size*sizeof(long));
-	for(long i=0; i<X.size; i++){
-		data[i] = X.container[i].container;
-		result[i] = y.container[i];
-	}
-	data[pt] = (double*)malloc(X.size*sizeof(double));
-	for(j=0; j<X.size; j++){
-		data[pt][j]=X.container[j];
-	}
-	result[pt] = y;
-	pt ++;
-	return 1;
-}
-__AQEXPORT__(bool) fit(vector_type<vector_type<double>> v, vector_type<long> res){
-	double** data = (double**)malloc(v.size*sizeof(double*));
-	for(int i = 0; i < v.size; ++i)
-		data[i] = v.container[i].container;
-	dt->fit(data, res.container, v.size);
+DecisionTree *dt = nullptr;
+RandomForest *rf = nullptr;
+
+__AQEXPORT__(bool)
+newtree(int height, long f, ColRef<int> X, double forget, long maxf, long noclasses, Evaluation e, long r, long rb)
+{
+	if (X.size != f)
+		return false;
+	int *X_cpy = (int *)malloc(f * sizeof(int));
+	
+	memcpy(X_cpy, X.container, f);
+
+	if (maxf < 0)
+		maxf = f;
+	dt = new DecisionTree(f, X_cpy, forget, maxf, noclasses, e);
+	rf = new RandomForest(height, f, X_cpy, forget, noclasses, e);
 	return true;
 }
 
-__AQEXPORT__(vectortype_cstorage) predict(vector_type<vector_type<double>> v){
-	int* result = (int*)malloc(v.size*sizeof(int));
-	
-	for(long i=0; i<v.size; i++){
-		result[i]=dt->Test(v.container[i].container, dt->DTree);
-		//printf("%d ", result[i]);
-	}
-	auto container = (vector_type<int>*)malloc(sizeof(vector_type<int>));
-	container->size = v.size;
-	container->capacity = 0;
-	container->container = result;
-	// container->out(10);
-	// ColRef<vector_type<int>>* col = (ColRef<vector_type<int>>*)malloc(sizeof(ColRef<vector_type<int>>));
-	auto ret = vectortype_cstorage{.container = container, .size = 1, .capacity = 0};
-	// col->initfrom(ret, "sibal");
-	// print(*col);
-	return ret;
-	//return true;
+// size_t pt = 0;
+// __AQEXPORT__(bool) fit(ColRef<ColRef<double>> X, ColRef<int> y){
+// 	if(X.size != y.size)return 0;
+// 	double** data = (double**)malloc(X.size*sizeof(double*));
+// 	long* result = (long*)malloc(y.size*sizeof(long));
+// 	for(long i=0; i<X.size; i++){
+// 		data[i] = X.container[i].container;
+// 		result[i] = y.container[i];
+// 	}
+// 	data[pt] = (double*)malloc(X.size*sizeof(double));
+// 	for(uint32_t j=0; j<X.size; j++){
+// 		data[pt][j]=X.container[j];
+// 	}
+// 	result[pt] = y;
+// 	pt ++;
+// 	return 1;
+// }
+
+__AQEXPORT__(bool)
+fit_inc(vector_type<vector_type<double>> v, vector_type<long> res)
+{
+	static uint32_t last_offset = 0;
+	double **data = (double **)malloc(v.size * sizeof(double *));
+	if(last_offset >= v.size) 
+		last_offset = 0;
+	for (int i = last_offset; i < v.size; ++i)
+		data[i] = v.container[i].container;
+	rf->fit(data, res.container, v.size);
+	free(data);
+	return true;
 }
 
 
+__AQEXPORT__(bool)
+fit(vector_type<vector_type<double>> v, vector_type<long> res)
+{
+	double **data = (double **)malloc(v.size * sizeof(double *));
+	for (int i = 0; i < v.size; ++i)
+		data[i] = v.container[i].container;
+	// dt->fit(data, res.container, v.size);
+	rf->fit(data, res.container, v.size);
+	return true;
+}
+
+
+__AQEXPORT__(vectortype_cstorage)
+predict(vector_type<vector_type<double>> v)
+{
+	int *result = (int *)malloc(v.size * sizeof(int));
+
+	for (uint32_t i = 0; i < v.size; i++)
+		//result[i] = dt->Test(v.container[i].container, dt->DTree);
+		result[i] = int(rf->Test(v[i].container));
+	auto container = (vector_type<int> *)malloc(sizeof(vector_type<int>));
+	container->size = v.size;
+	container->capacity = 0;
+	container->container = result;
+	auto ret = vectortype_cstorage{.container = container, .size = 1, .capacity = 0};
+	return ret;
+}

server/aggregations.h

@@ -1,5 +1,6 @@
 #pragma once
 #include "types.h"
+#include "gc.h"
 #include <utility>
 #include <limits>
 #include <deque>
@@ -12,7 +13,7 @@ size_t count(const VT<T>& v) {
 }
 
 template <class T>
-constexpr static inline size_t count(const T&) { return 1; }
+constexpr static size_t count(const T&) { return 1; }
 
 // TODO: Specializations for dt/str/none
 template<class T, template<typename ...> class VT>
@@ -29,14 +30,19 @@ double avg(const VT<T>& v) {
 	return (sum<T>(v) / static_cast<double>(v.size));
 }
 
+template<class T, template<typename ...> class VT, class Ret>
+void sqrt(const VT<T>& v, Ret& ret) {
+	for (uint32_t i = 0; i < v.size; ++i) 
+		ret[i] = sqrt(v[i]);
+}
+
 template<class T, template<typename ...> class VT>
 VT<double> sqrt(const VT<T>& v) {
-	VT<double> ret{ v.size };
-	for (uint32_t i = 0; i < v.size; ++i) {
-		ret[i] = sqrt(v[i]);
-	}
+	VT<double> ret(v.size);
+	sqrt(v, ret);
 	return ret;
 }
+
 template <class T>
 T truncate(const T& v, const uint32_t precision) {
 	auto multiplier = pow(10, precision);
@@ -52,7 +58,7 @@ VT<T> truncate(const VT<T>& v, const uint32_t precision) {
 		return v.subvec_memcpy();
 	auto multiplier = pow(10, precision);
 	auto max_truncate = std::numeric_limits<T>::max()/multiplier;
-	VT<T> ret{ v.size };
+	VT<T> ret(v.size);
 	for (uint32_t i = 0; i < v.size; ++i) { // round or trunc??
 		ret[i] = v[i] < max_truncate ? round(v[i] * multiplier)/multiplier : v[i];
 	}
@@ -73,109 +79,153 @@ T min(const VT<T>& v) {
 		min_v = min_v < _v ? min_v : _v;
 	return min_v;
 }
-template<class T, template<typename ...> class VT>
-decayed_t<VT, T> mins(const VT<T>& arr) {
+
+// simplify this using a template std::binary_function<T, T, bool> = std::less;
+template<class T, template<typename ...> class VT, class Ret>
+void mins(const VT<T>& arr, Ret& ret) {
 	const uint32_t& len = arr.size;
-	std::deque<std::pair<T, uint32_t>> cache;
-	decayed_t<VT, T> ret(len);
 	T min = std::numeric_limits<T>::max();
 	for (int i = 0; i < len; ++i) {
 		if (arr[i] < min)
 			min = arr[i];
 		ret[i] = min;
 	}
+}
+
+template<class T, template<typename ...> class VT>
+decayed_t<VT, T> mins(const VT<T>& arr) {
+	decayed_t<VT, T> ret(arr.size);
+	mins(arr, ret);
 	return ret;
 }
-template<class T, template<typename ...> class VT>
-decayed_t<VT, T> maxs(const VT<T>& arr) {
+
+template<class T, template<typename ...> class VT, class Ret>
+void maxs(const VT<T>& arr, Ret& ret) {
 	const uint32_t& len = arr.size;
-	decayed_t<VT, T> ret(len);
 	T max = std::numeric_limits<T>::min();
 	for (int i = 0; i < len; ++i) {
 		if (arr[i] > max)
 			max = arr[i];
 		ret[i] = max;
 	}
-	return ret;
 }
 
 template<class T, template<typename ...> class VT>
-decayed_t<VT, T> minw(uint32_t w, const VT<T>& arr) {
+decayed_t<VT, T> maxs(const VT<T>& arr) {
+	decayed_t<VT, T> ret(arr.size);
+	maxs(arr, ret);
+	return ret;
+}
+
+template<class T, template<typename ...> class VT, class Ret>
+void minw(uint32_t w, const VT<T>& arr, Ret& ret) {
 	const uint32_t& len = arr.size;
-	decayed_t<VT, T> ret{ len };
 	std::deque<std::pair<T, uint32_t>> cache;
 	for (int i = 0; i < len; ++i) {
 		if (!cache.empty() && cache.front().second == i - w) cache.pop_front();
+		
 		while (!cache.empty() && cache.back().first > arr[i]) cache.pop_back();
 		cache.push_back({ arr[i], i });
 		ret[i] = cache.front().first;
 	}
-	return ret;
 }
 
 template<class T, template<typename ...> class VT>
-decayed_t<VT, T> maxw(uint32_t w, const VT<T>& arr) {
+decayed_t<VT, T> minw(uint32_t w, const VT<T>& arr) {
+	decayed_t<VT, T> ret(arr.size);
+	minw(w, arr, ret);
+	return ret;
+}
+
+template<class T, template<typename ...> class VT, class Ret>
+void maxw(uint32_t w, const VT<T>& arr, Ret& ret) {
 	const uint32_t& len = arr.size;
-	decayed_t<VT, T> ret(len);
 	std::deque<std::pair<T, uint32_t>> cache;
 	for (int i = 0; i < len; ++i) {
 		if (!cache.empty() && cache.front().second == i - w) cache.pop_front();
-		while (!cache.empty() && cache.back().first > arr[i]) cache.pop_back();
+		while (!cache.empty() && cache.back().first < arr[i]) cache.pop_back();
 		cache.push_back({ arr[i], i });
-		arr[i] = cache.front().first;
+		ret[i] = cache.front().first;
 	}
-	return ret;
 }
 
 template<class T, template<typename ...> class VT>
-decayed_t<VT, types::GetFPType<T>> ratiow(uint32_t w, const VT<T>& arr) {
+inline decayed_t<VT, T> maxw(uint32_t w, const VT<T>& arr) {
+	decayed_t<VT, T> ret(arr.size);
+	maxw(w, arr, ret);
+	return ret;
+}
+
+template<class T, template<typename ...> class VT, class Ret>
+void ratiow(uint32_t w, const VT<T>& arr, Ret& ret) {
 	typedef std::decay_t<types::GetFPType<T>> FPType;
 	uint32_t len = arr.size;
 	if (arr.size <= w) 
 		len = 1;
 	w = w > len ? len : w;
-	decayed_t<VT, FPType> ret(arr.size);
 	ret[0] = 0;
 	for (uint32_t i = 0; i < w; ++i) 
 		ret[i] = arr[i] / (FPType)arr[0];
 	for (uint32_t i = w; i < arr.size; ++i) 
 		ret[i] = arr[i] / (FPType) arr[i - w];
+}
+
+template<class T, template<typename ...> class VT>
+inline decayed_t<VT, types::GetFPType<T>> ratiow(uint32_t w, const VT<T>& arr) {
+	typedef std::decay_t<types::GetFPType<T>> FPType;
+	decayed_t<VT, FPType> ret(arr.size);
+	ratiow(w, arr, ret);
 	return ret;
 }
 
 template<class T, template<typename ...> class VT>
-decayed_t<VT, types::GetFPType<T>> ratios(const VT<T>& arr) { 
+inline decayed_t<VT, types::GetFPType<T>> ratios(const VT<T>& arr) { 
 	return ratiow(1, arr);
 }
 
-template<class T, template<typename ...> class VT>
-decayed_t<VT, types::GetLongType<T>> sums(const VT<T>& arr) {
+template<class T, template<typename ...> class VT, class Ret>
+inline void ratios(const VT<T>& arr, Ret& ret) { 
+	return ratiow(1, arr, ret);
+}
+
+template<class T, template<typename ...> class VT, class Ret>
+void sums(const VT<T>& arr, Ret& ret) {
 	const uint32_t& len = arr.size;
-	decayed_t<VT, types::GetLongType<T>> ret(len);
 	uint32_t i = 0;
 	if (len) ret[i++] = arr[0];
 	for (; i < len; ++i)
 		ret[i] = ret[i - 1] + arr[i];
-	return ret;
 }
 
 template<class T, template<typename ...> class VT>
-decayed_t<VT, types::GetFPType<types::GetLongType<T>>> avgs(const VT<T>& arr) {
+inline decayed_t<VT, types::GetLongType<T>> sums(const VT<T>& arr) {
+	decayed_t<VT, types::GetLongType<T>> ret(arr.size);
+	sums(arr, ret);
+	return ret;
+}
+
+template<class T, template<typename ...> class VT, class Ret>
+void avgs(const VT<T>& arr, Ret& ret) {
 	const uint32_t& len = arr.size;
 	typedef types::GetFPType<types::GetLongType<T>> FPType;
-	decayed_t<VT, FPType> ret(len);
 	uint32_t i = 0;
 	types::GetLongType<T> s;
 	if (len) s = ret[i++] = arr[0];
 	for (; i < len; ++i)
 		ret[i] = (s += arr[i]) / (FPType)(i + 1);
-	return ret;
 }
 
 template<class T, template<typename ...> class VT>
-decayed_t<VT, types::GetLongType<T>> sumw(uint32_t w, const VT<T>& arr) {
+inline decayed_t<VT, types::GetFPType<types::GetLongType<T>>> avgs(const VT<T>& arr) {
+	typedef types::GetFPType<types::GetLongType<T>> FPType;
+	decayed_t<VT, FPType> ret(arr.size);
+	avgs(arr, ret);
+	return ret;
+}
+
+template<class T, template<typename ...> class VT, class Ret>
+void sumw(uint32_t w, const VT<T>& arr, Ret& ret) {
 	const uint32_t& len = arr.size;
-	decayed_t<VT, types::GetLongType<T>> ret(len);
 	uint32_t i = 0;
 	w = w > len ? len : w;
 	if (len) ret[i++] = arr[0];
@@ -183,30 +233,43 @@ decayed_t<VT, types::GetLongType<T>> sumw(uint32_t w, const VT<T>& arr) {
 		ret[i] = ret[i - 1] + arr[i];
 	for (; i < len; ++i)
 		ret[i] = ret[i - 1] + arr[i] - arr[i - w];
-	return ret;
 }
 
 template<class T, template<typename ...> class VT>
-decayed_t<VT, types::GetFPType<types::GetLongType<T>>> avgw(uint32_t w, const VT<T>& arr) {
+decayed_t<VT, types::GetLongType<T>> sumw(uint32_t w, const VT<T>& arr) {
+	decayed_t<VT, types::GetLongType<T>> ret(arr.size);
+	sumw(w, arr, ret);
+	return ret;
+}
+
+template<class T, template<typename ...> class VT, class Ret>
+void avgw(uint32_t w, const VT<T>& arr, Ret& ret) {
 	typedef types::GetFPType<types::GetLongType<T>> FPType;
 	const uint32_t& len = arr.size;
-	decayed_t<VT, FPType> ret(len);
 	uint32_t i = 0;
 	types::GetLongType<T> s{};
 	w = w > len ? len : w;
-	if (len)	s = ret[i++] = arr[0];
+	if (len) s = ret[i++] = arr[0];
 	for (; i < w; ++i)
 		ret[i] = (s += arr[i]) / (FPType)(i + 1);
 	for (; i < len; ++i)
 		ret[i] = ret[i - 1] + (arr[i] - arr[i - w]) / (FPType)w;
+}
+
+template<class T, template<typename ...> class VT>
+inline decayed_t<VT, types::GetFPType<types::GetLongType<T>>> avgw(uint32_t w, const VT<T>& arr) {
+	typedef types::GetFPType<types::GetLongType<T>> FPType;
+	const uint32_t& len = arr.size;
+	decayed_t<VT, FPType> ret(len);
+	avgw(w, arr, ret);
 	return ret;
 }
 
-template<class T, template<typename ...> class VT, bool sd = false>
-decayed_t<VT, types::GetFPType<types::GetLongType<T>>> varw(uint32_t w, const VT<T>& arr) {
+template<class T, template<typename ...> class VT, class Ret, bool sd = false>
+void varw(uint32_t w, const VT<T>& arr, 
+	Ret& ret) {
 	using FPType = types::GetFPType<types::GetLongType<T>>;
 	const uint32_t& len = arr.size;
-	decayed_t<VT, FPType> ret(len);
 	uint32_t i = 0;
 	types::GetLongType<T> s{};
 	w = w > len ? len : w;
@@ -238,7 +301,14 @@ decayed_t<VT, types::GetFPType<types::GetLongType<T>>> varw(uint32_t w, const VT
 	if constexpr(sd) 
 		if(i)
 			ret[i-1] = sqrt(ret[i-1]);
-	
+}
+
+
+template<class T, template<typename ...> class VT, bool sd = false>
+inline decayed_t<VT, types::GetFPType<types::GetLongType<T>>> varw(uint32_t w, const VT<T>& arr) {
+	using FPType = types::GetFPType<types::GetLongType<T>>;
+	decayed_t<VT, FPType> ret(arr.size);
+	varw<T, VT, decayed_t<VT, types::GetFPType<types::GetLongType<T>>>, sd>(w, arr, ret);
 	return ret;
 }
 
@@ -260,11 +330,10 @@ types::GetFPType<types::GetLongType<decays<T>>> var(const VT<T>& arr) {
 	return (ssq - s * s / (FPType)(len + 1)) / (FPType)(len + 1);
 }
 
-template<class T, template<typename ...> class VT, bool sd = false>
-decayed_t<VT, types::GetFPType<types::GetLongType<T>>> vars(const VT<T>& arr) {
+template<class T, template<typename ...> class VT, class Ret, bool sd = false>
+void vars(const VT<T>& arr, Ret& ret) {
 	typedef types::GetFPType<types::GetLongType<T>> FPType;
 	const uint32_t& len = arr.size;
-	decayed_t<VT, FPType> ret(len);
 	uint32_t i = 0;
 	types::GetLongType<T> s{};
 	FPType MnX{};
@@ -284,70 +353,103 @@ decayed_t<VT, types::GetFPType<types::GetLongType<T>>> vars(const VT<T>& arr) {
 		ret[i] = MnX / (FPType)(i + 1);
 		if constexpr(sd) ret[i] = sqrt(ret[i]);
 	}
+}
+
+template<class T, template<typename ...> class VT, bool sd = false>
+inline decayed_t<VT, types::GetFPType<types::GetLongType<T>>> vars(const VT<T>& arr) {
+	typedef types::GetFPType<types::GetLongType<T>> FPType;
+	decayed_t<VT, FPType> ret(arr.size);
+	vars<T, VT, decayed_t<VT, types::GetFPType<types::GetLongType<T>>>, sd>(arr, ret);
 	return ret;
 }
+
 template<class T, template<typename ...> class VT>
-types::GetFPType<types::GetLongType<decays<T>>> stddev(const VT<T>& arr) {
+inline types::GetFPType<types::GetLongType<decays<T>>> stddev(const VT<T>& arr) {
 	return sqrt(var(arr));
 }
+
 template<class T, template<typename ...> class VT>
-decayed_t<VT, types::GetFPType<types::GetLongType<T>>> stddevs(const VT<T>& arr) {
+inline decayed_t<VT, types::GetFPType<types::GetLongType<T>>> stddevs(const VT<T>& arr) {
 	return vars<T, VT, true>(arr);
 }
+
 template<class T, template<typename ...> class VT>
-decayed_t<VT, types::GetFPType<types::GetLongType<T>>> stddevw(uint32_t w, const VT<T>& arr) {
+inline decayed_t<VT, types::GetFPType<types::GetLongType<T>>> stddevw(uint32_t w, const VT<T>& arr) {
 	return varw<T, VT, true>(w, arr);
 }
+
+template<class T, template<typename ...> class VT, class Ret>
+inline auto stddevs(const VT<T>& arr, Ret& ret) {
+	return vars<T, VT, Ret, true>(arr, ret);
+}
+
+template<class T, template<typename ...> class VT, class Ret>
+inline auto stddevw(uint32_t w, const VT<T>& arr, Ret& ret) {
+	return varw<T, VT, Ret, true>(w, arr, ret);
+}
+
+
 // use getSignedType
-template<class T, template<typename ...> class VT>
-decayed_t<VT, T> deltas(const VT<T>& arr) {
+template<class T, template<typename ...> class VT, class Ret>
+void deltas(const VT<T>& arr, Ret& ret) {
 	const uint32_t& len = arr.size;
-	decayed_t<VT, T> ret(len);
 	uint32_t i = 0;
 	if (len) ret[i++] = 0;
 	for (; i < len; ++i)
 		ret[i] = arr[i] - arr[i - 1];
-	return ret;
 }
 
 template<class T, template<typename ...> class VT>
-decayed_t<VT, T> prev(const VT<T>& arr) {
+inline decayed_t<VT, T> deltas(const VT<T>& arr) {
+	decayed_t<VT, T> ret(arr.size);
+	deltas(arr, ret);
+	return ret;
+}
+
+template<class T, template<typename ...> class VT, class Ret>
+void prev(const VT<T>& arr, Ret& ret) {
 	const uint32_t& len = arr.size;
-	decayed_t<VT, T> ret(len);
 	uint32_t i = 0;
 	if (len) ret[i++] = arr[0];
 	for (; i < len; ++i)
 		ret[i] = arr[i - 1];
-	return ret;
 }
 
 template<class T, template<typename ...> class VT>
-decayed_t<VT, T> aggnext(const VT<T>& arr) {
+inline decayed_t<VT, T> prev(const VT<T>& arr) {
+	decayed_t<VT, T> ret(arr.size);
+	prev(arr, ret);
+	return ret;
+}
+
+template<class T, template<typename ...> class VT, class Ret>
+void aggnext(const VT<T>& arr, Ret& ret) {
 	const uint32_t& len = arr.size;
-	decayed_t<VT, T> ret(len);
 	uint32_t i = 1;
 	for (; i < len; ++i)
 		ret[i - 1] = arr[i];
 	if (len > 0) ret[len - 1] = arr[len - 1];
+}
+
+template<class T, template<typename ...> class VT>
+inline decayed_t<VT, T> aggnext(const VT<T>& arr) {
+	decayed_t<VT, T> ret(arr.size);
+	aggnext(arr, ret);
 	return ret;
 }
 
 template<class T, template<typename ...> class VT>
 T last(const VT<T>& arr) {
 	if (!arr.size) return 0;
-	const uint32_t& len = arr.size;
 	return arr[arr.size - 1];
 }
 
 template<class T, template<typename ...> class VT>
 T first(const VT<T>& arr) {
 	if (!arr.size) return 0;
-	const uint32_t& len = arr.size;
 	return arr[0];
 }
 
-
-
 #define __DEFAULT_AGGREGATE_FUNCTION__(NAME, RET) \
 template <class T> constexpr T NAME(const T& v) { return RET; }
 

server/cpp.hint

@@ -4,3 +4,4 @@
 #define Ops(o) template<typename T> vector_type<typename types::Coercion<_Ty, T>::type> operator##o(const vector_type<T>& r) { [[likely]] if (r.size == size) { return add(r); } else if (r.size == 1 || size == 1) { const bool lr = size == 1; const uint32_t& _size = lr ? r.size : size; const auto& _container = lr ? r.container : container; const auto& scalar = *(lr ? container : r.container); vector_type<typename types::Coercion<_Ty, T>::type> ret(_size); for (int i = 0; i < _size; ++i) ret[i] = _container[i] o scalar; return ret; } }
 #define Op(o, x) template<typename T> vector_type<typename types::Coercion<_Ty, T>::type> inline x(const vector_type<T>& r) { vector_type<typename types::Coercion<_Ty, T>::type> ret(size); for (int i = 0; i < size; ++i) ret[i] = container[i] o r[i]; return ret; }
 #define _Make_Ops(M) M(+, add) M(-, minus) M(*, multi) M(/, div) M(%, mod) M(&, and) M(|, or) M(^, xor) 
+#define QUERY_DECLSPEC

server/gc.h

@@ -1,8 +1,46 @@
-#ifndef __AQ_USE_THREADEDGC__
+#pragma once
 #include <atomic>
-class GC {
-private:;
+#ifndef QUERY_DECLSPEC
+#define QUERY_DECLSPEC
+#endif
 
+class QUERY_DECLSPEC ScratchSpace {
+public:
+	void* ret;
+	char* scratchspace;
+	size_t ptr;
+	size_t cnt;
+	size_t capacity;
+	size_t initial_capacity;
+	void* temp_memory_fractions;
+	
+	//uint8_t status; 
+	// record maximum size
+	constexpr static uint8_t Grow = 0x1;
+	// no worry about overflow
+	constexpr static uint8_t Use = 0x0; 
+
+	void init(size_t initial_capacity);
+
+	// apply for memory
+	void* alloc(uint32_t sz);
+
+	void register_ret(void* ret);
+
+	// reorganize memory space
+	void release();
+
+	// reset status of the scratch space 
+	void reset();
+
+	// reset scratch space to initial capacity.
+	void cleanup();
+};
+
+
+#ifndef __AQ_USE_THREADEDGC__
+class QUERY_DECLSPEC GC {
+private:
 	size_t max_slots, 
 		   interval, forced_clean, 
 		   forceclean_timer = 0;
@@ -16,6 +54,7 @@ private:;
 	std::atomic<uint32_t> alive_cnt;
 	std::atomic<uint64_t> current_size;
 	volatile bool lock;
+	using gc_deallocator_t = void (*)(void*);
 	// maybe use volatile std::thread::id instead
 protected:
 	void acquire_lock();
@@ -26,26 +65,42 @@ protected:
 	void terminate_daemon();
 
 public:
-	void reg(void* v, uint32_t sz = 1, 
+	ScratchSpace scratch;
+	void reg(void* v, uint32_t sz = 0xffffffff, 
 			void(*f)(void*) = free
 		);
 
+	uint32_t get_threshold() const {
+		return threshould;
+	}
+
 	GC(
 		uint64_t max_size = 0xfffffff, uint32_t max_slots = 4096, 
 		uint32_t interval = 10000, uint32_t forced_clean = 1000000,
-		uint32_t threshould = 64 //one seconds
+		uint32_t threshould = 64, //one seconds
+		uint32_t scratch_sz = 0x1000000 // 16 MB
 	) : max_size(max_size), max_slots(max_slots), 
 		interval(interval), forced_clean(forced_clean), 
 		threshould(threshould) {
 
 		start_deamon();
 		GC::gc_handle = this;
+		this->scratch.init(1);
 	} // 256 MB
 
 	~GC(){
 		terminate_daemon();
+		scratch.cleanup();
 	}
+
 	static GC* gc_handle;
+	static ScratchSpace *scratch_space;
+	template <class T>
+	static inline gc_deallocator_t _delete(T*) {
+		return [](void* v){
+			delete (T*)v;
+		};
+	} 
     constexpr static void(*_free) (void*) = free;
 };
 

server/hasher.h

@@ -3,18 +3,28 @@
 #include <type_traits>
 #include <tuple>
 #include <functional>
+#include <string_view>
 #include "types.h"
+// #include "robin_hood.h"
+#include "unordered_dense.h"
+template<typename Key, typename Val>
+using aq_map = ankerl::unordered_dense::map<Key, Val>;
+
+template<typename Key>
+using aq_set = ankerl::unordered_dense::set<Key>;
+
 // only works for 64 bit systems
-constexpr size_t _FNV_offset_basis = 14695981039346656037ULL;
-constexpr size_t _FNV_prime = 1099511628211ULL;
+namespace hasher_consts{
+	constexpr size_t _FNV_offset_basis = 14695981039346656037ULL;
+	constexpr size_t _FNV_prime = 1099511628211ULL;
+}
 
 inline size_t append_bytes(const unsigned char* _First) noexcept {
-	size_t _Val = _FNV_offset_basis;
+	size_t _Val = hasher_consts::_FNV_offset_basis;
 	for (; *_First; ++_First) {
 		_Val ^= static_cast<size_t>(*_First);
-		_Val *= _FNV_prime;
+		_Val *= hasher_consts::_FNV_prime;
 	}
-
 	return _Val;
 }
 
@@ -65,37 +75,44 @@ struct hasher {
 #else
 		#define _current_type current_type
 #endif
-		return std::hash<_current_type>()(std::get<i>(record)) ^ hashi<i + 1>(record);
+		return ankerl::unordered_dense::hash<_current_type>()(std::get<i>(record)) ^ hashi<i + 1>(record);
 	}
 	size_t operator()(const std::tuple<Types...>& record) const {
 		return hashi(record);
 	}
 };
+template <class T>
+struct hasher<T>{
+	size_t operator()(const std::tuple<T>& record) const {
+		return ankerl::unordered_dense::hash<T>()(std::get<0>(record));
+	}
+};
 
-
-namespace std{
-
+namespace ankerl::unordered_dense{
 	template<>
 	struct hash<astring_view> {
 		size_t operator()(const astring_view& _Keyval) const noexcept {
-			return append_bytes(_Keyval.str);
+			
+			return ankerl::unordered_dense::hash<std::string_view>()(_Keyval.rstr);
+			//return append_bytes(_Keyval.str);
+			
 		}
 	};
 
 	template<>
 	struct hash<types::date_t> {
 		size_t operator() (const types::date_t& _Keyval) const noexcept {
-			return std::hash<unsigned int>()(*(unsigned int*)(&_Keyval));
+			return ankerl::unordered_dense::hash<unsigned int>()(*(unsigned int*)(&_Keyval));
 		}
 	};
 
 	template<>
 	struct hash<types::time_t> {
 		size_t operator() (const types::time_t& _Keyval) const noexcept {
-			return std::hash<unsigned int>()(_Keyval.ms) ^ 
-			std::hash<unsigned char>()(_Keyval.seconds) ^
-			std::hash<unsigned char>()(_Keyval.minutes) ^
-			std::hash<unsigned char>()(_Keyval.hours)
+			return ankerl::unordered_dense::hash<unsigned int>()(_Keyval.ms) ^ 
+			ankerl::unordered_dense::hash<unsigned char>()(_Keyval.seconds) ^
+			ankerl::unordered_dense::hash<unsigned char>()(_Keyval.minutes) ^
+			ankerl::unordered_dense::hash<unsigned char>()(_Keyval.hours)
 			;
 		}
 	};
@@ -103,8 +120,8 @@ namespace std{
 	template<>
 	struct hash<types::timestamp_t>{
 		size_t operator() (const types::timestamp_t& _Keyval) const noexcept {
-			return std::hash<types::date_t>()(_Keyval.date) ^ 
-				std::hash<types::time_t>()(_Keyval.time);
+			return ankerl::unordered_dense::hash<types::date_t>()(_Keyval.date) ^ 
+				ankerl::unordered_dense::hash<types::time_t>()(_Keyval.time);
 		}
 	};
 #ifdef __SIZEOF_INT128__
@@ -112,12 +129,11 @@ namespace std{
 	template<>
 	struct hash<int128_struct>{
 		size_t operator() (const int128_struct& _Keyval) const noexcept {
-			return std::hash<uint64_t>()(_Keyval.__struct.low) ^ std::hash<uint64_t>()(_Keyval.__struct.high);
+			return ankerl::unordered_dense::hash<uint64_t>()(_Keyval.__struct.low) ^ ankerl::unordered_dense::hash<uint64_t>()(_Keyval.__struct.high);
 		}
 	};
 #endif
 	template <class ...Types>
 	struct hash<std::tuple<Types...>> : public hasher<Types...>{ };
-
 }
 

server/io.h

@@ -4,6 +4,7 @@
 #include <string>
 #include <limits>
 #include <cstring>
+#include <string_view>
 template <class ...Types>
 std::string generate_printf_string(const char* sep = " ", const char* end = "\n") {
 	std::string str;
@@ -25,6 +26,11 @@ inline decltype(auto) print_hook<bool>(const bool& v) {
 	return v? "true" : "false";
 }
 
+template<>
+inline decltype(auto) print_hook<std::string_view>(const std::string_view& v) {
+	return v.data();
+}
+
 extern char* gbuf;
 
 void setgbuf(char* buf = 0);

server/jeaiii_to_text.h

@@ -82,7 +82,7 @@ inline char* to_text(char text[], unsigned long long i) { return to_text_from_in
 
 // Copyright (c) 2022 Bill Sun
 
-//#if defined(SIZEOF___INT128) || (defined(SIZEOF___INT128_T) && defined(SIZEOF___UINT128_T))
+#if defined(__SIZEOF_INT128__) 
 constexpr static __uint128_t _10_19 = 10000000000000000000ull, 
     _10_37 = _10_19*_10_19 / 10;
 
@@ -118,4 +118,4 @@ char* jeaiii_i128(char* buf, T v){
     buf = to_text(buf, uint64_t(v % _10_19));
     return buf;
 }
-// #endif
+#endif // INT128

server/libaquery.cpp

@@ -9,6 +9,7 @@
 #include "utils.h"
 #include "libaquery.h"
 #include <random>
+#include "gc.h"
 
 char* gbuf = nullptr;
 
@@ -30,12 +31,14 @@ void print<__int128_t>(const __int128_t& v, const char* delimiter){
 	s[40] = 0;
 	std::cout<< get_int128str(v, s+40)<< delimiter;
 }
+
 template <>
 void print<__uint128_t>(const __uint128_t&v, const char* delimiter){
 	char s[41];
 	s[40] = 0;
     std::cout<< get_uint128str(v, s+40) << delimiter;
 }
+
 std::ostream& operator<<(std::ostream& os, __int128 & v)
 {
 	print(v);
@@ -54,26 +57,7 @@ void print<bool>(const bool&v, const char* delimiter){
 	std::cout<< (v?"true":"false") << delimiter;
 }
 
-template<class T> 
-T getInt(const char*& buf){
-	T ret = 0;
-	while(*buf >= '0' and *buf <= '9'){
-		ret = ret*10 + *buf - '0';
-		buf++;
-	}
-	return ret;
-}
 
-template<class T> 
-char* intToString(T val, char* buf){
-
-	while (val > 0){
-		*--buf = val%10 + '0';
-		val /= 10;
-	}
-	
-	return buf;
-}
 void skip(const char*& buf){ 
 	while(*buf && (*buf >'9' || *buf < '0')) buf++; 
 }
@@ -262,8 +246,8 @@ std::string base62uuid(int l) {
     static uniform_int_distribution<uint64_t> u(0x10000, 0xfffff);
     uint64_t uuid = (u(engine) << 32ull) + 
 		(std::chrono::system_clock::now().time_since_epoch().count() & 0xffffffff);
-    //printf("%llu\n", uuid);
-    string ret;
+    
+	string ret;
     while (uuid && l-- >= 0) {
         ret = string("") + base62alp[uuid % 62] + ret;
         uuid /= 62;
@@ -276,10 +260,29 @@ inline const char* str(const bool& v) {
 	return v ? "true" : "false";
 }
 
+class A {
+	public:
+	std::chrono::high_resolution_clock::time_point tp;
+	A(){
+		tp = std::chrono::high_resolution_clock::now();
+		printf("A %llu created.\n", tp.time_since_epoch().count());
+	}
+	~A() {
+		printf("A %llu died after %lldns.\n", tp.time_since_epoch().count(),
+		 (std::chrono::high_resolution_clock::now() - tp).count());
+	}
+};
 
 Context::Context() {
     current.memory_map = new std::unordered_map<void*, deallocator_t>;
-    init_session();
+#ifndef __AQ_USE_THREADEDGC__
+	this->gc = new GC();
+#endif
+	GC::gc_handle->reg(new A(), 6553600, [](void* a){
+		puts("deleting");
+		delete ((A*)a);
+	});
+	init_session();
 }
 
 Context::~Context() {
@@ -328,7 +331,6 @@ void* Context::get_module_function(const char* fname){
 // 	std::cout << ')';
 // }
 
-#include "gc.h"
 #include <utility>
 #include <thread>
 #ifndef __AQ_USE_THREADEDGC__
@@ -383,7 +385,7 @@ void GC::daemon() {
 
 	while (alive) {
 		if (running) {
-			if (current_size - max_size > 0 || 
+			if (uint64_t(current_size) > max_size || 
 				forceclean_timer > forced_clean) 
 			{
 				gc();
@@ -432,19 +434,88 @@ void GC::reg(void* v, uint32_t sz, void(*f)(void*)) { //~ 40ns expected v. free
 		f(v);
 		return;
 	}
+	else if (sz == 0xffffffff)
+		sz = this->threshould;
+	
 	auto _q = static_cast<memoryqueue_t>(q);
 	while(lock);
 	++alive_cnt;
 	current_size += sz;
 	auto _slot = (slot_pos += 1);
-	_q[_slot] = {v, f};
+	_q[_slot-1] = {v, f};
 	--alive_cnt;
 	running = true;
 }
 
 #endif
 
-GC* GC::gc_handle = nullptr;
+inline GC* GC::gc_handle = nullptr;
+inline ScratchSpace* GC::scratch_space = nullptr;
+
+void ScratchSpace::init(size_t initial_capacity) {
+	ret = nullptr;
+	scratchspace = static_cast<char*>(malloc(initial_capacity));
+	ptr = cnt = 0;
+	capacity = initial_capacity;
+	this->initial_capacity = initial_capacity;
+	temp_memory_fractions = new vector_type<void*>();
+}
+
+inline void* ScratchSpace::alloc(uint32_t sz){
+    ptr = this->cnt;
+	this->cnt += sz; // major cost
+	if (this->cnt > capacity) {
+		[[unlikely]] 
+		capacity = this->cnt + (capacity >> 1);
+		auto vec_tmpmem_fractions = static_cast<vector_type<char *>*>(temp_memory_fractions);
+		vec_tmpmem_fractions->emplace_back(scratchspace);
+		scratchspace = static_cast<char*>(malloc(capacity));
+		ptr = 0;
+	}
+	return scratchspace + ptr;
+}
+
+inline void ScratchSpace::register_ret(void* ret){
+	this->ret = ret;
+}
+
+inline void ScratchSpace::release(){
+	ptr = cnt = 0;
+	auto vec_tmpmem_fractions = 
+		static_cast<vector_type<void*>*>(temp_memory_fractions);
+	if (vec_tmpmem_fractions->size) {
+    	[[unlikely]]
+		for(auto& mem : *vec_tmpmem_fractions){
+			//free(mem);
+			GC::gc_handle->reg(mem);
+		}
+		vec_tmpmem_fractions->clear();
+	}
+}
+
+inline void ScratchSpace::reset() {
+	this->release();
+	ret = nullptr;
+	if (capacity != initial_capacity){
+		capacity = initial_capacity;
+		scratchspace = static_cast<char*>(realloc(scratchspace, capacity));
+	}
+}
+
+void ScratchSpace::cleanup(){
+	auto vec_tmpmem_fractions = 
+		static_cast<vector_type<void*>*>(temp_memory_fractions);
+	if (vec_tmpmem_fractions->size) {
+		for(auto& mem : *vec_tmpmem_fractions){
+			//free(mem); 
+			GC::gc_handle->reg(mem);
+		}
+		vec_tmpmem_fractions->clear();
+	}
+	delete vec_tmpmem_fractions;
+	free(this->scratchspace);
+}
+
 
 #include "dragonbox/dragonbox_to_chars.hpp" 
 
@@ -518,4 +589,34 @@ aq_to_chars<types::timestamp_t>(void* value, char* buffer) {
 	return buffer;
 }
 
+template<> 
+char* 
+aq_to_chars<std::string_view>(void* value, char* buffer){
+	const auto& src = *static_cast<std::string_view*>(value);
+	memcpy(buffer, src.data(), src.size());
+	return buffer + src.size();
+}
 
+// Defined in vector_type.h
+template <>
+vector_type<std::string_view>::vector_type(const char** container, uint32_t len, 
+		typename std::enable_if_t<true>*) noexcept
+{
+	size = capacity = len;
+	this->container = static_cast<std::string_view*>(
+		malloc(sizeof(std::string_view) * len));
+	for(uint32_t i = 0; i < len; ++i){
+		this->container[i] = container[i];
+	}
+}
+
+template<>
+vector_type<std::string_view>::vector_type(const uint32_t size, void* data) : 
+	size(size), capacity(0) {
+	this->container = static_cast<std::string_view*>(
+		malloc(sizeof(std::string_view) * size));
+	for(uint32_t i = 0; i < size; ++i){
+		this->container[i] = ((const char**)data)[i];
+	}
+	//std::cout<<size << container[1];
+}

server/libaquery.h

@@ -4,11 +4,12 @@
 #ifdef __INTELLISENSE__
 	#define __AQUERY_ITC_USE_SEMPH__
 	#define THREADING
-	#define __AQ_THREADED_GC__
 #endif
 
 #include <unordered_map>
 #include <chrono>
+#include <filesystem>
+#include <cstring>
 class aq_timer {
 private:
 	std::chrono::high_resolution_clock::time_point now;
@@ -32,6 +33,27 @@ public:
 
 #include "table.h"
 
+template<class T = int> 
+T getInt(const char*& buf){
+	T ret = 0;
+	while(*buf >= '0' and *buf <= '9'){
+		ret = ret*10 + *buf - '0';
+		buf++;
+	}
+	return ret;
+}
+
+template<class T> 
+char* intToString(T val, char* buf){
+
+	while (val > 0){
+		*--buf = val%10 + '0';
+		val /= 10;
+	}
+	
+	return buf;
+}
+
 
 enum Log_level {
 	LOG_INFO,
@@ -66,24 +88,34 @@ struct Session{
     void* memory_map;
 };
 
-struct Context{
+struct StoredProcedure {
+	uint32_t cnt, postproc_modules;
+	char **queries;
+	const char* name;
+	void **__rt_loaded_modules;
+};
+struct Trigger;
+struct IntervalBasedTriggerHost;
+struct CallbackBasedTriggerHost;
+
+struct Context {
     typedef int (*printf_type) (const char *format, ...);
 
-	void* module_function_maps = 0;
+	void* module_function_maps = nullptr;
 	Config* cfg;
 
 	int n_buffers, *sz_bufs;
 	void **buffers;
 
-	void* alt_server = 0;
+	void* alt_server = nullptr;
 	Log_level log_level = LOG_INFO;
 
 	Session current;
-
+	const char* aquery_root_path;
 #ifdef THREADING
 	void* thread_pool;
 #endif	
-#ifdef __AQ_THREADED_GC__
+#ifndef __AQ_USE_THREADEDGC__
 	void* gc;
 #endif
 	printf_type print = &printf;
@@ -102,11 +134,21 @@ struct Context{
 	void init_session();
 	void end_session();
 	void* get_module_function(const char*);
-	std::unordered_map<const char*, void*> tables;
-    std::unordered_map<const char*, uColRef *> cols;
+	std::unordered_map<std::string, void*> tables;
+    std::unordered_map<std::string, uColRef *> cols;
+    std::unordered_map<std::string, StoredProcedure> stored_proc;
+    std::unordered_map<std::string, Trigger> triggers;
+	IntervalBasedTriggerHost *it_host;
+	CallbackBasedTriggerHost *ct_host;
 };
 
 
+struct StoredProcedurePayload {
+	StoredProcedure *p;
+	Context* cxt;
+};
+
+int execTriggerPayload(void*);
 
 #ifdef _WIN32
 #define __DLLEXPORT__  __declspec(dllexport) __stdcall 
@@ -153,4 +195,94 @@ template<> char* aq_to_chars<char*>(void* , char*);
 template<> char* aq_to_chars<types::date_t>(void* , char*);
 template<> char* aq_to_chars<types::time_t>(void* , char*);
 template<> char* aq_to_chars<types::timestamp_t>(void* , char*);
+template<> char* aq_to_chars<std::string_view>(void* , char*);
+typedef int (*code_snippet)(void*);
+
+template <class _This_Struct>
+inline void AQ_ZeroMemory(_This_Struct& __val) {
+	memset(&__val, 0, sizeof(_This_Struct));
+}
+
+template <typename _This_Type>
+inline _This_Type* AQ_DupObject(_This_Type* __val) {
+	auto ret = (_This_Type*)(malloc(sizeof(_This_Type)));
+	memcpy(ret, __val, sizeof(_This_Type));
+	return ret;
+}
+
+#ifdef __USE_STD_SEMAPHORE__
+	#include <semaphore>
+	class A_Semaphore {
+	private:
+		std::binary_semaphore native_handle;
+	public:
+		A_Semaphore(bool v = false) {
+			native_handle = std::binary_semaphore(v);
+		}
+		void acquire() {
+			native_handle.acquire();
+		}
+		void release() {
+			native_handle.release();
+		}
+		~A_Semaphore() { }
+	};
+#else
+	#ifdef _WIN32
+		class A_Semaphore {
+		private:
+			void* native_handle;
+		public:
+			A_Semaphore(bool);
+			void acquire();
+			void release();
+			~A_Semaphore();
+		};
+	#else
+		#ifdef __APPLE__
+			#include <dispatch/dispatch.h>
+			class A_Semaphore {
+			private:
+				dispatch_semaphore_t native_handle;
+			public:
+				explicit A_Semaphore(bool v = false) {
+					native_handle = dispatch_semaphore_create(v);
+				}
+				void acquire() {
+					// puts("acquire");
+					dispatch_semaphore_wait(native_handle, DISPATCH_TIME_FOREVER);
+				}
+				void release() {
+					// puts("release");
+					dispatch_semaphore_signal(native_handle);
+				}
+				~A_Semaphore() {
+				}
+			};
+		#else
+			#include <semaphore.h>
+			class A_Semaphore {
+			private:
+				sem_t native_handle;
+			public:
+				A_Semaphore(bool v = false) {
+					sem_init(&native_handle, v, 1);
+				}
+				void acquire() {
+					sem_wait(&native_handle);
+				}
+				void release() {
+					sem_post(&native_handle);
+				}
+				~A_Semaphore() {
+					sem_destroy(&native_handle);
+				}
+			};
+		#endif // __APPLE__
+
+	#endif // _WIN32 
+#endif //__USE_STD_SEMAPHORE__
+
+void print_monetdb_results(void* _srv, const char* sep, const char* end, uint32_t limit);
+
 #endif

server/mariadb_conn.h

@@ -6,17 +6,17 @@
 struct Context;
 
 struct Server{
-    MYSQL *server = 0;
-    Context *cxt = 0;
-    bool status = 0;
+    MYSQL *server = nullptr;
+    Context *cxt = nullptr;
+    bool status = false;
     bool has_error = false;
-    char* query = 0;
+    char* query = nullptr;
     int type = 0;
     
     void connect(Context* cxt, const char* host = "bill.local",
         const char* user = "root", const char* passwd = "0508", 
         const char* db_name = "db", const unsigned int port = 3306, 
-        const char* unix_socket = 0, const unsigned long client_flag = 0
+        const char* unix_socket = nullptr, const unsigned long client_flag = 0
     );
     void exec(const char* q);
     void close();

server/monetdb_conn.cpp

@@ -5,9 +5,25 @@
 #include <string>
 #include "monetdb_conn.h"
 #include "monetdbe.h"
-#include "table.h"
 
-#undef static_assert
+#include "table.h"
+#include <thread>
+
+#ifdef _WIN32
+    #include "winhelper.h"
+#else 
+    #include <dlfcn.h>
+    #include <fcntl.h>
+    #include <sys/mman.h>
+    #include <atomic>
+#endif // _WIN32
+
+#ifdef ERROR
+    #undef ERROR
+#endif
+#ifdef static_assert
+    #undef static_assert
+#endif
 
 constexpr const char* monetdbe_type_str[] = {
 	"monetdbe_bool", "monetdbe_int8_t", "monetdbe_int16_t", "monetdbe_int32_t", "monetdbe_int64_t",
@@ -73,11 +89,11 @@ void Server::connect(Context *cxt){
         printf("Error: Server %p already connected. Restart? (Y/n). \n", server);
         char c[50];
         std::cin.getline(c, 49);
-        for(int i = 0; i < 50; ++i){
-            if (!c[i] || c[i] == 'y' || c[i] == 'Y'){
+        for(int i = 0; i < 50; ++i) {
+            if (!c[i] || c[i] == 'y' || c[i] == 'Y') {
                 monetdbe_close(*server);
                 free(*server);
-                this->server = 0;
+                this->server = nullptr;
                 break;
             }
             else if(c[i]&&!(c[i] == ' ' || c[i] == '\t'))
@@ -86,7 +102,10 @@ void Server::connect(Context *cxt){
     }
 
     server = (monetdbe_database*)malloc(sizeof(monetdbe_database));
-    auto ret = monetdbe_open(server, nullptr, nullptr);
+    monetdbe_options ops;
+    AQ_ZeroMemory(ops);
+    ops.nr_threads = std::thread::hardware_concurrency();
+    auto ret = monetdbe_open(server, nullptr, &ops);
     if (ret == 0){
         status = true;
         this->server = server;
@@ -105,7 +124,7 @@ void Server::exec(const char* q){
     auto _res = static_cast<monetdbe_result*>(this->res);
     monetdbe_cnt _cnt = 0;
     auto qresult = monetdbe_query(*server, const_cast<char*>(q), &_res, &_cnt);
-    if (_res != 0){
+    if (_res != nullptr){
         this->cnt = _res->nrows;
         this->res = _res;
     }
@@ -147,9 +166,10 @@ void Server::print_results(const char* sep, const char* end){
             col_data[i] = static_cast<char *>(cols[i]->data);
             szs [i] = monetdbe_type_szs[cols[i]->type];
             header_string = header_string + cols[i]->name + sep + '|' + sep;
+            if (err_msg) [[unlikely]]
+                puts(err_msg);
         }
-        const size_t l_sep = strlen(sep) + 1;
-		if (header_string.size() - l_sep >= 0)
+		if (const size_t l_sep = strlen(sep) + 1; header_string.size() >= l_sep)
 			header_string.resize(header_string.size() - l_sep);
         header_string += end + std::string(header_string.size(), '=') + end;
         fputs(header_string.c_str(), stdout);
@@ -169,7 +189,7 @@ void Server::print_results(const char* sep, const char* end){
 void Server::close(){
     if(this->server){
         auto server = static_cast<monetdbe_database*>(this->server);
-        monetdbe_close(*(server));
+        monetdbe_close(*server);
         free(server);
         this->server = nullptr;
     }
@@ -211,3 +231,46 @@ bool Server::havehge() {
     return false;
 #endif
 }
+
+
+void ExecuteStoredProcedureEx(const StoredProcedure *p, Context* cxt){
+    auto server = static_cast<Server*>(cxt->alt_server);
+    void* handle = nullptr;
+    uint32_t procedure_module_cursor = 0;
+    for(uint32_t i = 0; i < p->cnt; ++i) {
+        switch(p->queries[i][0]){
+            case 'Q': {
+                server->exec(p->queries[i]);
+            } 
+            break;
+            case 'P': {
+                auto c = code_snippet(dlsym(handle, p->queries[i]+1));
+                c(cxt);
+            }
+            break;
+            case 'N': {
+                if(procedure_module_cursor < p->postproc_modules)
+                    handle = p->__rt_loaded_modules[procedure_module_cursor++];
+            }
+            break;
+            case 'O': {
+                uint32_t limit;
+                memcpy(&limit, p->queries[i] + 1, sizeof(uint32_t));
+                if (limit == 0)
+                    continue;
+                print_monetdb_results(server, " ", "\n", limit); 
+            }
+            break;
+            default:
+                printf("Warning Q%u: unrecognized command %c.\n", 
+                    i, p->queries[i][0]);
+        }
+    }
+}
+
+int execTriggerPayload(void* args) {
+    auto spp = (StoredProcedurePayload*)(args);
+    ExecuteStoredProcedureEx(spp->p, spp->cxt);
+    delete spp;
+    return 0;
+}

server/monetdb_conn.h

@@ -4,18 +4,18 @@
 struct Context;
 
 struct Server{
-    void *server = 0;
-    Context *cxt = 0;
-    bool status = 0;
-    char* query = 0;
+    void *server = nullptr;
+    Context *cxt = nullptr;
+    bool status = false;
+    char* query = nullptr;
     int type = 1;
 
-    void* res = 0;
-    void* ret_col = 0;
+    void* res = nullptr;
+    void* ret_col = nullptr;
     long long cnt = 0;
-    char* last_error = 0;
+    char* last_error = nullptr;
     
-    Server(Context* cxt = nullptr);
+    explicit Server(Context* cxt = nullptr);
     void connect(Context* cxt);
     void exec(const char* q);
     void *getCol(int col_idx);
@@ -24,7 +24,7 @@ struct Server{
     static bool havehge();
     void test(const char*);
     void print_results(const char* sep = " ", const char* end = "\n");
-    friend void print_monetdb_results(Server* srv, const char* sep, const char* end, int limit);
+    friend void print_monetdb_results(void* _srv, const char* sep, const char* end, int limit);
     ~Server();
 };
 
@@ -34,4 +34,9 @@ struct monetdbe_table_data{
     void* cols;
 };
 
+size_t 
+monetdbe_get_size(void* dbhdl, const char *table_name);
+
+void* 
+monetdbe_get_col(void* dbhdl, const char *table_name, uint32_t col_id);
 #endif

server/monetdb_ext.c

@@ -0,0 +1,90 @@
+// Non-standard Extensions for MonetDBe, may break concurrency control!
+
+#include "monetdbe.h"
+#include <stdint.h>
+#include "mal_client.h"
+#include "sql_mvc.h"
+#include "sql_semantic.h"
+#include "mal_exception.h"
+
+typedef struct column_storage {
+	int refcnt;
+	int bid;
+	int ebid;		/* extra bid */
+	int uibid;		/* bat with positions of updates */
+	int uvbid;		/* bat with values of updates */
+	storage_type st; /* ST_DEFAULT, ST_DICT, ST_FOR */
+	bool cleared;
+	bool merged;	/* only merge changes once */
+	size_t ucnt;	/* number of updates */
+	ulng ts;		/* version timestamp */
+} column_storage;
+
+typedef struct segment {
+	BUN start;
+	BUN end;
+	bool deleted;	/* we need to keep a dense segment set, 0 - end of last segemnt,
+					   some segments maybe deleted */
+	ulng ts;		/* timestamp on this segment, ie tid of some active transaction or commit time of append/delete or
+					   rollback time, ie ready for reuse */
+	ulng oldts;		/* keep previous ts, for rollbacks */
+	struct segment *next;	/* usualy one should be enough */
+	struct segment *prev;	/* used in destruction list */
+} segment;
+
+/* container structure to allow sharing this structure */
+typedef struct segments {
+	sql_ref r;
+	struct segment *h;
+	struct segment *t;
+} segments;
+
+typedef struct storage {
+	column_storage cs;	/* storage on disk */
+	segments *segs;	/* local used segements */
+	struct storage *next;
+} storage;
+
+typedef struct {
+	char 	language;		/* 'S' or 's' or 'X' */
+	char	depth;			/* depth >= 1 means no output for trans/schema statements */
+	int 	remote;			/* counter to make remote function names unique */
+	mvc 	*mvc;
+    char others[];
+} backend;
+
+typedef struct {
+	Client c;
+	char *msg;
+	monetdbe_data_blob blob_null;
+	monetdbe_data_date date_null;
+	monetdbe_data_time time_null;
+	monetdbe_data_timestamp timestamp_null;
+	str mid;
+} monetdbe_database_internal;
+
+size_t
+monetdbe_get_size(monetdbe_database dbhdl, const char *table_name)
+{
+	monetdbe_database_internal* hdl = (monetdbe_database_internal*)dbhdl;
+	backend* be = ((backend *)(((monetdbe_database_internal*)dbhdl)->c->sqlcontext));
+	mvc *m = be->mvc;
+	sql_table *t = find_table_or_view_on_scope(m, NULL, "sys", table_name, "CATALOG", false);
+	sql_column *col = ol_first_node(t->columns)->data;
+	sqlstore* store = m->store;
+	size_t sz = store->storage_api.count_col(m->session->tr, col, QUICK);
+	return sz; 
+}
+
+void* 
+monetdbe_get_col(monetdbe_database dbhdl, const char *table_name, uint32_t col_id) {
+    monetdbe_database_internal* hdl = (monetdbe_database_internal*)dbhdl;
+	backend* be = ((backend *)(((monetdbe_database_internal*)dbhdl)->c->sqlcontext));
+	mvc *m = be->mvc;
+	sql_table *t = find_table_or_view_on_scope(m, NULL, "sys", table_name, "CATALOG", false);
+	sql_column *col = ol_fetch(t->columns, col_id);
+	sqlstore* store = m->store;
+	BAT *b = store->storage_api.bind_col(m->session->tr, col, QUICK);
+	BATiter iter = bat_iterator(b);
+    return iter.base;
+}

server/priority_vector.hpp

@@ -6,7 +6,7 @@ template <class Comparator, typename T = uint32_t>
 class priority_vector : public vector_type<T> {
 	const Comparator comp;
 public:
-	priority_vector(Comparator comp = std::less<T>{}) : 
+	explicit priority_vector(Comparator comp = std::less<T>{}) : 
 		comp(comp), vector_type<T>(0) {}
 	void emplace_back(T val) {
 		vector_type<T>::emplace_back(val);

server/server.cpp

@@ -1,5 +1,5 @@
 #include "pch_msc.hpp"
-
+#define TESTMAIN
 #include <iostream>
 #include <string>
 #include <chrono>
@@ -7,6 +7,7 @@
 
 #include "libaquery.h"
 #include "monetdb_conn.h"
+#pragma region misc
 #ifdef THREADING
 #include "threading.h"
 #endif
@@ -19,10 +20,6 @@
 #include <sys/mman.h>
 #include <atomic>
 
-// fast numeric to string conversion
-#include "jeaiii_to_text.h"
-#include "dragonbox/dragonbox_to_chars.h"
-
 struct SharedMemory
 {
     std::atomic<bool> a;
@@ -40,69 +37,7 @@ struct SharedMemory
     }
 };
 
-#ifndef __USE_STD_SEMAPHORE__
-#ifdef __APPLE__
-#include <dispatch/dispatch.h>
-class A_Semaphore {
-private:
-	dispatch_semaphore_t native_handle;
-public:
-	A_Semaphore(bool v = false) {
-		native_handle = dispatch_semaphore_create(v);
-	}
-	void acquire() {
-        // puts("acquire");
-		dispatch_semaphore_wait(native_handle, DISPATCH_TIME_FOREVER);
-	}
-	void release() {
-        // puts("release");
-		dispatch_semaphore_signal(native_handle);
-	}
-	~A_Semaphore() {
-	}
-};
-#else
-#include <semaphore.h>
-class A_Semaphore {
-private:
-	sem_t native_handle;
-public:
-	A_Semaphore(bool v = false) {
-		sem_init(&native_handle, v, 1);
-	}
-	void acquire() {
-		sem_wait(&native_handle);
-	}
-	void release() {
-		sem_post(&native_handle);
-	}
-	~A_Semaphore() {
-		sem_destroy(&native_handle);
-	}
-};
-#endif
-#endif
-#endif
-
-#ifdef __USE_STD_SEMAPHORE__
-#define __AQUERY_ITC_USE_SEMPH__
-#include <semaphore>
-class A_Semaphore {
-private:
-    std::binary_semaphore native_handle;
-public:
-    A_Semaphore(bool v = false) {
-        native_handle = std::binary_semaphore(v);
-    }
-    void acquire() {
-        native_handle.acquire();
-    }
-    void release() {
-        native_handle.release();
-    }
-    ~A_Semaphore() { }
-};
-#endif
+#endif // _WIN32
 
 #ifdef __AQUERY_ITC_USE_SEMPH__
 A_Semaphore prompt{ true }, engine{ false };
@@ -117,10 +52,8 @@ A_Semaphore prompt{ true }, engine{ false };
 #define ENGINE_RELEASE() 
 #endif
 
-typedef int (*code_snippet)(void*);
 typedef void (*module_init_fn)(Context*);
 
-
 int n_recv = 0;
 char** n_recvd = nullptr;
 
@@ -153,7 +86,8 @@ extern "C" int __DLLEXPORT__ binary_info() {
 #endif
 }
 
-__AQEXPORT__(bool) have_hge(){
+__AQEXPORT__(bool) 
+have_hge() {
 #if defined(__MONETDB_CONN_H__)
     return Server::havehge();
 #else
@@ -161,8 +95,35 @@ __AQEXPORT__(bool) have_hge(){
 #endif
 }
 
+__AQEXPORT__(StoredProcedure)
+get_procedure(Context* cxt, const char* name) {
+    auto res = cxt->stored_proc.find(name);
+    if (res == cxt->stored_proc.end())
+        return { .cnt = 0, 
+            .postproc_modules = 0, 
+            .queries = nullptr, 
+            .name = nullptr, 
+            .__rt_loaded_modules = nullptr
+        };
+    return res->second;
+}
+
 using prt_fn_t = char* (*)(void*, char*);
 
+// This function contains heap allocations, free after use
+template<class String_T>
+char* to_lpstr(const String_T& str){
+    auto ret = static_cast<char*>(malloc(str.size() + 1));
+    memcpy(ret, str.c_str(), str.size());
+    ret[str.size()] = '\0';
+    return ret;
+}
+char* copy_lpstr(const char* str){
+    auto len = strlen(str);
+    auto ret = static_cast<char*>(malloc(len + 1));
+    memcpy(ret, str, len + 1);
+    return ret;
+}
 
 constexpr prt_fn_t monetdbe_prtfns[] = {
 	aq_to_chars<bool>, aq_to_chars<int8_t>, aq_to_chars<int16_t>, aq_to_chars<int32_t>, 
@@ -178,7 +139,24 @@ constexpr prt_fn_t monetdbe_prtfns[] = {
 	aq_to_chars<std::nullptr_t>
 };
 
+#ifndef __AQ_USE_THREADEDGC__
+void aq_init_gc(void *handle, Context* cxt)
+{
+    typedef void (*aq_gc_init_t) (Context*);
+    if (handle && cxt){
+        auto sym = dlsym(handle, "__AQ_Init_GC__");
+        if(sym){
+            ((aq_gc_init_t)sym)(cxt);
+        }
+    }
+}
+#else //__AQ_USE_THREADEDGC__
+#define aq_init_gc(h, c) 
+#endif //__AQ_USE_THREADEDGC__
+
 #include "monetdbe.h"
+#undef max
+#undef min
 inline constexpr static unsigned char monetdbe_type_szs[] = {
     sizeof(monetdbe_column_bool::null_value), sizeof(monetdbe_column_int8_t::null_value), 
     sizeof(monetdbe_column_int16_t::null_value), sizeof(monetdbe_column_int32_t::null_value), 
@@ -195,9 +173,10 @@ inline constexpr static unsigned char monetdbe_type_szs[] = {
     1
 };
 constexpr uint32_t output_buffer_size = 65536;
-void print_monetdb_results(Server* srv, const char* sep = " ", const char* end = "\n", 
+void print_monetdb_results(void* _srv, const char* sep = " ", const char* end = "\n", 
     uint32_t limit = std::numeric_limits<uint32_t>::max()) {
-    if (!srv->haserror() && srv->cnt && limit){
+    auto srv = static_cast<Server *>(_srv);
+    if (!srv->haserror() && srv->cnt && limit) {
         char buffer[output_buffer_size];
         auto _res = static_cast<monetdbe_result*> (srv->res);
         const auto ncols = _res->ncols;
@@ -225,7 +204,7 @@ void print_monetdb_results(Server* srv, const char* sep = " ", const char* end =
             puts("Error: separator or end string too long");
             goto cleanup;
         }
-		if (header_string.size() - l_sep - 1>= 0)
+		if (header_string.size() >= l_sep + 1)
 			header_string.resize(header_string.size() - l_sep - 1);
         header_string += end + std::string(header_string.size(), '=') + end;
         fputs(header_string.c_str(), stdout);
@@ -266,10 +245,35 @@ void initialize_module(const char* module_name, void* module_handle, Context* cx
     }
 }
 
+#pragma endregion
 int dll_main(int argc, char** argv, Context* cxt){
     aq_timer timer;
     Config *cfg = reinterpret_cast<Config *>(argv[0]);
     std::unordered_map<std::string, void*> user_module_map;
+    std::string pwd = (char*)(std::filesystem::current_path().string().c_str());
+    char sep = std::filesystem::path::preferred_separator;
+    pwd += sep;
+    std::string procedure_root = (pwd + "procedures") + sep;
+    std::string procedure_name = "";
+    StoredProcedure current_procedure;
+    vector_type<char *> recorded_queries;
+    vector_type<void *> recorded_libraries;
+    bool procedure_recording = false, 
+         procedure_replaying = false;
+    uint32_t procedure_module_cursor = 0;
+    try {
+        if (!std::filesystem::is_directory(procedure_root)) {
+            if (std::filesystem::exists(procedure_root))
+                std::filesystem::remove_all(procedure_root);
+        }
+        if (!std::filesystem::exists(procedure_root)) {
+            std::filesystem::create_directory(procedure_root);
+        }
+    }
+    catch (std::filesystem::filesystem_error& e) {
+        printf("Failed to create directory %s: %s\n", procedure_root.c_str(), e.what());
+    }
+    
     if (cxt->module_function_maps == nullptr)
         cxt->module_function_maps = new std::unordered_map<std::string, void*>();
     auto module_fn_map = 
@@ -279,7 +283,6 @@ int dll_main(int argc, char** argv, Context* cxt){
     void** buffers = (void**) malloc (sizeof(void*) * cfg->n_buffers);
     for (int i = 0; i < cfg->n_buffers; i++) 
         buffers[i] = static_cast<void *>(argv[i + 1]);
-
     cxt->buffers = buffers;
     cxt->cfg = cfg;
     cxt->n_buffers = cfg->n_buffers;
@@ -296,6 +299,7 @@ int dll_main(int argc, char** argv, Context* cxt){
         if (cfg->new_query) {
             cfg->stats.postproc_time = 0;
             cfg->stats.monet_time = 0;
+start:
 
             void *handle = nullptr;
             void *user_module_handle = nullptr;
@@ -305,7 +309,29 @@ int dll_main(int argc, char** argv, Context* cxt){
                 Server* server = reinterpret_cast<Server*>(cxt->alt_server);
                 if(n_recv > 0){
                     if (cfg->backend_type == BACKEND_AQuery || cfg->has_dll) {
-                        handle = dlopen("./dll.so", RTLD_NOW);
+                        const char* proc_name = "./dll.so";
+                        std::string dll_path;
+                        if (procedure_recording) {
+                            dll_path = procedure_root + 
+                                procedure_name + std::to_string(recorded_libraries.size) + ".so";
+                            
+                            try{
+                                if (std::filesystem::exists(dll_path))
+                                    std::filesystem::remove(dll_path);
+                                std::filesystem::copy_file(proc_name, dll_path);
+                            } catch(std::filesystem::filesystem_error& e){
+                                puts(e.what());
+                                dll_path = proc_name;
+                            }
+                            proc_name = dll_path.c_str();
+                            //if(recorded_libraries.size)
+                            recorded_queries.emplace_back(copy_lpstr("N"));
+                        }
+                        handle = dlopen(proc_name, RTLD_NOW);
+                        aq_init_gc(handle, cxt);
+                        if (procedure_recording) {
+                            recorded_libraries.emplace_back(handle);
+                        }
                     }
                     for (const auto& module : user_module_map){
                         initialize_module(module.first.c_str(), module.second, cxt);
@@ -313,19 +339,26 @@ int dll_main(int argc, char** argv, Context* cxt){
                     cxt->init_session();
                     for(int i = 0; i < n_recv; ++i)
                     {
-                        //printf("%s, %d\n", n_recvd[i], n_recvd[i][0] == 'Q');
+                        printf("%s, %d\n", n_recvd[i], n_recvd[i][0] == 'Q');
                         switch(n_recvd[i][0]){
                         case 'Q': // SQL query for monetdbe
                             {
+                                if(procedure_recording){
+                                    recorded_queries.emplace_back(copy_lpstr(n_recvd[i]));
+                                }
                                 timer.reset();
                                 server->exec(n_recvd[i] + 1);
                                 cfg->stats.monet_time += timer.elapsed();
-                                // printf("Exec Q%d: %s", i, n_recvd[i]);
+                                printf("Exec Q%d: %s", i, n_recvd[i]);
                             }
                             break;
                         case 'P': // Postprocessing procedure 
                             if(handle && !server->haserror()) {
+                                if (procedure_recording) {
+                                    recorded_queries.emplace_back(copy_lpstr(n_recvd[i]));
+                                }
                                 code_snippet c = reinterpret_cast<code_snippet>(dlsym(handle, n_recvd[i]+1));
+                                printf("%p", dlsym(handle, n_recvd[i] + 1));
                                 timer.reset();
                                 c(cxt);
                                 cfg->stats.postproc_time += timer.elapsed();
@@ -334,7 +367,7 @@ int dll_main(int argc, char** argv, Context* cxt){
                         case 'M': // Load Module
                             {
                                 auto mname = n_recvd[i] + 1;
-                                user_module_handle = dlopen(mname, RTLD_LAZY);
+                                user_module_handle = dlopen(mname, RTLD_NOW);
                                 //getlasterror
                                 
                                 if (!user_module_handle)
@@ -358,9 +391,14 @@ int dll_main(int argc, char** argv, Context* cxt){
                         case 'O':
                             {
                                 if(!server->haserror()){
+                                    if (procedure_recording){
+                                        char* buf = (char*) malloc (sizeof(char) * 6);
+                                        memcpy(buf, n_recvd[i], 5);
+                                        buf[5] = '\0';
+                                        recorded_queries.emplace_back(buf);
+                                    }
                                     uint32_t limit;
                                     memcpy(&limit, n_recvd[i] + 1, sizeof(uint32_t));
-                                    // printf("Limit: %x\n", limit);
                                     if (limit == 0)
                                         continue;
                                     timer.reset();
@@ -379,32 +417,206 @@ int dll_main(int argc, char** argv, Context* cxt){
                                 user_module_map.erase(it);
                             }
                             break;
+                        case 'N':
+                            {
+                                if(procedure_module_cursor < current_procedure.postproc_modules)
+                                    handle = current_procedure.__rt_loaded_modules[procedure_module_cursor++];
+                                printf("Load %i = %p\n", procedure_module_cursor, handle);
+                            }
+                            break;
+                        case 'R': //recorded procedure
+                            {
+                                auto proc_name = n_recvd[i] + 2;
+                                proc_name = *proc_name?proc_name : proc_name + 1;
+                                puts(proc_name);
+                                const auto& load_modules = [&](StoredProcedure &p) {
+                                    if (!p.__rt_loaded_modules){
+                                        p.__rt_loaded_modules = static_cast<void**>(
+                                            malloc(sizeof(void*) * p.postproc_modules));
+                                        for(uint32_t j = 0; j < p.postproc_modules; ++j){
+                                            auto pj = dlopen((procedure_root + p.name + std::to_string(j) + ".so").c_str(), RTLD_NOW);
+                                            if (pj == nullptr){
+                                                printf("Error: failed to load module %s\n", p.name);
+                                                return true;
+                                            }
+                                            aq_init_gc(pj, cxt);
+
+                                            p.__rt_loaded_modules[j] = pj;
+                                        }
+                                    }
+                                    return false;
+                                };
+                                const auto& save_proc_tofile = [&](const StoredProcedure& p)  {
+                                    auto config_name = procedure_root + procedure_name + ".aqp";
+                                    auto fp = fopen(config_name.c_str(), "wb");
+                                    if (fp == nullptr){
+                                        printf("Error: failed to open file %s\n", config_name.c_str());
+                                        return true;
+                                    }
+                                    fwrite(&p.cnt, sizeof(p.cnt), 1, fp);
+                                    fwrite(&p.postproc_modules, sizeof(p.postproc_modules), 1, fp);
+                                    for(uint32_t j = 0; j < p.cnt; ++j){
+                                        auto current_query = p.queries[j];
+                                        auto len_query = strlen(current_query);
+                                        fwrite(current_query, len_query + 1, 1, fp);
+                                    }
+                                    fclose(fp);
+                                    return false;
+                                };
+                                const auto& load_proc_fromfile = [&](StoredProcedure& p)  {
+                                    auto config_name = procedure_root + p.name + ".aqp";
+                                    puts(p.name);
+                                    auto fp = fopen(config_name.c_str(), "rb");
+                                    if(fp == nullptr){
+                                        puts("ERROR: Procedure not found on disk.");
+                                        return false;
+                                    }
+                                    fread(&p.cnt, sizeof(p.cnt), 1, fp);
+                                    fread(&p.postproc_modules, sizeof(p.postproc_modules), 1, fp);
+                                    auto offset_now = ftell(fp);
+                                    fseek(fp, 0, SEEK_END);
+                                    auto queries_size = ftell(fp) - offset_now;
+                                    fseek(fp, offset_now, SEEK_SET);
+
+                                    p.queries = static_cast<char**>(malloc(sizeof(char*) * p.cnt));
+                                    p.queries[0] = static_cast<char*>(malloc(sizeof(char) * queries_size));
+                                    fread(p.queries[0], 1, queries_size, fp);
+
+                                    for(uint32_t j = 1; j < p.cnt; ++j){
+                                        p.queries[j] = p.queries[j-1];
+                                        while(*(p.queries[j]) != '\0')
+                                            ++p.queries[j];
+                                        ++p.queries[j];
+                                        puts(p.queries[j-1]);
+                                    }
+                                    fclose(fp);
+                                    p.__rt_loaded_modules = 0;
+                                    return load_modules(p);
+                                };
+                                switch(n_recvd[i][1]){
+                                    case '\0':
+                                        current_procedure.name = copy_lpstr(proc_name);
+                                        AQ_ZeroMemory(current_procedure);
+                                        procedure_recording = true;
+                                        procedure_name = proc_name;
+                                    break;
+                                    case 'T':
+                                        current_procedure.queries = recorded_queries.container;
+                                        current_procedure.cnt = recorded_queries.size;
+                                        current_procedure.name = copy_lpstr(proc_name);
+                                        current_procedure.postproc_modules = recorded_libraries.size;
+                                        current_procedure.__rt_loaded_modules = recorded_libraries.container;
+                                        AQ_ZeroMemory(recorded_queries);
+                                        AQ_ZeroMemory(recorded_libraries);
+                                        procedure_recording = false;
+                                        save_proc_tofile(current_procedure);
+                                        cxt->stored_proc.insert_or_assign(procedure_name, current_procedure);
+                                        procedure_name = "";
+                                    break;
+                                    case 'E': // execute procedure
+                                    {
+                                        procedure_module_cursor = 0;
+                                        auto _proc = cxt->stored_proc.find(proc_name);
+                                        if (_proc == cxt->stored_proc.end()){
+                                            printf("Procedure %s not found. Trying load from disk.\n", proc_name);
+                                            current_procedure.name = copy_lpstr(proc_name);
+                                            if (!load_proc_fromfile(current_procedure)){
+                                                cxt->stored_proc.insert_or_assign(proc_name, current_procedure);
+                                            }
+                                            else {
+                                                continue;
+                                            }
+                                        }
+                                        else{
+                                            current_procedure = _proc->second;
+                                        }
+                                        n_recv = current_procedure.cnt;
+                                        n_recvd = current_procedure.queries;
+                                        load_modules(current_procedure);
+                                        procedure_replaying = true;
+                                        goto start; // yes, I know, refactor later!!
+                                    }
+                                    break;
+                                    case 'D': // delete procedure
+                                    break;
+                                    case 'S': // save procedure
+                                    break;
+                                    case 'L': // load procedure
+                                    if (!load_proc_fromfile(current_procedure)) {
+                                        cxt->stored_proc.insert_or_assign(proc_name, current_procedure);
+                                    }
+                                    break;
+                                    case 'd': // display all procedures
+                                    for(const auto& p : cxt->stored_proc){
+                                        printf("Procedure: %s, %d queries, %d modules:\n", p.first.c_str(), 
+                                            p.second.cnt, p.second.postproc_modules);
+                                        for(uint32_t j = 0; j < p.second.cnt; ++j){
+                                            printf("\tQuery %d: %s\n", j, p.second.queries[j]);
+                                        }
+                                        puts("");
+                                    }
+                                    break;
+                                }
+                            }
+                            break;
+                        case 'T': // triggers
+                        {
+                            switch(n_recvd[i][1]){
+                                case 'I': // register interval based trigger
+                                {
+                                    const char* action_name = n_recvd[i] + 2;
+                                    while(*action_name++);
+                                    if(auto p = get_procedure(cxt, action_name); p.name == nullptr) 
+                                        printf("Invalid action name: %s\n", action_name);
+                                    else {
+                                        auto action = AQ_DupObject(&p);
+                                        const char* interval = action_name;
+                                        while(*interval++);
+                                        const auto i_interval = getInt<uint32_t>(interval);
+                                        cxt->it_host->add_trigger(n_recvd[i] + 2, action, i_interval);
+                                    }
+                                }
+                                break;
+                                case 'C': // activate callback based trigger
+                                break;
+                                case 'R': // remove trigger
+                                {
+                                    cxt->it_host->remove_trigger(n_recvd[i] + 2);
+                                }
+                                break;
+                                default:
+                                printf("Corrupted message from prompt: %s\n", n_recvd[i]);
+                                break;
+                            }
+                        }
+                        break;
                         }
                     }
-                    if(handle) {
-                        dlclose(handle);
-                        handle = nullptr;
-                    }
+                    
                     printf("%lld, %lld", cfg->stats.monet_time, cfg->stats.postproc_time);
                     cxt->end_session();
                     n_recv = 0;
                 }
-                if(server->last_error == nullptr){
-                    // TODO: Add feedback to prompt.
-                }   
-                else{
+                if (server->last_error != nullptr) {
+                    printf("Monetdbe Error: %s\n", server->last_error);
                     server->last_error = nullptr;
                     //goto finalize;
-                } 
+                }   
             }
             
             // puts(cfg->has_dll ? "true" : "false");
             if (cfg->backend_type == BACKEND_AQuery) {
-                handle = dlopen("./dll.so", RTLD_LAZY);
+                handle = dlopen("./dll.so", RTLD_NOW);
                 code_snippet c = reinterpret_cast<code_snippet>(dlsym(handle, "dllmain"));
                 c(cxt);
             }
-            if (handle) dlclose(handle);
+            if (handle && 
+                !procedure_replaying && !procedure_recording) { 
+                printf("Destroy %p\n", handle);
+                dlclose(handle);
+                handle = nullptr;
+            }
+            procedure_replaying = false;
             cfg->new_query = 0;
         }
         //puts(cfg->running? "true": "false");
@@ -437,23 +649,23 @@ int launcher(int argc, char** argv){
     str = std::string("cd ") + pwd + std::string("&& python3 ./prompt.py ") + str;
     return system(str.c_str());
 }
-
+#if true || !defined(TESTMAIN) && !( defined(_MSC_VER) && defined(_DEBUG) )
 extern "C" int __DLLEXPORT__ main(int argc, char** argv) {
 #ifdef __AQ_BUILD_LAUNCHER__
    return launcher(argc, argv);
 #endif
    // puts("running");
    Context* cxt = new Context();
+   cxt->aquery_root_path = to_lpstr(std::filesystem::current_path().string());
    // cxt->log("%d %s\n", argc, argv[1]);
 
 #ifdef THREADING
     auto tp = new ThreadPool();
     cxt->thread_pool = tp;
+    cxt->it_host = new IntervalBasedTriggerHost(tp);
+    cxt->ct_host = new CallbackBasedTriggerHost(tp);
 #endif
     
-#ifdef __AQ_THREADED_GC__
-    cxt->gc_thread = new std::thread(gc_thread, cxt);
-#endif    
    const char* shmname;
    if (argc < 0)
         return dll_main(argc, argv, cxt);
@@ -472,7 +684,7 @@ extern "C" int __DLLEXPORT__ main(int argc, char** argv) {
        if(ready){
            cxt->log("running: %s\n", running? "true":"false");
            cxt->log("ready: %s\n", ready? "true":"false");
-           void* handle = dlopen("./dll.so", RTLD_LAZY);
+           void* handle = dlopen("./dll.so", RTLD_NOW);
            cxt->log("handle: %p\n", handle);
            if (handle) {
                cxt->log("inner\n");
@@ -490,3 +702,4 @@ extern "C" int __DLLEXPORT__ main(int argc, char** argv) {
    return 0;
 }
 
+#endif // MSCDBG

server/table.h

@@ -10,6 +10,7 @@
 #include <algorithm>
 #include <cstdarg>
 #include <vector>
+#include <string_view>
 #include "io.h"
 #include "hasher.h"
 
@@ -24,7 +25,7 @@ namespace types {
 	enum Type_t;
 	template <typename T>
 	struct Types;
-	template <class T1, class T2>
+	template <class T1, class T2, class ...Ts>
 	struct Coercion;
 }
 #endif
@@ -35,7 +36,8 @@ struct ColRef_cstorage {
 	int ty; // what if enum is not int?
 };
 
-template <template <class...> class VT, class T>
+template <template <class...> class VT, class T, 
+	std::enable_if_t<std::is_base_of_v<vector_base<T>, VT<T>>>* = nullptr>
 std::ostream& operator<<(std::ostream& os, const VT<T>& v)
 {
 	v.out();
@@ -77,7 +79,6 @@ public:
 	}
 	template<template <typename> class VT, typename T>
 	void initfrom(VT<T>&& v, const char* name = "") {
-		puts("rref");
 		ty = types::Types<_Ty>::getType();
 		this->size = v.size;
 		this->capacity = v.capacity;
@@ -87,7 +88,6 @@ public:
 	}
 	template<template <typename> class VT, typename T>
 	void initfrom(VT<T>& v, const char* name = "") {
-		puts("lref");
 		ty = types::Types<_Ty>::getType();
 		this->size = v.size;
 		this->capacity = 0;
@@ -97,7 +97,6 @@ public:
 	}
 	template<template <typename> class VT, typename T>
 	void initfrom(const VT<T>& v, const char* name = "") {
-		puts("constlref");
 		ty = types::Types<_Ty>::getType();
 		this->size = v.size;
 		this->capacity = 0;
@@ -145,13 +144,23 @@ public:
 		vector_type<_Ty>::operator=(vt);
 		return *this;
 	}
-	ColRef<_Ty>& operator =(ColRef<_Ty>&& vt) {
+	ColRef<_Ty>& operator =(ColRef<_Ty>&& vt) noexcept {
 		vector_type<_Ty>::operator=(std::move(vt));
 		return *this;
+	
 	}
-	ColView<_Ty> operator [](const vector_type<uint32_t>& idxs) const {
-		return ColView<_Ty>(*this, idxs);
+	 ColView<_Ty> operator [](vector_type<uint32_t>& idxs) const {
+	 	return ColView<_Ty>(*this, std::move(idxs));
 	}
+	 ColView<_Ty> operator [](const vector_type<uint32_t>& idxs) const {
+	 	return ColView<_Ty>(*this, idxs);
+	 }
+	//vector_type<_Ty> operator[](vector_type<uint32_t>& idxs) const {
+	//	vector_type<_Ty> ret(idxs.size);
+	//	for (uint32_t i = 0; i < idxs.size; ++i)
+	//		ret.container[i] = this->container[idxs[i]];
+	//	return ret;
+	//}
 	vector_type<_Ty> operator [](const std::vector<bool>& idxs) const {
 		vector_type<_Ty> ret (this->size);
 		uint32_t i = 0;
@@ -229,7 +238,7 @@ class ColView : public vector_base<_Ty> {
 public:
 	typedef ColRef<_Ty> Decayed_t;
 	const uint32_t size;
-	const ColRef<_Ty> orig;
+	const ColRef<_Ty>& orig;
 	vector_type<uint32_t> idxs;
 	ColView(const ColRef<_Ty>& orig, vector_type<uint32_t>&& idxs) : orig(orig), size(idxs.size), idxs(std::move(idxs)) {}
 	ColView(const ColRef<_Ty>& orig, const vector_type<uint32_t>& idxs) : orig(orig), idxs(idxs), size(idxs.size) {}
@@ -277,10 +286,12 @@ public:
 			ret[i] = orig[idxs[i]];
 		return ret;
 	}
+
 	ColView<_Ty> subvec(uint32_t start, uint32_t end) const {
 		uint32_t len = end - start;
 		return ColView<_Ty>(orig, idxs.subvec(start, end));
 	}
+
 	ColRef<_Ty> subvec_deep(uint32_t start, uint32_t end) const {
 		uint32_t len = end - start;
 		ColRef<_Ty> subvec(len);
@@ -321,7 +332,7 @@ template<class ...Types> struct TableInfo;
 template<class ...Types> struct TableView;
 
 template <long long _Index, bool order = true, class... _Types>
-constexpr inline auto& get(const TableInfo<_Types...>& table) noexcept {
+constexpr auto& get(const TableInfo<_Types...>& table) noexcept {
 	if constexpr (order)
 		return *(ColRef<std::tuple_element_t<_Index, std::tuple<_Types...>>> *) & (table.colrefs[_Index]);
 	else
@@ -329,7 +340,7 @@ constexpr inline auto& get(const TableInfo<_Types...>& table) noexcept {
 }
 
 template <long long _Index, class... _Types>
-constexpr inline ColRef<std::tuple_element_t<_Index, std::tuple<_Types...>>>& get(const TableView<_Types...>& table) noexcept {
+constexpr ColRef<std::tuple_element_t<_Index, std::tuple<_Types...>>>& get(const TableView<_Types...>& table) noexcept {
 	return *(ColRef<std::tuple_element_t<_Index, std::tuple<_Types...>>> *) & (table.info.colrefs[_Index]);
 }
 
@@ -340,9 +351,6 @@ struct is_vector_impl<ColView<V>> : std::true_type {};
 template <class V>
 struct is_vector_impl<vector_type<V>> : std::true_type {};
 
-template<class ...Types>
-struct TableView;
-
 template<class ...Types>
 struct TableInfo {
 	const char* name;
@@ -451,8 +459,7 @@ struct TableInfo {
 		std::string header_string = std::string();
 		for (uint32_t i = 0; i < sizeof...(Types); ++i)
 			header_string += std::string(this->colrefs[i].name) + sep + '|' + sep;
-		const size_t l_sep = strlen(sep) + 1;
-		if (header_string.size() - l_sep >= 0)
+		if (const size_t l_sep = strlen(sep) + 1; header_string.size() >= l_sep)
 			header_string.resize(header_string.size() - l_sep);
 		header_string += end + std::string(header_string.size(), '=') + end;
 		return header_string;
@@ -479,6 +486,7 @@ struct TableInfo {
 			if (header_string.size() - l_sep >= 0)
 				header_string.resize(header_string.size() - l_sep);
 		}
+		
 		const auto& prt_loop = [&fp, &view, &printf_string, *this, &limit](const auto& f) {
 #ifdef __AQ__HAS__INT128__			
 			constexpr auto num_hge = count_type<__int128_t, __uint128_t>((tuple_type*)(0));
@@ -523,7 +531,7 @@ struct TableInfo {
 		}
 	}
 	template <int ...vals> struct applier {
-		inline constexpr static void apply(const TableInfo<Types...>& t, const char* __restrict sep = ",", const char* __restrict end = "\n",
+		constexpr static void apply(const TableInfo<Types...>& t, const char* __restrict sep = ",", const char* __restrict end = "\n",
 			const vector_type<uint32_t>* __restrict view = nullptr, FILE* __restrict fp = nullptr, uint32_t limit = std::numeric_limits<uint32_t>::max()
 			) 
 		{
@@ -567,7 +575,7 @@ struct TableInfo {
 	TableInfo<Types ...>* distinct() {
 		std::unordered_set<tuple_type> d_records;
 		std::make_index_sequence<sizeof...(Types)> seq;
-
+		d_records.reserve(colrefs[0].size);
 		for (uint32_t j = 0; j < colrefs[0].size; ++j) {
 			d_records.insert(get_record(seq, j));
 		}
@@ -648,11 +656,11 @@ struct TableView {
 };
 
 template <class T>
-constexpr static inline bool is_vector(const ColRef<T>&) {
+constexpr static bool is_vector(const ColRef<T>&) {
 	return true;
 }
 template <class T>
-constexpr static inline bool is_vector(const vector_type<T>&) {
+constexpr static bool is_vector(const vector_type<T>&) {
 	return true;
 }
 
@@ -902,6 +910,42 @@ VT<bool> operator >(const T2& lhs, const VT<T1>& rhs) {
 	return ret;
 }
 
+#define _AQ_OP_(x) __AQ_OP__##x
+#define __AQ_OP__add +
+#define __AQ_OP__minus -
+#define __AQ_OP__mul *
+#define __AQ_OP__div /
+#define __AQ_OP__and &
+#define __AQ_OP__or |
+#define __AQ_OP__xor ^
+#define __AQ_OP__gt >
+#define __AQ_OP__lt <
+#define __AQ_OP__gte >=
+#define __AQ_OP__lte <=
+#define __AQ_OP__eq ==
+#define __AQ_OP__neq !=
+
+#define __D_AQOP(x) \
+template <class T1, class T2, template<typename> class VT, class Ret>\
+void aqop_##x (const VT<T1>& lhs, const VT<T2>& rhs, Ret& ret){\
+	for (uint32_t i = 0; i < ret.size; ++i)\
+		ret[i] = lhs[i] _AQ_OP_(x) rhs[i];\
+}
+
+__D_AQOP(add)
+__D_AQOP(minus)
+__D_AQOP(mul)
+__D_AQOP(div)
+__D_AQOP(and)
+__D_AQOP(or)
+__D_AQOP(xor)
+__D_AQOP(gt)
+__D_AQOP(lt)
+__D_AQOP(gte)
+__D_AQOP(lte)
+__D_AQOP(eq)
+__D_AQOP(neq)
+
 
 template <class ...Types>
 void print(const TableInfo<Types...>& v, const char* delimiter = " ", const char* endline = "\n") {
@@ -911,6 +955,7 @@ template <class ...Types>
 void print(const TableView<Types...>& v, const char* delimiter = " ", const char* endline = "\n") {
 	v.print(delimiter, endline);
 }
+
 template <class T>
 void print(const T& v, const char* delimiter = " ") {
 	std::cout << v << delimiter;
@@ -925,7 +970,6 @@ void print<__uint128_t>(const __uint128_t& v, const char* delimiter);
 #endif
 template <>
 void print<bool>(const bool& v, const char* delimiter);
-
 template <class T>
 void inline print_impl(const T& v, const char* delimiter, const char* endline) {
 	for (const auto& vi : v) {

server/table_ext_monetdb.hpp

@@ -28,7 +28,7 @@ inline constexpr monetdbe_types AQType_2_monetdbe[] = {
 #else	
 		monetdbe_int64_t,
 #endif
-		monetdbe_timestamp, monetdbe_int64_t, monetdbe_int64_t
+		monetdbe_timestamp, monetdbe_int8_t, monetdbe_str, monetdbe_int64_t, monetdbe_int64_t
 };
 
 template<class ...Ts>
@@ -75,12 +75,14 @@ void TableInfo<Ts ...>::monetdb_append_table(void* srv, const char* alt_name) {
 			auto err = monetdbe_append(*((monetdbe_database*)server->server), "sys", alt_name, monetdbe_cols, sizeof...(Ts));
 			if (err)
 				puts(err);
-			return;
+			goto finialize;
 		}
 	}
-	// for(uint32_t i = 0; i < n_vecs; ++i) 
-	// 		free(gc_vecs[i]);
 	puts("Error! Empty table.");
+
+finialize:	
+	for(uint32_t i = 0; i < cnt; ++i) 
+		;//GC::gc_handle->reg(gc_vecs[i]);
 }
 
 
@@ -92,6 +94,16 @@ void* ColRef<Type>::monetdb_get_col(void** gc_vecs, uint32_t& cnt) {
 	col->type = aq_type;
 	col->count = this->size;
 	col->data = this->container;
+
+	if constexpr(std::is_same_v<Type, std::string_view>) {
+		auto _data = static_cast<const char **>(
+			malloc(sizeof(const char *) * this->size));
+		for(uint32_t i = 0; i < this->size; ++i){
+			_data[i] = this->container[i].data();
+		}
+		col->data = _data;
+		gc_vecs[cnt++] = _data;
+	}
 	col->name = const_cast<char*>(this->name);
 	// auto arr = (types::timestamp_t*) malloc (sizeof(types::timestamp_t)* this->size);
 	// if constexpr (is_vector_type<Type>){
@@ -102,5 +114,4 @@ void* ColRef<Type>::monetdb_get_col(void** gc_vecs, uint32_t& cnt) {
 	// }
 	return col;
 }
-
 #endif

server/tests/thread_pool.hpp

@@ -4,21 +4,26 @@
 #include <thread>
 #include <cstdio>
 #include <cstdlib>
-using namespace std;
 
 FILE *fp;
 long long testing_throughput(uint32_t n_jobs, bool prompt = true){
+    using namespace std::chrono_literals;
     printf("Threadpool througput test with %u jobs. Press any key to start.\n", n_jobs);
 
-    auto tp = ThreadPool(thread::hardware_concurrency());
+    auto tp = ThreadPool(std::thread::hardware_concurrency());
     getchar();
     auto i = 0u;
-    fp = fopen("tmp.tmp", "w");
-    auto time = chrono::high_resolution_clock::now();
-    while(i++ < n_jobs) tp.enqueue_task({ [](void* f) {fprintf(fp, "%d ", *(int*)f); free(f); }, new int(i) });
+    fp = fopen("tmp.tmp", "wb");
+    auto time = std::chrono::high_resolution_clock::now();
+    while(i++ < n_jobs) {
+        payload_t payload;
+        payload.f = [](void* f) {fprintf(fp, "%d ", *(int*)f); free(f); return 0; };
+        payload.args = new int(i);
+        tp.enqueue_task(payload);
+    }
     puts("done dispatching.");
-    while (tp.busy()) this_thread::sleep_for(1s);
-    auto t = (chrono::high_resolution_clock::now() - time).count();
+    while (tp.busy()) std::this_thread::sleep_for(1s);
+    auto t = (std::chrono::high_resolution_clock::now() - time).count();
     printf("\nTr: %u, Ti: %lld \nThroughput: %lf transactions/ns\n", i, t, i/(double)(t));
     //this_thread::sleep_for(2s);
     fclose(fp);
@@ -27,48 +32,53 @@ long long testing_throughput(uint32_t n_jobs, bool prompt = true){
 
 long long testing_transaction(uint32_t n_burst, uint32_t n_batch, 
     uint32_t base_time, uint32_t var_time, bool prompt = true, FILE* _fp = stdout){
+    using namespace std::chrono_literals;
     printf("Threadpool transaction test: burst: %u, batch: %u, time: [%u, %u].\n"
          , n_burst, n_batch, base_time, var_time + base_time);
     if (prompt) {
         puts("Press any key to start.");
         getchar();
     }
-    auto tp = ThreadPool(thread::hardware_concurrency());
+    auto tp = ThreadPool(std::thread::hardware_concurrency());
     fp = _fp;
     auto i = 0u, j = 0u;
-    auto time = chrono::high_resolution_clock::now();
+    auto time = std::chrono::high_resolution_clock::now();
     while(j++ < n_batch){
         i = 0u;
-        while(i++ < n_burst) 
-            tp.enqueue_task({ [](void* f) { fprintf(fp, "%d ", *(int*)f); free(f); }, new int(j) });
+        while(i++ < n_burst) {
+            payload_t payload;
+            payload.f = [](void* f) { fprintf(fp, "%d ", *(int*)f); free(f); return 0; };
+            payload.args = new int(j);
+            tp.enqueue_task(payload);
+        }
         fflush(stdout);
-        this_thread::sleep_for(chrono::microseconds(rand()%var_time + base_time));
+        std::this_thread::sleep_for(std::chrono::microseconds(rand()%var_time + base_time));
     }
     puts("done dispatching.");
-    while (tp.busy()) this_thread::sleep_for(1s);
-    auto t = (chrono::high_resolution_clock::now() - time).count();
+    while (tp.busy()) std::this_thread::sleep_for(1s);
+    auto t = (std::chrono::high_resolution_clock::now() - time).count();
     printf("\nTr: %u, Ti: %lld \nThroughput: %lf transactions/ns\n", j*i, t, j*i/(double)(t));
     return t;
 
 }
 
 long long testing_destruction(bool prompt = true){
-    fp = fopen("tmp.tmp", "w");
+    fp = fopen("tmp.tmp", "wb");
     if (prompt) {
         puts("Press any key to start.");
         getchar();
     }
-    auto time = chrono::high_resolution_clock::now();
+    auto time = std::chrono::high_resolution_clock::now();
     for(int i = 0; i < 8; ++i)
         testing_transaction(0xfff, 0xff, 400, 100, false, fp);
     for(int i = 0; i < 64; ++i)
         testing_transaction(0xff, 0xf, 60, 20, false, fp);
     for(int i = 0; i < 1024; ++i) {
-        auto tp = new ThreadPool(256);
+        auto tp = new ThreadPool(255);
         delete tp; 
     }
     return 0;
-    auto t = (chrono::high_resolution_clock::now() - time).count();
+    auto t = (std::chrono::high_resolution_clock::now() - time).count();
     fclose(fp);
     return t;
 }

server/threading.cpp

@@ -1,4 +1,5 @@
 #include "threading.h"
+#include "libaquery.h"
 #include <thread>
 #include <atomic>
 #include <mutex>
@@ -105,7 +106,8 @@ ThreadPool::ThreadPool(uint32_t n_threads)
     current_payload = new payload_t[n_threads];
 
     for (uint32_t i = 0; i < n_threads; ++i){
-        atomic_init(tf + i, static_cast<unsigned char>(0b10));
+        // atomic_init(tf + i, static_cast<unsigned char>(0b10));
+        tf[i] = static_cast<unsigned char>(0b10);
         th[i] = thread(&ThreadPool::daemon_proc, this, i);
     }
 
@@ -152,21 +154,62 @@ bool ThreadPool::busy(){
     return true;
 }
 
-Trigger::Trigger(ThreadPool* tp){
-    
+IntervalBasedTriggerHost::IntervalBasedTriggerHost(ThreadPool* tp){
+    this->tp = tp;
+    this->triggers = new aq_map<std::string, IntervalBasedTrigger>;
+    trigger_queue_lock = new mutex();
+    this->now = std::chrono::high_resolution_clock::now().time_since_epoch().count();
 }
 
-void IntervalBasedTrigger::timer::reset(){
+void IntervalBasedTriggerHost::add_trigger(const char* name, StoredProcedure *p, uint32_t interval) {
+    auto tr = IntervalBasedTrigger{.interval = interval, .time_remaining = 0, .sp = p};
+    auto vt_triggers = static_cast<aq_map<std::string, IntervalBasedTrigger> *>(this->triggers);
+    trigger_queue_lock->lock();
+    vt_triggers->emplace(name, tr);
+    //(*vt_triggers)[name] = tr;
+    trigger_queue_lock->unlock();
+}
+
+void IntervalBasedTriggerHost::tick() {
+    const auto current_time = std::chrono::high_resolution_clock::now().time_since_epoch().count();
+    const auto delta_t = static_cast<uint32_t>((current_time - now) / 1000000); // miliseconds precision
+    now = current_time;
+    auto vt_triggers = static_cast<aq_map<std::string, IntervalBasedTrigger> *>(this->triggers);
+    trigger_queue_lock->lock();
+    for(auto& [_, t] : vt_triggers->values()) {
+        if(t.tick(delta_t)) {
+            payload_t payload;
+            payload.f = execTriggerPayload;
+            payload.args = static_cast<void*>(new StoredProcedurePayload {t.sp, cxt});
+
+            tp->enqueue_task(payload);
+        }
+    }
+    trigger_queue_lock->unlock();
+}
+
+void IntervalBasedTriggerHost::remove_trigger(const char* name) {
+    auto vt_triggers = static_cast<aq_map<std::string, IntervalBasedTrigger> *>(this->triggers);
+    vt_triggers->erase(name);
+}
+
+void IntervalBasedTrigger::reset() {
     time_remaining = interval;
 }
 
-bool IntervalBasedTrigger::timer::tick(uint32_t t){
-    if (time_remaining > t) {
-        time_remaining -= t;
-        return false;
-    }
-    else{
-        time_remaining = interval - t%interval;
-        return true;
-    }
-}
+bool IntervalBasedTrigger::tick(uint32_t delta_t) {
+    bool ret = false; 
+    if (time_remaining <= delta_t) 
+        ret = true; 
+    if (auto curr_dt = delta_t % interval; time_remaining <= curr_dt) 
+        time_remaining = interval  + time_remaining - curr_dt;
+    else 
+        time_remaining = time_remaining - curr_dt;
+    return ret;
+}
+
+CallbackBasedTriggerHost::CallbackBasedTriggerHost(ThreadPool *tp) {
+    this->tp = tp;
+}
+
+void CallbackBasedTriggerHost::tick() {}

server/threading.h

@@ -2,12 +2,14 @@
 #define _AQ_THREADING_H
 
 #include <stdint.h>
+#include <thread>
+#include <mutex>
 
 typedef int(*payload_fn_t)(void*);
 struct payload_t{
     payload_fn_t f;
     void* args;
-    constexpr payload_t(payload_fn_t f, void* args) noexcept
+    constexpr payload_t(payload_fn_t f, void* args) noexcept 
         : f(f), args(args) {}
     constexpr payload_t() noexcept
         : f(nullptr), args(nullptr) {};
@@ -19,7 +21,7 @@ struct payload_t{
 class ThreadPool{
 
 public:
-    ThreadPool(uint32_t n_threads = 0);
+    explicit ThreadPool(uint32_t n_threads = 0);
     void enqueue_task(const payload_t& payload);
     bool busy();
     virtual ~ThreadPool();
@@ -39,30 +41,47 @@ private:
 
 };
 
-class Trigger{
-private:
-    void* triggers;  //min-heap by t-rem
-    virtual void tick() = 0;
 
+class A_Semphore;
+
+class TriggerHost { 
 public:
-    Trigger(ThreadPool* tp);
+    void* triggers;  
+    std::thread* handle;
+    ThreadPool *tp;
+    Context* cxt;
+    std::mutex* trigger_queue_lock;
+
+    virtual void tick() = 0;
+    TriggerHost() = default;
+    virtual ~TriggerHost() = default;
 };
 
-class IntervalBasedTrigger : public Trigger{
+struct StoredProcedure;
+
+struct Trigger{};
+
+struct IntervalBasedTrigger : Trigger {
+    uint32_t interval; // in milliseconds
+    uint32_t time_remaining;
+    StoredProcedure* sp; 
+    void reset();
+    bool tick(uint32_t t); 
+};
+
+class IntervalBasedTriggerHost : public TriggerHost {
 public:
-    struct timer{
-        uint32_t interval; // in milliseconds
-        uint32_t time_remaining;
-        void reset();
-        bool tick(uint32_t t); 
-    };
-    void add_trigger();
+    explicit IntervalBasedTriggerHost(ThreadPool *tp);
+    void add_trigger(const char* name, StoredProcedure* stored_procedure, uint32_t interval);
+    void remove_trigger(const char* name);
 private:
+    unsigned long long now;
     void tick() override;
 };
 
-class CallbackBasedTrigger : public Trigger{
+class CallbackBasedTriggerHost : public TriggerHost {
 public:
+    explicit CallbackBasedTriggerHost(ThreadPool *tp);
     void add_trigger();
 private:
     void tick() override;

server/types.h

@@ -3,6 +3,9 @@
 #include <cstdint>
 #include <type_traits>
 #include <tuple>
+#include <string_view>
+#include <string>
+#include <utility>
 using std::size_t;
 
 #if  defined(__SIZEOF_INT128__) and not defined(_WIN32)
@@ -13,6 +16,9 @@ using std::size_t;
 #define __restrict__ __restrict
 #endif
 
+template<class T>
+struct vector_base {};
+
 template <class T>
 constexpr static inline bool is_vector(const T&) {
 	return false;
@@ -32,23 +38,23 @@ struct aqis_same_impl {
 		
 		std::conditional_t<
 			std::is_same_v<T1, bool> || std::is_same_v<T2, bool>, 
+			std::bool_constant<std::is_same_v<T1, bool> && std::is_same_v<T2, bool>>,
 			Cond(
-				(std::is_same_v<T1, bool> && std::is_same_v<T2, bool>), 
-				std::true_type, 
-				std::false_type
-			),
-			Cond(
-				std::is_signed_v<T1> == std::is_signed_v<T2>,
-				Cond(
-					std::is_floating_point_v<T1> == std::is_floating_point_v<T2>,
-					Cond(
-						aq_szof<T1> == aq_szof<T2>, // deal with sizeof(void)
-						std::true_type,
-						std::false_type
-					),
-					std::false_type
-				),
-				std::false_type
+                !(std::is_class_v<T1> || std::is_class_v<T2>),
+                Cond(
+                    std::is_signed_v<T1> == std::is_signed_v<T2>,
+                    Cond(
+                        std::is_floating_point_v<T1> == std::is_floating_point_v<T2>,
+                        std::bool_constant<aq_szof<T1> == aq_szof<T2>>, // deal with sizeof(void)
+                        std::false_type
+                    ),
+				    std::false_type
+                ),
+                Cond( 
+                    (std::is_class_v<T1> && std::is_class_v<T2>),
+                    std::bool_constant<(std::is_base_of_v<T1, T2> || std::is_base_of_v<T2, T1>)>,
+                    std::false_type
+                )
 			)
 		>::value;
 };
@@ -63,12 +69,12 @@ constexpr bool aqis_same<T1, T2> = aqis_same_impl<T1, T2>::value;
 namespace types {
 	enum Type_t {
 		AINT32, AFLOAT, ASTR, ADOUBLE, ALDOUBLE, AINT64, AINT128, AINT16, ADATE, ATIME, AINT8,
-		AUINT32, AUINT64, AUINT128, AUINT16, AUINT8, ABOOL, VECTOR, ATIMESTAMP, ACHAR, NONE, ERROR
+		AUINT32, AUINT64, AUINT128, AUINT16, AUINT8, ABOOL, VECTOR, ATIMESTAMP, ACHAR, ASV, NONE, ERROR
 	};
-	static constexpr const char* printf_str[] = { "%d", "%f", "%s", "%lf", "%Lf", "%ld", "%d", "%hi", "%s", "%s", "%hhd",
-		"%u", "%lu", "%s", "%hu", "%hhu", "%s", "%s", "Vector<%s>", "%s", "%c", "NULL", "ERROR" };
+	static constexpr const char* printf_str[] = { "%d", "%f", "%s", "%lf", "%Lf", "%ld", "%s", "%hi", "%s", "%s", "%hhd",
+		"%u", "%lu", "%s", "%hu", "%hhu", "%s", "Vector<%s>", "%s", "%c", "%s", "NULL", "ERROR" };
 	static constexpr const char* SQL_Type[] = { "INT", "REAL", "TEXT", "DOUBLE", "DOUBLE", "BIGINT", "HUGEINT", "SMALLINT", "DATE", "TIME", "TINYINT",
-		"INT", "BIGINT", "HUGEINT", "SMALLINT", "TINYINT", "BOOL", "HUGEINT", "TIMESTAMP", "CHAR", "NULL", "ERROR"};
+		"INT", "BIGINT", "HUGEINT", "SMALLINT", "TINYINT", "BOOL", "HUGEINT", "TIMESTAMP", "CHAR", "TEXT", "NULL", "ERROR"};
 	
 	
 	// TODO: deal with data/time <=> str/uint conversion
@@ -169,6 +175,8 @@ namespace types {
 		f(unsigned short, AUINT16) \
 		f(bool, ABOOL) \
 		f(timestamp_t, ATIMESTAMP) \
+		f(std::string_view, ASV) \
+		f(std::string, ASV) \
 		F_INT128(f)
 
 		inline constexpr static Type_t getType() {
@@ -385,9 +393,9 @@ template <class ...T1s, class ...T2s, template <class...> class T1>
 struct same_class_impl<T1<T1s...>, T1<T2s...>> : std::true_type {};
 
 template <class T1, class T2>
-bool same_class = same_class_impl<T1, T2>::value;
+constexpr bool same_class = same_class_impl<T1, T2>::value;
 template <class T1, template <class...> class T2>
-bool instance_of = instance_of_impl<T1, T2>::value;
+constexpr bool instance_of = instance_of_impl<T1, T2>::value;
 
 template <class lT, template <typename ...> class rT>
 using transTypes = typename transTypes_s<lT, rT>::type;
@@ -399,7 +407,6 @@ struct transValues_s<lT<vT, T...>, vT, rT> {
 	using type = rT<T...>;
 };
 
-#include <utility>
 template <class vT, int i, template <vT ...> class rT>
 using transValues = typename transValues_s<std::make_integer_sequence<vT, i>, vT, rT>::type;
 template <int i, template <int ...> class rT>
@@ -427,8 +434,17 @@ template <class ...T>
 using get_first = typename get_first_impl<T...>::first;
 template <class T>
 struct value_type_rec_impl { typedef T type; };
+
 template <template <class...> class VT, class ...V>
-struct value_type_rec_impl<VT<V...>> { typedef typename value_type_rec_impl<get_first<V...>>::type type; };
+struct value_type_rec_impl<VT<V...>> { 
+    typedef typename 
+    std::conditional_t<
+        std::is_base_of_v<vector_base<get_first<V...>>, VT<V...>>, 
+        typename value_type_rec_impl<get_first<V...>>::type, 
+        VT<V...>
+    > type; 
+};
+
 template <class T>
 using value_type_r = typename value_type_rec_impl<T>::type;
 

server/vector_type.hpp

@@ -15,12 +15,10 @@
 #include <iostream>
 #include "hasher.h"
 #include "types.h"
-
+#include "gc.h"
 #pragma pack(push, 1)
-template<class T>
-struct vector_base {};
 
-struct vectortype_cstorage{
+struct vectortype_cstorage {
 	void* container;
 	unsigned int size, capacity;
 };
@@ -32,11 +30,10 @@ public:
 	void inline _copy(const vector_type<_Ty>& vt) {
 		// quick init while using malloc
 		//if (capacity > 0) free(container);
-		
 		this->size = vt.size;
 		this->capacity = vt.capacity;
 		if (capacity) {
-			// puts("copy");
+			//puts("copy");
 			this->container = (_Ty*)malloc(size * sizeof(_Ty));
 			memcpy(container, vt.container, sizeof(_Ty) * size);
 		}
@@ -52,25 +49,36 @@ public:
 		this->container = vt.container;
 		// puts("move");
 		vt.size = vt.capacity = 0;	
-		vt.container = 0;
+		vt.container = nullptr;
 	}
 public:
 	_Ty* container;
 	uint32_t size, capacity;
 	typedef _Ty* iterator_t;
 	typedef std::conditional_t<is_cstr<_Ty>(), astring_view, _Ty> value_t;
-	vector_type(const uint32_t& size) : size(size), capacity(size) {
-		container = (_Ty*)malloc(size * sizeof(_Ty));
+	explicit vector_type(const uint32_t& size) : size(size), capacity(size) {
+		if (GC::scratch_space != nullptr) {
+			[[likely]]
+			container = (_Ty*)GC::scratch_space->alloc(size * sizeof(_Ty));
+			this->capacity = 0;
+		}
+		else{
+			container = (_Ty*)malloc(size * sizeof(_Ty));
+		}
+		// TODO: calloc for objects. 
 	}
-	constexpr vector_type(std::initializer_list<_Ty> _l) {
+	explicit constexpr vector_type(std::initializer_list<_Ty> _l) {
 		size = capacity = _l.size();
-		_Ty* _container = this->container = (_Ty*)malloc(sizeof(_Ty) * _l.size());
+		this->container = (_Ty*)malloc(sizeof(_Ty) * capacity);
+		_Ty* _container = this->container;
 		for (const auto& l : _l) {
 			*(_container++) = l;
 		}
 	}
 	constexpr vector_type() noexcept : size(0), capacity(0), container(0) {};
 	constexpr vector_type(_Ty* container, uint32_t len) noexcept : size(len), capacity(0), container(container) {};
+	vector_type(const char** container, uint32_t len, 
+		typename std::enable_if_t<!std::is_same_v<_Ty, const char*>>* = nullptr) noexcept = delete;
 	constexpr explicit vector_type(const vector_type<_Ty>& vt) noexcept : capacity(0) {
 		_copy(vt);
 	}
@@ -80,13 +88,20 @@ public:
 	constexpr vector_type(vector_type<_Ty>&& vt) noexcept : capacity(0) {
 		_move(std::move(vt));
 	}
-	vector_type(vectortype_cstorage vt) noexcept : capacity(vt.capacity), size(vt.size), container((_Ty*)vt.container) {
-		out(10);
+	explicit vector_type(vectortype_cstorage vt) noexcept : 
+		capacity(vt.capacity), size(vt.size), container((_Ty*)vt.container) {
+		// out(10);
 	};
 	// size >= capacity ==> readonly vector
-	constexpr vector_type(const uint32_t size, void* data) : 
+	vector_type(const uint32_t size, void* data) : 
 		size(size), capacity(0), container(static_cast<_Ty*>(data)) {}
 
+	void init_from(const uint32_t size, void* data) {
+		this->container = static_cast<_Ty*>(data);
+		this->size = size;
+		this->capacity = 0;
+	}
+	
 	vector_type<_Ty>& operator =(const _Ty& vt) {
 		if (!container) { 
 			container = (_Ty*)malloc(sizeof(_Ty)); 
@@ -108,7 +123,7 @@ public:
 		_copy(vt);
 		return *this;
 	}
-	vector_type<_Ty>& operator =(vector_type<_Ty>&& vt) {
+	vector_type<_Ty>& operator =(vector_type<_Ty>&& vt) noexcept {
 		_move(std::move(vt));
 		return *this;
 	}
@@ -124,16 +139,16 @@ public:
 		
 		return *this;
 	}
-	inline std::unordered_set<value_t> distinct_common(){
+	inline std::unordered_set<value_t> distinct_common() {
 		return std::unordered_set<value_t>(container, container + size);
 	}
-	vector_type<_Ty>& distinct_inplace(){
+	vector_type<_Ty>& distinct_inplace() {
 		uint32_t i = 0;
 		for(const auto& v : distinct_common()){
 			container[i++] = v;
 		}
 		return *this;
-	}
+	} 
 	vector_type<_Ty> distinct_copy(){
 		auto d_vals = distinct_common();
 		vector_type<_Ty> ret(d_vals.size());
@@ -152,27 +167,44 @@ public:
 		else
 			return distinct_copy();
 	}
-	inline void grow() {
-		if (size >= capacity) { // geometric growth
-			uint32_t new_capacity = size + 1 + (size >> 1);
-			_Ty* n_container = (_Ty*)malloc(new_capacity * sizeof(_Ty));
-			memcpy(n_container, container, sizeof(_Ty) * size);
+	// TODO: think of situations where this is a temp!! (copy on write!!!)
+	template <bool _grow = true>
+	inline void grow(uint32_t sz = 0) {
+		if constexpr (_grow)
+			sz = this->size;
+		if (sz >= capacity) { // geometric growth
+			uint32_t new_capacity;
+			if constexpr (_grow)
+				new_capacity = size + 1 + (size >> 1);
+			else	
+				new_capacity = sz;
+
+			_Ty* n_container = (_Ty*)realloc(container, new_capacity * sizeof(_Ty));
+			// memcpy(n_container, container, sizeof(_Ty) * size);
 			memset(n_container + size, 0, sizeof(_Ty) * (new_capacity - size));
-			if (capacity)
-				free(container);
+			// if (capacity)
+			// 	free(container);
 			container = n_container;
 			capacity = new_capacity;
 		}
 	}
-	void emplace_back(const _Ty& _val) {
+	inline void resize(const uint32_t sz){
+		size = sz;
+		grow<false>(sz);
+	}
+	inline void reserve(const uint32_t sz){
+		grow<false>(sz);
+	}
+
+	inline void emplace_back(const _Ty& _val) {
 		grow();
 		container[size++] = _val;
 	}
-	void emplace_back(_Ty& _val) {
+	inline void emplace_back(_Ty& _val) {
 		grow();
 		container[size++] = std::move(_val);
 	}
-	void emplace_back(_Ty&& _val) {
+	inline void emplace_back(_Ty&& _val) {
 		grow();
 		container[size++] = std::move(_val);
 	}
@@ -189,10 +221,10 @@ public:
 		return _it;
 	}
 
-	iterator_t begin() const {
+	inline iterator_t begin() const {
 		return container;
 	}
-	iterator_t end() const {
+	inline iterator_t end() const {
 		return container + size;
 	}
 
@@ -206,7 +238,7 @@ public:
 		return container[_i];
 	}
 
-	void shrink_to_fit() {
+	inline void shrink_to_fit() {
 		if (size && capacity != size) {
 			capacity = size;
 			_Ty* _container = (_Ty*)malloc(sizeof(_Ty) * size);
@@ -216,13 +248,17 @@ public:
 		}
 	}
 
-	_Ty& back() {
+	inline void clear() {
+		this->size = 0;
+	}
+
+	inline _Ty& back() {
 		return container[size - 1];
 	}
-	void qpop() {
+	inline void qpop() {
 		size = size ? size - 1 : size;
 	}
-	void pop_resize() {
+	inline void pop_resize() {
 		if (size) {
 			--size;
 			if (capacity > (size << 1))
@@ -235,7 +271,7 @@ public:
 			}
 		}
 	}
-	_Ty pop() {
+	inline _Ty pop() {
 		return container[--size];
 	}
 	void merge(vector_type<_Ty>& _other) {
@@ -307,8 +343,9 @@ public:
 	inline vector_type<_Ty> subvec_deep(uint32_t start = 0) const { return subvec_deep(start, size); }
 	vector_type<_Ty> getRef() { return vector_type<_Ty>(container, size); }
 	~vector_type() {
-		if (capacity > 0) free(container);
-		container = 0; size = capacity = 0;
+		if (capacity > 0) GC::gc_handle->reg(container, sizeof(_Ty) * capacity);//free(container);
+		container = nullptr; 
+		size = capacity = 0;
 	}
 #define Compare(_op) \
 	template<typename T> \
@@ -345,7 +382,7 @@ public:
 #define Ops(o, x) \
 	template<typename T>\
 	vector_type<typename types::Coercion<_Ty, T>::type> operator o (const vector_type<T>& r) const {\
-		/*[[likely]] if (r.size == size) {*/\
+		/*if (r.size == size) { [[likely]] */\
 			return x(r);\
 		/*}*/\
 	}
@@ -353,7 +390,7 @@ public:
 #define Opseq(o, x) \
 	template<typename T>\
 	vector_type<typename types::Coercion<_Ty, T>::type> operator o##= (const vector_type<T>& r) {\
-		/*[[likely]] if (r.size == size) {*/\
+		/*if (r.size == size) { [[likely]] */\
 			return x##eq(r);\
 		/*}*/\
 	}
@@ -372,6 +409,30 @@ public:
 };
 
 
+// template<>
+// void vector_type<std::string_view>::init_from(const uint32_t size, void* data) {
+// 	this->size = this->capacity = size;
+// 	this->container = static_cast<std::string_view*>(
+// 		malloc(sizeof(std::string_view) * size));
+// 	for(uint32_t i = 0; i < size; ++i){
+// 		this->container[i] = container[i];
+// 	}
+// }
+
+// template<template <typename> class VT>
+// inline void 
+// prealloc_vector (VT &vt, uint32_t sz) { 
+// 	vt.reserve(sz); 
+// }
+
+// template<class T>
+// inline void 
+// prealloc_vector (vector_type<vector_type<T>> &vt, 
+// 				uint32_t outer_sz, uint32_t inner_sz) { 
+// 	vt.reserve(outer_sz); 
+// 	auto mem = static_cast<T*>(malloc(inner_sz * sizeof(T)));
+// }
+
 template <>
 class vector_type<void> {
 public:
@@ -404,4 +465,55 @@ public:
 	vector_type<void> subvec_deep(uint32_t);
 };
 #pragma pack(pop)
+
+template <class Key, class Hash>
+class AQHashTable : public ankerl::unordered_dense::set<Key, Hash> {
+public:
+	uint32_t* reversemap, *mapbase, *ht_base;
+	AQHashTable() = default;
+	explicit AQHashTable(uint32_t sz) 
+		: ankerl::unordered_dense::set<Key, Hash>{} {
+		this->reserve(sz);
+		this->m_values.reserve(sz);
+		reversemap = static_cast<uint32_t *>(malloc(sizeof(uint32_t) * sz * 2));
+		mapbase = reversemap + sz;
+		ht_base =  static_cast<uint32_t *>(calloc(sz, sizeof(uint32_t)));
+	}
+
+	void init(uint32_t sz) {
+		ankerl::unordered_dense::set<Key, Hash>::reserve(sz);
+		reversemap = static_cast<uint32_t *>(malloc(sizeof(uint32_t) * sz * 2));
+		mapbase = reversemap + sz;
+		ht_base =  static_cast<uint32_t *>(calloc(sz, sizeof(uint32_t)));
+	}
+
+	inline void hashtable_push(Key&& k, uint32_t i){
+		reversemap[i] = ankerl::unordered_dense::set<Key, Hash>::hashtable_push(std::move(k));
+		++ht_base[reversemap[i]];
+	}
+
+	auto ht_postproc(uint32_t sz) {
+		auto& arr_values = this->values();
+		const auto& len = this->size();
+
+		auto vecs = static_cast<vector_type<uint32_t>*>(malloc(sizeof(vector_type<uint32_t>) * len));
+		vecs[0].init_from(ht_base[0], mapbase);
+		for (uint32_t i = 1; i < len; ++i) {
+			vecs[i].init_from(ht_base[i], mapbase + ht_base[i - 1]);
+			ht_base[i] += ht_base[i - 1];
+		}
+		for (uint32_t i = 0; i < sz; ++i) {
+			auto id = reversemap[i];
+			mapbase[--ht_base[id]] = i;    
+		}
+		return vecs;
+	}
+};
+
+template<>
+vector_type<std::string_view>::vector_type(const uint32_t size, void* data);
+template <>
+vector_type<std::string_view>::vector_type(const char** container, uint32_t len, 
+		typename std::enable_if_t<true>*) noexcept;
+
 #endif

server/winhelper.h

@@ -1,7 +1,8 @@
 #ifndef _WINHELPER_H
 #define _WINHELPER_H
+
 #ifdef _WIN32
-static constexpr int RTLD_LAZY = 1;
+static constexpr int RTLD_LAZY = 1, RTLD_NOW = 1;
 void* dlopen(const char*, int);
 void* dlsym(void*, const char*);
 int dlclose(void*);
@@ -15,16 +16,6 @@ struct SharedMemory
     void FreeMemoryMap();
 };
 
-#ifndef __USE_STD_SEMAPHORE__
-class A_Semaphore {
-private:
-	void* native_handle;
-public:
-	A_Semaphore();
-	void acquire();
-	void release();
-	~A_Semaphore();
-};
-#endif
+#endif // WIN32
 
-#endif
+#endif // WINHELPER

tests/dt2.a

@@ -5,18 +5,20 @@ LOAD MODULE FROM "./libirf.so"
      predict(X:vecvecdouble) -> vecint
  );
 
- create table source(x1 double, x2 double, x3 double, x4 double, x5 int64);
- load data infile "data/benchmark" into table source fields terminated by ",";
+create table source(x1 double, x2 double, x3 double, x4 double, x5 int64);
+-- Create trigger 1 ~~ to predict whenever sz(source > ?)
+-- Create trigger 2 ~~ to auto feed ~
+load data infile "data/benchmark" into table source fields terminated by ",";
 
- create table sparse(x int);
- insert into sparse values (1);
- insert into sparse values (1);
- insert into sparse values (1);
- insert into sparse values (1);
+create table sparse(x int);
+insert into sparse values (1);
+insert into sparse values (1);
+insert into sparse values (1);
+insert into sparse values (1);
 
- select newtree(6, 4, sparse.x, 0, 4, 2, 0, 400, 2147483647) from sparse
+select newtree(6, 4, sparse.x, 0, 4, 2, 0, 400, 2147483647) from sparse
 
- select fit(pack(x1, x2, x3, x4), x5) from source
+select fit(pack(x1, x2, x3, x4), x5) from source
 
 -- select pack(x1, x2, x3, x4) from source
-  select predict(pack(x1, x2, x3, x4)) from source
+select predict(pack(x1, x2, x3, x4)) from source