From 6adb7900cbe98a14b98c81375ab1ec74dc2d4fe7 Mon Sep 17 00:00:00 2001 From: taozizhuo Date: Mon, 5 Dec 2022 14:28:29 -0800 Subject: [PATCH 1/7] add benchmark queries --- benchmark/quries/Aquery/load_data.a | 6 ++++++ benchmark/quries/Aquery/q0.a | 5 +++++ benchmark/quries/Aquery/q1.a | 7 +++++++ benchmark/quries/Aquery/q10.a | 4 ++++ benchmark/quries/Aquery/q2.a | 4 ++++ benchmark/quries/Aquery/q3.a | 7 +++++++ benchmark/quries/Aquery/q4.a | 5 +++++ benchmark/quries/Aquery/q7.a | 5 +++++ benchmark/quries/Aquery/q8.a | 6 ++++++ benchmark/quries/Aquery/q9.a | 6 ++++++ benchmark/quries/Clickhouse/q0 | 3 +++ benchmark/quries/Clickhouse/q1 | 4 ++++ benchmark/quries/Clickhouse/q10 | 8 ++++++++ benchmark/quries/Clickhouse/q2 | 2 ++ benchmark/quries/Clickhouse/q3 | 4 ++++ benchmark/quries/Clickhouse/q4 | 2 ++ benchmark/quries/Clickhouse/q7 | 5 +++++ benchmark/quries/Clickhouse/q8 | 3 +++ benchmark/quries/Clickhouse/q9 | 3 +++ benchmark/quries/Timescaledb/q0 | 3 +++ benchmark/quries/Timescaledb/q1 | 4 ++++ benchmark/quries/Timescaledb/q10 | 7 +++++++ benchmark/quries/Timescaledb/q2 | 2 ++ benchmark/quries/Timescaledb/q3 | 4 ++++ benchmark/quries/Timescaledb/q4 | 2 ++ benchmark/quries/Timescaledb/q7 | 5 +++++ benchmark/quries/Timescaledb/q8 | 3 +++ benchmark/quries/Timescaledb/q9 | 3 +++ 28 files changed, 122 insertions(+) create mode 100644 benchmark/quries/Aquery/load_data.a create mode 100644 benchmark/quries/Aquery/q0.a create mode 100644 benchmark/quries/Aquery/q1.a create mode 100644 benchmark/quries/Aquery/q10.a create mode 100644 benchmark/quries/Aquery/q2.a create mode 100644 benchmark/quries/Aquery/q3.a create mode 100644 benchmark/quries/Aquery/q4.a create mode 100644 benchmark/quries/Aquery/q7.a create mode 100644 benchmark/quries/Aquery/q8.a create mode 100644 benchmark/quries/Aquery/q9.a create mode 100644 benchmark/quries/Clickhouse/q0 create mode 100644 benchmark/quries/Clickhouse/q1 create mode 100644 benchmark/quries/Clickhouse/q10 create mode 100644 benchmark/quries/Clickhouse/q2 create mode 100644 benchmark/quries/Clickhouse/q3 create mode 100644 benchmark/quries/Clickhouse/q4 create mode 100644 benchmark/quries/Clickhouse/q7 create mode 100644 benchmark/quries/Clickhouse/q8 create mode 100644 benchmark/quries/Clickhouse/q9 create mode 100644 benchmark/quries/Timescaledb/q0 create mode 100644 benchmark/quries/Timescaledb/q1 create mode 100644 benchmark/quries/Timescaledb/q10 create mode 100644 benchmark/quries/Timescaledb/q2 create mode 100644 benchmark/quries/Timescaledb/q3 create mode 100644 benchmark/quries/Timescaledb/q4 create mode 100644 benchmark/quries/Timescaledb/q7 create mode 100644 benchmark/quries/Timescaledb/q8 create mode 100644 benchmark/quries/Timescaledb/q9 diff --git a/benchmark/quries/Aquery/load_data.a b/benchmark/quries/Aquery/load_data.a new file mode 100644 index 0000000..54bc36f --- /dev/null +++ b/benchmark/quries/Aquery/load_data.a @@ -0,0 +1,6 @@ +CREATE TABLE trade01m(stocksymbol STRING, time INT, quantity INT, price INT) +load data infile "../tables/trade01m.csv" into table trade01m fields terminated by ',' +CREATE TABLE trade1m(stocksymbol STRING, time INT, quantity INT, price INT) +load data infile "../tables/trade1m.csv" into table trade1m fields terminated by ',' +CREATE TABLE trade10m(stocksymbol STRING, time INT, quantity INT, price INT) +load data infile "../tables/trade10m.csv" into table trade10m fields terminated by ',' \ No newline at end of file diff --git a/benchmark/quries/Aquery/q0.a b/benchmark/quries/Aquery/q0.a new file mode 100644 index 0000000..a18deec --- /dev/null +++ b/benchmark/quries/Aquery/q0.a @@ -0,0 +1,5 @@ +-- select rows + +CREATE TABLE res0 AS +SELECT * FROM trade10m + \ No newline at end of file diff --git a/benchmark/quries/Aquery/q1.a b/benchmark/quries/Aquery/q1.a new file mode 100644 index 0000000..f3077a9 --- /dev/null +++ b/benchmark/quries/Aquery/q1.a @@ -0,0 +1,7 @@ +-- groupby_multi_different_functions + +CREATE TABLE res1 AS +SELECT avg(quantity) AS avg_quan, min(price) AS min_p +FROM trade1m +GROUP BY stocksymbol, time + \ No newline at end of file diff --git a/benchmark/quries/Aquery/q10.a b/benchmark/quries/Aquery/q10.a new file mode 100644 index 0000000..8c891ba --- /dev/null +++ b/benchmark/quries/Aquery/q10.a @@ -0,0 +1,4 @@ +SELECT stocksymbol, MAX(stddevs(3, price)) +FROM trade1m +ASSUMING ASC time +GROUP BY stocksymbol \ No newline at end of file diff --git a/benchmark/quries/Aquery/q2.a b/benchmark/quries/Aquery/q2.a new file mode 100644 index 0000000..28e6368 --- /dev/null +++ b/benchmark/quries/Aquery/q2.a @@ -0,0 +1,4 @@ +-- count values + +SELECT COUNT(*) FROM trade10m + \ No newline at end of file diff --git a/benchmark/quries/Aquery/q3.a b/benchmark/quries/Aquery/q3.a new file mode 100644 index 0000000..c6f7a5b --- /dev/null +++ b/benchmark/quries/Aquery/q3.a @@ -0,0 +1,7 @@ +-- group by multiple keys + +create table res3 AS +SELECT sum(quantity) as sum_quantity +FROM trade01m +GROUP BY stocksymbol, price + \ No newline at end of file diff --git a/benchmark/quries/Aquery/q4.a b/benchmark/quries/Aquery/q4.a new file mode 100644 index 0000000..bab175f --- /dev/null +++ b/benchmark/quries/Aquery/q4.a @@ -0,0 +1,5 @@ +-- append tables + +CREATE TABLE res4 AS +SELECT * FROM trade10m UNION ALL SELECT * FROM trade10m + \ No newline at end of file diff --git a/benchmark/quries/Aquery/q7.a b/benchmark/quries/Aquery/q7.a new file mode 100644 index 0000000..7e384c8 --- /dev/null +++ b/benchmark/quries/Aquery/q7.a @@ -0,0 +1,5 @@ +CREATE table res7 AS +SELECT stocksymbol, avgs(5, price) +FROM trade10m +ASSUMING ASC time +GROUP BY stocksymbol \ No newline at end of file diff --git a/benchmark/quries/Aquery/q8.a b/benchmark/quries/Aquery/q8.a new file mode 100644 index 0000000..6642520 --- /dev/null +++ b/benchmark/quries/Aquery/q8.a @@ -0,0 +1,6 @@ + +CREATE TABLE res8 AS +SELECT stocksymbol, quantity, price +FROM trade10m +WHERE time >= 5288 and time <= 7000 + \ No newline at end of file diff --git a/benchmark/quries/Aquery/q9.a b/benchmark/quries/Aquery/q9.a new file mode 100644 index 0000000..7348b8e --- /dev/null +++ b/benchmark/quries/Aquery/q9.a @@ -0,0 +1,6 @@ + +CREATE TABLE res9 AS +SELECT stocksymbol, MAX(price) - MIN(price) +FROM trade10m +GROUP BY stocksymbol + \ No newline at end of file diff --git a/benchmark/quries/Clickhouse/q0 b/benchmark/quries/Clickhouse/q0 new file mode 100644 index 0000000..e06e534 --- /dev/null +++ b/benchmark/quries/Clickhouse/q0 @@ -0,0 +1,3 @@ +-- q0 select rows +CREATE TABLE res0 (a String, b Int32, c Int32, d Int32) ENGINE = MergeTree() ORDER BY b AS +SELECT * FROM benchmark.trade10m \ No newline at end of file diff --git a/benchmark/quries/Clickhouse/q1 b/benchmark/quries/Clickhouse/q1 new file mode 100644 index 0000000..21ef83b --- /dev/null +++ b/benchmark/quries/Clickhouse/q1 @@ -0,0 +1,4 @@ +-- groupby_multi_different_functions +SELECT avg(quantity), min(price) +FROM benchmark.trade10m +GROUP BY stocksymbol, time \ No newline at end of file diff --git a/benchmark/quries/Clickhouse/q10 b/benchmark/quries/Clickhouse/q10 new file mode 100644 index 0000000..c251cb6 --- /dev/null +++ b/benchmark/quries/Clickhouse/q10 @@ -0,0 +1,8 @@ +-- max rolling std +select + stocksymbol, + max(stddevPop(price)) over + (partition by stocksymbol rows between 2 preceding AND CURRENT row) as maxRollingStd +from +(SELECT * FROM benchmark.trade01m ORDER BY time) +GROUP BY stocksymbol \ No newline at end of file diff --git a/benchmark/quries/Clickhouse/q2 b/benchmark/quries/Clickhouse/q2 new file mode 100644 index 0000000..1267934 --- /dev/null +++ b/benchmark/quries/Clickhouse/q2 @@ -0,0 +1,2 @@ +-- count values +SELECT COUNT(*) FROM benchmark.trade10m \ No newline at end of file diff --git a/benchmark/quries/Clickhouse/q3 b/benchmark/quries/Clickhouse/q3 new file mode 100644 index 0000000..79ea85e --- /dev/null +++ b/benchmark/quries/Clickhouse/q3 @@ -0,0 +1,4 @@ +-- group by multiple keys +SELECT sum(quantity) +FROM benchmark.trade10m +GROUP BY stocksymbol, price \ No newline at end of file diff --git a/benchmark/quries/Clickhouse/q4 b/benchmark/quries/Clickhouse/q4 new file mode 100644 index 0000000..016f3fc --- /dev/null +++ b/benchmark/quries/Clickhouse/q4 @@ -0,0 +1,2 @@ +-- append two tables +SELECT * FROM benchmark.trade10m UNION ALL SELECT * FROM benchmark.trade10m \ No newline at end of file diff --git a/benchmark/quries/Clickhouse/q7 b/benchmark/quries/Clickhouse/q7 new file mode 100644 index 0000000..ed57058 --- /dev/null +++ b/benchmark/quries/Clickhouse/q7 @@ -0,0 +1,5 @@ +-- moving_avg +SELECT stocksymbol, groupArrayMovingAvg(5)(price) AS moving_avg_price +FROM +(SELECT * FROM benchmark.trade01m ORDER BY time) +GROUP BY stocksymbol \ No newline at end of file diff --git a/benchmark/quries/Clickhouse/q8 b/benchmark/quries/Clickhouse/q8 new file mode 100644 index 0000000..550abbd --- /dev/null +++ b/benchmark/quries/Clickhouse/q8 @@ -0,0 +1,3 @@ +SELECT stocksymbol, quantity, price +FROM benchmark.trade10m +WHERE time >= 5288 and time <= 7000 \ No newline at end of file diff --git a/benchmark/quries/Clickhouse/q9 b/benchmark/quries/Clickhouse/q9 new file mode 100644 index 0000000..48312c9 --- /dev/null +++ b/benchmark/quries/Clickhouse/q9 @@ -0,0 +1,3 @@ +SELECT stocksymbol, MAX(price) - MIN(price) +FROM benchmark.trade1m +GROUP BY stocksymbol \ No newline at end of file diff --git a/benchmark/quries/Timescaledb/q0 b/benchmark/quries/Timescaledb/q0 new file mode 100644 index 0000000..b6dec8f --- /dev/null +++ b/benchmark/quries/Timescaledb/q0 @@ -0,0 +1,3 @@ +-- select rows +CREATE TABLE res0 AS +SELECT * FROM trade10m; \ No newline at end of file diff --git a/benchmark/quries/Timescaledb/q1 b/benchmark/quries/Timescaledb/q1 new file mode 100644 index 0000000..0ac4c46 --- /dev/null +++ b/benchmark/quries/Timescaledb/q1 @@ -0,0 +1,4 @@ +-- groupby_multi_different_functions +SELECT avg(quantity), min(price) +FROM trade10m +GROUP BY stocksymbol, time; \ No newline at end of file diff --git a/benchmark/quries/Timescaledb/q10 b/benchmark/quries/Timescaledb/q10 new file mode 100644 index 0000000..6d4b326 --- /dev/null +++ b/benchmark/quries/Timescaledb/q10 @@ -0,0 +1,7 @@ +select + stocksymbol, + max(stddev(price)) over + (partition by stocksymbol rows between 2 preceding AND CURRENT row) as maxRollingStd +from +(SELECT * FROM trade01m ORDER BY time) as t +GROUP BY stocksymbol; \ No newline at end of file diff --git a/benchmark/quries/Timescaledb/q2 b/benchmark/quries/Timescaledb/q2 new file mode 100644 index 0000000..b1f00f6 --- /dev/null +++ b/benchmark/quries/Timescaledb/q2 @@ -0,0 +1,2 @@ +-- count values +SELECT COUNT(*) FROM trade10m; \ No newline at end of file diff --git a/benchmark/quries/Timescaledb/q3 b/benchmark/quries/Timescaledb/q3 new file mode 100644 index 0000000..0176182 --- /dev/null +++ b/benchmark/quries/Timescaledb/q3 @@ -0,0 +1,4 @@ +-- group by multiple keys +SELECT sum(quantity) +FROM trade10m +GROUP BY stocksymbol, price; \ No newline at end of file diff --git a/benchmark/quries/Timescaledb/q4 b/benchmark/quries/Timescaledb/q4 new file mode 100644 index 0000000..a3e7f14 --- /dev/null +++ b/benchmark/quries/Timescaledb/q4 @@ -0,0 +1,2 @@ +-- append tables +SELECT * FROM trade10m UNION ALL SELECT * FROM trade10m; \ No newline at end of file diff --git a/benchmark/quries/Timescaledb/q7 b/benchmark/quries/Timescaledb/q7 new file mode 100644 index 0000000..c0aa976 --- /dev/null +++ b/benchmark/quries/Timescaledb/q7 @@ -0,0 +1,5 @@ +select + stocksymbol, + coalesce(avg(price) over + (partition by stocksymbol order by time rows between 4 preceding AND CURRENT row), price) as rollingAvg +from trade10m; \ No newline at end of file diff --git a/benchmark/quries/Timescaledb/q8 b/benchmark/quries/Timescaledb/q8 new file mode 100644 index 0000000..db6be13 --- /dev/null +++ b/benchmark/quries/Timescaledb/q8 @@ -0,0 +1,3 @@ +SELECT stocksymbol, quantity, price +FROM trade01m +WHERE time >= 5288 and time <= 7000 \ No newline at end of file diff --git a/benchmark/quries/Timescaledb/q9 b/benchmark/quries/Timescaledb/q9 new file mode 100644 index 0000000..e8c0b92 --- /dev/null +++ b/benchmark/quries/Timescaledb/q9 @@ -0,0 +1,3 @@ +SELECT stocksymbol, MAX(price) - MIN(price) +FROM trade01m +GROUP BY stocksymbol; \ No newline at end of file From 4942dc1f5044bdd79b72d554f1b04cb0c6587762 Mon Sep 17 00:00:00 2001 From: Bill Date: Tue, 6 Dec 2022 11:34:39 +0800 Subject: [PATCH 2/7] fixes on stored proc --- Makefile | 4 ++-- aquery_config.py | 2 +- build.py | 4 ++-- server/server.cpp | 26 ++++++++++++++++++-------- 4 files changed, 23 insertions(+), 13 deletions(-) diff --git a/Makefile b/Makefile index c438529..4240bf6 100644 --- a/Makefile +++ b/Makefile @@ -4,10 +4,10 @@ MonetDB_INC = Defines = CXXFLAGS = --std=c++2a ifeq ($(AQ_DEBUG), 1) - OPTFLAGS = -g3 -fsanitize=address -fsanitize=leak + OPTFLAGS = -g3 #-fsanitize=address LINKFLAGS = else - OPTFLAGS = -O3 -DNDEBUG -fno-stack-protector + OPTFLAGS = -Ofast -DNDEBUG -fno-stack-protector LINKFLAGS = -flto -s endif SHAREDFLAGS = -shared diff --git a/aquery_config.py b/aquery_config.py index 094bc47..df2511a 100644 --- a/aquery_config.py +++ b/aquery_config.py @@ -2,7 +2,7 @@ ## GLOBAL CONFIGURATION FLAGS -version_string = '0.5.4a' +version_string = '0.6.0a' add_path_to_ldpath = True rebuild_backend = False run_backend = True diff --git a/build.py b/build.py index ec59122..e8c5255 100644 --- a/build.py +++ b/build.py @@ -117,7 +117,7 @@ class build_manager: else: mgr.cxx = os.environ['CXX'] if 'AQ_DEBUG' not in os.environ: - os.environ['AQ_DEBUG'] = '0' if mgr.OptimizationLv else '1' + os.environ['AQ_DEBUG'] = ('0' if mgr.OptimizationLv != '0' else '1') def libaquery_a(self): self.build_cmd = [['rm', 'libaquery.a'],['make', 'libaquery']] @@ -184,7 +184,7 @@ class build_manager: def __init__(self) -> None: self.method = 'make' self.cxx = '' - self.OptimizationLv = '0' # [O0, O1, O2, O3, Ofast] + self.OptimizationLv = '4' # [O0, O1, O2, O3, Ofast] self.Platform = 'amd64' self.PCH = os.environ['PCH'] if 'PCH' in os.environ else 1 self.StaticLib = 1 diff --git a/server/server.cpp b/server/server.cpp index 64f6544..f2e8c77 100644 --- a/server/server.cpp +++ b/server/server.cpp @@ -503,6 +503,7 @@ start: }; const auto& load_proc_fromfile = [&](StoredProcedure& p) { auto config_name = procedure_root + p.name + ".aqp"; + puts(p.name); auto fp = fopen(config_name.c_str(), "rb"); if(fp == nullptr){ puts("ERROR: Procedure not found on disk."); @@ -517,12 +518,14 @@ start: p.queries = static_cast(malloc(sizeof(char*) * p.cnt)); p.queries[0] = static_cast(malloc(sizeof(char) * queries_size)); - fread(&p.queries[0], queries_size, 1, fp); + fread(p.queries[0], 1, queries_size, fp); for(uint32_t j = 1; j < p.cnt; ++j){ p.queries[j] = p.queries[j-1]; - while(*p.queries[j] != '\0') + while(*(p.queries[j]) != '\0') ++p.queries[j]; + ++p.queries[j]; + puts(p.queries[j-1]); } fclose(fp); return load_modules(p); @@ -553,18 +556,22 @@ start: auto _proc = cxt->stored_proc.find(proc_name); if (_proc == cxt->stored_proc.end()){ printf("Procedure %s not found. Trying load from disk.\n", proc_name); - if (load_proc_fromfile(current_procedure)){ + current_procedure.name = copy_lpstr(proc_name); + if (!load_proc_fromfile(current_procedure)){ cxt->stored_proc.insert_or_assign(proc_name, current_procedure); } + else { + continue; + } } else{ current_procedure = _proc->second; - n_recv = current_procedure.cnt; - n_recvd = current_procedure.queries; - load_modules(current_procedure); - procedure_replaying = true; - goto start; // yes, I know, refactor later!! } + n_recv = current_procedure.cnt; + n_recvd = current_procedure.queries; + load_modules(current_procedure); + procedure_replaying = true; + goto start; // yes, I know, refactor later!! } break; case 'D': // delete procedure @@ -572,6 +579,9 @@ start: case 'S': //save procedure break; case 'L': //load procedure + if (!load_proc_fromfile(current_procedure)) { + cxt->stored_proc.insert_or_assign(proc_name, current_procedure); + } break; case 'd': // display all procedures for(const auto& p : cxt->stored_proc){ From eebf507c6a9390ebaa74aefc8bb5332c67346a7a Mon Sep 17 00:00:00 2001 From: Bill Date: Fri, 23 Dec 2022 15:04:39 +0800 Subject: [PATCH 3/7] WIP: group by optimizations --- README.md | 65 +++++++++++++++++----------------------- server/aggregations.h | 15 ++++++++++ server/hasher.h | 4 +++ server/monetdb_conn.cpp | 7 ++++- server/server.cpp | 27 ++++++++++++----- server/table.h | 1 + server/unordered_dense.h | 12 ++++++++ 7 files changed, 86 insertions(+), 45 deletions(-) diff --git a/README.md b/README.md index ef96a71..3624a73 100644 --- a/README.md +++ b/README.md @@ -1,31 +1,20 @@ - # AQuery++ Database - -### Please try the latest code in dev branch if you encounter any problem. Use `git checkout dev` to switch branches. - ## Introduction AQuery++ Database is a cross-platform, In-Memory Column-Store Database that incorporates compiled query execution. (**Note**: If you encounter any problems, feel free to contact me via ys3540@nyu.edu) +# Architecture +![Architecture](./docs/arch-hybrid.svg) -## Docker (Recommended): - - See installation instructions from [docker.com](https://www.docker.com). Run **docker desktop** to start docker engine. - - In AQuery root directory, type `make docker` to build the docker image from scratch. - - For Arm-based Mac users, you would have to build and run the **x86_64** docker image because MonetDB doesn't offer official binaries for arm64 Linux. (Run `docker buildx build --platform=linux/amd64 -t aquery .` instead of `make docker`) - - Finally run the image in **interactive** mode (`docker run --name aquery -it aquery`) - - When you need to access the container again run `docker start -ai aquery` - - If there is a need to access the system shell within AQuery, type `dbg` to activate python interpreter and type `os.system('sh')` to launch a shell. - - Docker image is available on [Docker Hub](https://hub.docker.com/repository/docker/sunyinqi0508/aquery) but building image yourself is highly recommended (see [#2](../../issues/2)) -## CIMS Computer Lab (Only for NYU affiliates who have access) - 1. Clone this git repo in CIMS. - 2. Download the [patch](https://drive.google.com/file/d/1YkykhM6u0acZ-btQb4EUn4jAEXPT81cN/view?usp=sharing) - 3. Decompress the patch to any directory and execute script inside by typing (`source ./cims.sh`). Please use the source command or `. ./cims.sh` (dot space) to execute the script because it contains configurations for environment variables. Also note that this script can only work with bash and compatible shells (e.g. dash, zsh. but not csh) - 4. Execute `python3 ./prompt.py` +## AQuery Compiler +- The query is first processed by the AQuery Compiler which is composed of a frontend that parses the query into AST and a backend that generates target code that delivers the query. +- Front end of AQuery++ Compiler is built on top of [mo-sql-parsing](https://github.com/klahnakoski/mo-sql-parsing) with modifications to handle AQuery dialect and extension. +- Backend of AQuery++ Compiler generates target code dependent on the Execution Engine. It can either be the C++ code for AQuery Execution Engine or sql and C++ post-processor for Hybrid Engine or k9 for the k9 Engine. +## Execution Engines +- AQuery++ supports different execution engines thanks to the decoupled compiler structure. +- Hybrid Execution Engine: decouples the query into two parts. The sql-compliant part is executed by an Embedded version of Monetdb and everything else is executed by a post-process module which is generated by AQuery++ Compiler in C++ and then compiled and executed. +- AQuery Library: A set of header based libraries that provide column arithmetic and operations inspired by array programming languages like kdb. This library is used by C++ post-processor code which can significantly reduce the complexity of generated code, reducing compile time while maintaining the best performance. The set of libraries can also be used by UDFs as well as User modules which makes it easier for users to write simple, efficient yet powerful extensions. -## Singularity Container - 1. build container `singularity build aquery.sif aquery.def` - 2. execute container `singularity exec aquery.sif sh` - 3. run AQuery `python3 ./prompt.py` -# Native Installation: +# Installation: ## Requirements 1. Recent version of Linux, Windows or MacOS, with recent C++ compiler that has C++17 (1z) support. (however c++20 is recommended if available for heterogeneous lookup on unordered containers) - GCC: 9.0 or above (g++ 7.x, 8.x fail to handle fold-expressions due to a compiler bug) @@ -38,10 +27,6 @@ AQuery++ Database is a cross-platform, In-Memory Column-Store Database that inco - On MacOS, Monetdb can be easily installed in homebrew `brew install monetdb`. 3. Python 3.6 or above and install required packages in requirements.txt by `python3 -m pip install -r requirements.txt` - -## Installation -AQuery is tested on mainstream operating systems such as Windows, macOS and Linux - ### Windows There're multiple options to run AQuery on Windows. But for better consistency I recommend using a simulated Linux environment such as **Windows Subsystem for Linux** (1 or 2), **Docker** or **Linux Virtual Machines**. You can also use the native toolchain from Microsoft Visual Studio or gcc from Winlabs/Cygwin/MinGW. @@ -97,7 +82,24 @@ There're multiple options to run AQuery on Windows. But for better consistency I In this case, upgrade anaconda or your compiler or use the python from your OS or package manager instead. Or (**NOT recommended**) copy/link the library from your system (e.g. /usr/lib/x86_64-linux-gnu/libstdc++.so.6) to anaconda's library directory (e.g. ~/Anaconda3/lib/). +## Docker (Recommended): + - See installation instructions from [docker.com](https://www.docker.com). Run **docker desktop** to start docker engine. + - In AQuery root directory, type `make docker` to build the docker image from scratch. + - For Arm-based Mac users, you would have to build and run the **x86_64** docker image because MonetDB doesn't offer official binaries for arm64 Linux. (Run `docker buildx build --platform=linux/amd64 -t aquery .` instead of `make docker`) + - Finally run the image in **interactive** mode (`docker run --name aquery -it aquery`) + - When you need to access the container again run `docker start -ai aquery` + - If there is a need to access the system shell within AQuery, type `dbg` to activate python interpreter and type `os.system('sh')` to launch a shell. + - Docker image is available on [Docker Hub](https://hub.docker.com/repository/docker/sunyinqi0508/aquery) but building image yourself is highly recommended (see [#2](../../issues/2)) +## CIMS Computer Lab (Only for NYU affiliates who have access) + 1. Clone this git repo in CIMS. + 2. Download the [patch](https://drive.google.com/file/d/1YkykhM6u0acZ-btQb4EUn4jAEXPT81cN/view?usp=sharing) + 3. Decompress the patch to any directory and execute script inside by typing (`source ./cims.sh`). Please use the source command or `. ./cims.sh` (dot space) to execute the script because it contains configurations for environment variables. Also note that this script can only work with bash and compatible shells (e.g. dash, zsh. but not csh) + 4. Execute `python3 ./prompt.py` +## Singularity Container + 1. build container `singularity build aquery.sif aquery.def` + 2. execute container `singularity exec aquery.sif sh` + 3. run AQuery `python3 ./prompt.py` # Usage `python3 prompt.py` will launch the interactive command prompt. The server binary will be automatically rebuilt and started. ### Commands: @@ -268,17 +270,6 @@ SELECT * FROM my_table WHERE c1 > 10 - `sqrt(x), trunc(x), and other builtin math functions`: value-wise math operations. `sqrt(x)[i] = sqrt(x[i])` - `pack(cols, ...)`: pack multiple columns with exact same type into a single column. -# Architecture -![Architecture](./docs/arch-hybrid.svg) - -## AQuery Compiler -- The query is first processed by the AQuery Compiler which is composed of a frontend that parses the query into AST and a backend that generates target code that delivers the query. -- Front end of AQuery++ Compiler is built on top of [mo-sql-parsing](https://github.com/klahnakoski/mo-sql-parsing) with modifications to handle AQuery dialect and extension. -- Backend of AQuery++ Compiler generates target code dependent on the Execution Engine. It can either be the C++ code for AQuery Execution Engine or sql and C++ post-processor for Hybrid Engine or k9 for the k9 Engine. -## Execution Engines -- AQuery++ supports different execution engines thanks to the decoupled compiler structure. -- Hybrid Execution Engine: decouples the query into two parts. The sql-compliant part is executed by an Embedded version of Monetdb and everything else is executed by a post-process module which is generated by AQuery++ Compiler in C++ and then compiled and executed. -- AQuery Library: A set of header based libraries that provide column arithmetic and operations inspired by array programming languages like kdb. This library is used by C++ post-processor code which can significantly reduce the complexity of generated code, reducing compile time while maintaining the best performance. The set of libraries can also be used by UDFs as well as User modules which makes it easier for users to write simple but powerful extensions. # Roadmap - [x] SQL Parser -> AQuery Parser (Front End) diff --git a/server/aggregations.h b/server/aggregations.h index 0f1d8f8..bb8ca0e 100644 --- a/server/aggregations.h +++ b/server/aggregations.h @@ -186,6 +186,21 @@ decayed_t> sumw(uint32_t w, const VT& arr) { return ret; } +template class VT> +void avgw(uint32_t w, const VT& arr, + decayed_t>>& ret) { + typedef types::GetFPType> FPType; + const uint32_t& len = arr.size; + uint32_t i = 0; + types::GetLongType s{}; + w = w > len ? len : w; + if (len) s = ret[i++] = arr[0]; + for (; i < w; ++i) + ret[i] = (s += arr[i]) / (FPType)(i + 1); + for (; i < len; ++i) + ret[i] = ret[i - 1] + (arr[i] - arr[i - w]) / (FPType)w; +} + template class VT> decayed_t>> avgw(uint32_t w, const VT& arr) { typedef types::GetFPType> FPType; diff --git a/server/hasher.h b/server/hasher.h index 22a98e2..0675f96 100644 --- a/server/hasher.h +++ b/server/hasher.h @@ -132,3 +132,7 @@ namespace ankerl::unordered_dense{ struct hash> : public hasher{ }; } +struct aq_hashtable_value_t{ + uint32_t id; + uint32_t cnt; +}; \ No newline at end of file diff --git a/server/monetdb_conn.cpp b/server/monetdb_conn.cpp index a7827ae..c577c8b 100644 --- a/server/monetdb_conn.cpp +++ b/server/monetdb_conn.cpp @@ -6,6 +6,8 @@ #include "monetdb_conn.h" #include "monetdbe.h" #include "table.h" +#include + #undef ERROR #undef static_assert @@ -86,7 +88,10 @@ void Server::connect(Context *cxt){ } server = (monetdbe_database*)malloc(sizeof(monetdbe_database)); - auto ret = monetdbe_open(server, nullptr, nullptr); + monetdbe_options ops; + AQ_ZeroMemory(ops); + ops.nr_threads = std::thread::hardware_concurrency(); + auto ret = monetdbe_open(server, nullptr, &ops); if (ret == 0){ status = true; this->server = server; diff --git a/server/server.cpp b/server/server.cpp index f2e8c77..0176e5b 100644 --- a/server/server.cpp +++ b/server/server.cpp @@ -191,6 +191,21 @@ constexpr prt_fn_t monetdbe_prtfns[] = { aq_to_chars }; +#ifndef __AQ_USE_THREADEDGC__ +void aq_init_gc(void *handle, Context* cxt) +{ + typedef void (*aq_gc_init_t) (Context*); + if (handle && cxt){ + auto sym = dlsym(handle, "__AQ_Init_GC__"); + if(sym){ + ((aq_gc_init_t)sym)(cxt); + } + } +} +#else //__AQ_USE_THREADEDGC__ +#define aq_init_gc(h, c) +#endif //__AQ_USE_THREADEDGC__ + #include "monetdbe.h" #undef max #undef min @@ -363,12 +378,7 @@ start: recorded_queries.emplace_back(copy_lpstr("N")); } handle = dlopen(proc_name, RTLD_NOW); -#ifndef __AQ_USE_THREADEDGC__ - { - typedef void (*aq_gc_init_t) (Context*); - ((aq_gc_init_t)dlsym(handle, "__AQ_Init_GC__"))(cxt); - } -#endif + aq_init_gc(handle, cxt); if (procedure_recording) { recorded_libraries.emplace_back(handle); } @@ -474,11 +484,13 @@ start: p.__rt_loaded_modules = static_cast( malloc(sizeof(void*) * p.postproc_modules)); for(uint32_t j = 0; j < p.postproc_modules; ++j){ - auto pj = dlopen(p.name, RTLD_NOW); + auto pj = dlopen((procedure_root + p.name + std::to_string(j) + ".so").c_str(), RTLD_NOW); if (pj == nullptr){ printf("Error: failed to load module %s\n", p.name); return true; } + aq_init_gc(pj, cxt); + p.__rt_loaded_modules[j] = pj; } } @@ -528,6 +540,7 @@ start: puts(p.queries[j-1]); } fclose(fp); + p.__rt_loaded_modules = 0; return load_modules(p); }; switch(n_recvd[i][1]){ diff --git a/server/table.h b/server/table.h index 3a33136..9de4487 100644 --- a/server/table.h +++ b/server/table.h @@ -289,6 +289,7 @@ public: uint32_t len = end - start; return ColView<_Ty>(orig, idxs.subvec(start, end)); } + ColRef<_Ty> subvec_deep(uint32_t start, uint32_t end) const { uint32_t len = end - start; ColRef<_Ty> subvec(len); diff --git a/server/unordered_dense.h b/server/unordered_dense.h index 737d12b..d81a134 100644 --- a/server/unordered_dense.h +++ b/server/unordered_dense.h @@ -1059,6 +1059,18 @@ public: return do_insert_or_assign(std::move(key), std::forward(mapped)).first; } + template + auto hashtable_push(K&& key, M& mapped) { + ++ mapped.id; + ++ mapped.cnt; + auto it_isinserted = try_emplace(std::forward(key), std::forward(mapped)); + if (!it_isinserted.second) { + --mapped.cnt; + return it_isinserted.first->second.id; + } + return mapped.id; + } + template Date: Mon, 9 Jan 2023 22:59:41 +0800 Subject: [PATCH 4/7] group by optimization --- engine/types.py | 6 +-- proctool.py | 2 +- reconstruct/ast.py | 105 +++++++++++++++++++++++++++------------ server/hasher.h | 4 -- server/server.cpp | 1 + server/unordered_dense.h | 64 ++++++++++++------------ server/vector_type.hpp | 57 +++++++++++++++++++++ 7 files changed, 168 insertions(+), 71 deletions(-) diff --git a/engine/types.py b/engine/types.py index 5baf47f..31c5b37 100644 --- a/engine/types.py +++ b/engine/types.py @@ -107,9 +107,9 @@ ULongT = Types(8, name = 'uint64', sqlname = 'UINT64', fp_type=DoubleT) UIntT = Types(7, name = 'uint32', sqlname = 'UINT32', long_type=ULongT, fp_type=FloatT) UShortT = Types(6, name = 'uint16', sqlname = 'UINT16', long_type=ULongT, fp_type=FloatT) UByteT = Types(5, name = 'uint8', sqlname = 'UINT8', long_type=ULongT, fp_type=FloatT) -StrT = Types(200, name = 'str', cname = 'const char*', sqlname='TEXT', ctype_name = 'types::ASTR') -TextT = Types(200, name = 'text', cname = 'const char*', sqlname='TEXT', ctype_name = 'types::ASTR') -VarcharT = Types(200, name = 'varchar', cname = 'const char*', sqlname='VARCHAR', ctype_name = 'types::ASTR') +StrT = Types(200, name = 'str', cname = 'string_view', sqlname='TEXT', ctype_name = 'types::ASTR') +TextT = Types(200, name = 'text', cname = 'string_view', sqlname='TEXT', ctype_name = 'types::ASTR') +VarcharT = Types(200, name = 'varchar', cname = 'string_view', sqlname='VARCHAR', ctype_name = 'types::ASTR') VoidT = Types(200, name = 'void', cname = 'void', sqlname='Null', ctype_name = 'types::None') class VectorT(Types): diff --git a/proctool.py b/proctool.py index 81035bf..1ff726c 100644 --- a/proctool.py +++ b/proctool.py @@ -2,7 +2,7 @@ import struct import readline from typing import List -name : str = input() +name : str = input('Filename (in path ./procedures/.aqp):') def write(): s : str = input() diff --git a/reconstruct/ast.py b/reconstruct/ast.py index e9348ac..37c5e52 100644 --- a/reconstruct/ast.py +++ b/reconstruct/ast.py @@ -339,8 +339,8 @@ class projection(ast_node): return ', '.join([self.pyname2cname[n.name] for n in lst_names]) else: return self.pyname2cname[proj_name] - - for key, val in proj_map.items(): + gb_tovec = [False] * len(proj_map) + for i, (key, val) in enumerate(proj_map.items()): if type(val[1]) is str: x = True y = get_proj_name @@ -357,22 +357,27 @@ class projection(ast_node): out_typenames[key] = decltypestring else: out_typenames[key] = val[0].cname - if (type(val[2].udf_called) is udf and # should bulkret also be colref? + elemental_ret_udf = ( + type(val[2].udf_called) is udf and # should bulkret also be colref? val[2].udf_called.return_pattern == udf.ReturnPattern.elemental_return - or - self.group_node and - (self.group_node.use_sp_gb and + ) + folding_vector_groups = ( + self.group_node and + ( + self.group_node.use_sp_gb and val[2].cols_mentioned.intersection( self.datasource.all_cols().difference( self.datasource.get_joint_cols(self.group_node.refs) - )) - ) and val[2].is_compound # compound val not in key - # or - # val[2].is_compound > 1 - # (not self.group_node and val[2].is_compound) - ): - out_typenames[key] = f'vector_type<{out_typenames[key]}>' - self.out_table.columns[key].compound = True + ) + ) + ) and + val[2].is_compound # compound val not in key + ) + if (elemental_ret_udf or folding_vector_groups): + out_typenames[key] = f'vector_type<{out_typenames[key]}>' + self.out_table.columns[key].compound = True + if self.group_node is not None and self.group_node.use_sp_gb: + gb_tovec[i] = True outtable_col_nameslist = ', '.join([f'"{c.name}"' for c in self.out_table.columns]) self.outtable_col_names = 'names_' + base62uuid(4) self.context.emitc(f'const char* {self.outtable_col_names}[] = {{{outtable_col_nameslist}}};') @@ -384,12 +389,14 @@ class projection(ast_node): gb_vartable : Dict[str, Union[str, int]] = deepcopy(self.pyname2cname) gb_cexprs : List[str] = [] gb_colnames : List[str] = [] + gb_types : List[Types] = [] for key, val in proj_map.items(): col_name = 'col_' + base62uuid(6) self.context.emitc(f'decltype(auto) {col_name} = {self.out_table.contextname_cpp}->get_col<{key}>();') gb_cexprs.append((col_name, val[2])) gb_colnames.append(col_name) - self.group_node.finalize(gb_cexprs, gb_vartable, gb_colnames) + gb_types.append(val[0]) + self.group_node.finalize(gb_cexprs, gb_vartable, gb_colnames, gb_types, gb_tovec) else: for i, (key, val) in enumerate(proj_map.items()): if type(val[1]) is int: @@ -533,6 +540,7 @@ class groupby_c(ast_node): def init(self, node : List[Tuple[expr, Set[ColRef]]]): self.proj : projection = self.parent self.glist : List[Tuple[expr, Set[ColRef]]] = node + self.vecs : str = 'vecs_' + base62uuid(3) return super().init(node) def produce(self, node : List[Tuple[expr, Set[ColRef]]]): @@ -561,21 +569,22 @@ class groupby_c(ast_node): e = g_str g_contents_list.append(e) first_col = g_contents_list[0] + self.total_sz = 'len_' + base62uuid(4) + self.context.emitc(f'uint32_t {self.total_sz} = {first_col}.size;') g_contents_decltype = [f'decays' for c in g_contents_list] g_contents = ', '.join( [f'{c}[{scanner_itname}]' for c in g_contents_list] ) self.context.emitc(f'typedef record<{",".join(g_contents_decltype)}> {self.group_type};') - self.context.emitc(f'ankerl::unordered_dense::map<{self.group_type}, vector_type, ' - f'transTypes<{self.group_type}, hasher>> {self.group};') - self.context.emitc(f'{self.group}.reserve({first_col}.size);') + self.context.emitc(f'AQHashTable<{self.group_type}, ' + f'transTypes<{self.group_type}, hasher>> {self.group} {{{self.total_sz}}};') self.n_grps = len(self.glist) - self.scanner = scan(self, first_col + '.size', it_name=scanner_itname) - self.scanner.add(f'{self.group}[forward_as_tuple({g_contents})].emplace_back({self.scanner.it_var});') + self.scanner = scan(self, self.total_sz, it_name=scanner_itname) + self.scanner.add(f'{self.group}.hashtable_push(forward_as_tuple({g_contents}), {self.scanner.it_var});') def consume(self, _): self.scanner.finalize() - + self.context.emitc(f'auto {self.vecs} = {self.group}.ht_postproc({self.total_sz});') # def deal_with_assumptions(self, assumption:assumption, out:TableInfo): # gscanner = scan(self, self.group) # val_var = 'val_'+base62uuid(7) @@ -583,16 +592,40 @@ class groupby_c(ast_node): # gscanner.add(f'{self.datasource.cxt_name}->order_by<{assumption.result()}>(&{val_var});') # gscanner.finalize() - def finalize(self, cexprs : List[Tuple[str, expr]], var_table : Dict[str, Union[str, int]], col_names : List[str]): - for c in col_names: + def finalize(self, cexprs : List[Tuple[str, expr]], var_table : Dict[str, Union[str, int]], + col_names : List[str], col_types : List[Types], col_tovec : List[bool]): + tovec_columns = set() + for i, c in enumerate(col_names): self.context.emitc(f'{c}.reserve({self.group}.size());') - - gscanner = scan(self, self.group, loop_style = 'for_each') + if col_tovec[i]: # and type(col_types[i]) is VectorT: + typename : Types = col_types[i] # .inner_type + self.context.emitc(f'auto buf_{c} = static_cast<{typename.cname} *>(malloc({self.total_sz} * sizeof({typename.cname})));') + tovec_columns.add(c) + self.arr_len = 'arrlen_' + base62uuid(3) + self.arr_values = 'arrvals_' + base62uuid(3) + + if len(tovec_columns): + self.context.emitc(f'auto {self.arr_len} = {self.group}.size();') + self.context.emitc(f'auto {self.arr_values} = {self.group}.values();') + preproc_scanner = scan(self, self.arr_len) + preproc_scanner_it = preproc_scanner.it_var + for c in tovec_columns: + preproc_scanner.add(f'{c}[{preproc_scanner_it}].init_from' + f'({self.vecs}[{preproc_scanner_it}].size,' + f' {"buf_" + c} + {self.group}.ht_base' + f'[{preproc_scanner_it}]);' + ) + preproc_scanner.finalize() + + # gscanner = scan(self, self.group, loop_style = 'for_each') + gscanner = scan(self, self.arr_len) key_var = 'key_'+base62uuid(7) val_var = 'val_'+base62uuid(7) - gscanner.add(f'auto &{key_var} = {gscanner.it_var}.first;', position = 'front') - gscanner.add(f'auto &{val_var} = {gscanner.it_var}.second;', position = 'front') + # gscanner.add(f'auto &{key_var} = {gscanner.it_var}.first;', position = 'front') + # gscanner.add(f'auto &{val_var} = {gscanner.it_var}.second;', position = 'front') + gscanner.add(f'auto &{key_var} = {self.arr_values}[{gscanner.it_var}];', position = 'front') + gscanner.add(f'auto &{val_var} = {self.vecs}[{gscanner.it_var}];', position = 'front') len_var = None def define_len_var(): nonlocal len_var @@ -627,7 +660,7 @@ class groupby_c(ast_node): materialize_builtin = materialize_builtin, count=lambda:f'{val_var}.size') - for ce in cexprs: + for i, ce in enumerate(cexprs): ex = ce[1] materialize_builtin = {} if type(ex.udf_called) is udf: @@ -640,7 +673,16 @@ class groupby_c(ast_node): materialize_builtin['_builtin_ret'] = f'{ce[0]}.back()' gscanner.add(f'{ex.eval(c_code = True, y=get_var_names, materialize_builtin = materialize_builtin)};\n') continue - gscanner.add(f'{ce[0]}.emplace_back({get_var_names_ex(ex)});\n') + if col_tovec[i]: + if ex.opname == 'avgs': + patch_expr = get_var_names_ex(ex) + patch_expr = patch_expr[:patch_expr.rindex(')')] + patch_expr += ', ' + f'{ce[0]}[{gscanner.it_var}]' + ')' + gscanner.add(f'{patch_expr};\n') + else: + gscanner.add(f'{ce[0]}[{gscanner.it_var}] = {get_var_names_ex(ex)};\n') + else: + gscanner.add(f'{ce[0]}.emplace_back({get_var_names_ex(ex)});\n') gscanner.finalize() @@ -718,10 +760,11 @@ class groupby(ast_node): # self.parent.var_table. self.parent.col_ext.update(l[1]) - def finalize(self, cexprs : List[Tuple[str, expr]], var_table : Dict[str, Union[str, int]], col_names : List[str]): + def finalize(self, cexprs : List[Tuple[str, expr]], var_table : Dict[str, Union[str, int]], + col_names : List[str], col_types : List[Types], col_tovec : List[bool]): if self.use_sp_gb: self.dedicated_gb = groupby_c(self.parent, self.dedicated_glist) - self.dedicated_gb.finalize(cexprs, var_table, col_names) + self.dedicated_gb.finalize(cexprs, var_table, col_names, col_types, col_tovec) class join(ast_node): diff --git a/server/hasher.h b/server/hasher.h index b632319..22a98e2 100644 --- a/server/hasher.h +++ b/server/hasher.h @@ -132,7 +132,3 @@ namespace ankerl::unordered_dense{ struct hash> : public hasher{ }; } -struct aq_hashtable_value_t { - uint32_t id; - uint32_t cnt; -}; \ No newline at end of file diff --git a/server/server.cpp b/server/server.cpp index 507e944..3fcbe9b 100644 --- a/server/server.cpp +++ b/server/server.cpp @@ -295,6 +295,7 @@ void initialize_module(const char* module_name, void* module_handle, Context* cx printf("Warning: module %s have no session support.\n", module_name); } } + #pragma endregion int dll_main(int argc, char** argv, Context* cxt){ aq_timer timer; diff --git a/server/unordered_dense.h b/server/unordered_dense.h index 828615d..03b6fc0 100644 --- a/server/unordered_dense.h +++ b/server/unordered_dense.h @@ -1062,7 +1062,7 @@ public: // template // bool hashtable_push(K&& key) { // auto it_isinserted = try_emplace(std::forward(key), 1); - // if (!it_isinserted.second) + // if (!it_isinserted.second) // ++ it_isinserted.first->second; // return it_isinserted.second; // } @@ -1113,8 +1113,8 @@ public: template && is_transparent_v, bool> = true> + typename KE = KeyEqual>//, + //std::enable_if_t && is_transparent_v, bool> = true> auto hashtable_push(K&& key) -> unsigned { if (is_full()) { increase_size(); @@ -1141,35 +1141,35 @@ template (value_idx); } - template - auto hashtable_push(Args&&... args) -> unsigned { - if (is_full()) { - increase_size(); - } - - // we have to instantiate the value_type to be able to access the key. - // 1. emplace_back the object so it is constructed. 2. If the key is already there, pop it later in the loop. - auto& key = get_key(m_values.emplace_back(std::forward(args)...)); - auto hash = mixed_hash(key); - auto dist_and_fingerprint = dist_and_fingerprint_from_hash(hash); - auto bucket_idx = bucket_idx_from_hash(hash); - - while (dist_and_fingerprint <= at(m_buckets, bucket_idx).m_dist_and_fingerprint) { - if (dist_and_fingerprint == at(m_buckets, bucket_idx).m_dist_and_fingerprint && - m_equal(key, get_key(m_values[at(m_buckets, bucket_idx).m_value_idx]))) { - m_values.pop_back(); // value was already there, so get rid of it - return static_cast(at(m_buckets, bucket_idx).m_value_idx); - } - dist_and_fingerprint = dist_inc(dist_and_fingerprint); - bucket_idx = next(bucket_idx); - } - - // value is new, place the bucket and shift up until we find an empty spot - auto value_idx = static_cast(m_values.size() - 1); - place_and_shift_up({dist_and_fingerprint, value_idx}, bucket_idx); - - return static_cast(value_idx); - } + // template + // auto hashtable_push(Args&&... args) -> unsigned { + // if (is_full()) { + // increase_size(); + // } + + // // we have to instantiate the value_type to be able to access the key. + // // 1. emplace_back the object so it is constructed. 2. If the key is already there, pop it later in the loop. + // auto& key = get_key(m_values.emplace_back(std::forward(args)...)); + // auto hash = mixed_hash(key); + // auto dist_and_fingerprint = dist_and_fingerprint_from_hash(hash); + // auto bucket_idx = bucket_idx_from_hash(hash); + + // while (dist_and_fingerprint <= at(m_buckets, bucket_idx).m_dist_and_fingerprint) { + // if (dist_and_fingerprint == at(m_buckets, bucket_idx).m_dist_and_fingerprint && + // m_equal(key, get_key(m_values[at(m_buckets, bucket_idx).m_value_idx]))) { + // m_values.pop_back(); // value was already there, so get rid of it + // return static_cast(at(m_buckets, bucket_idx).m_value_idx); + // } + // dist_and_fingerprint = dist_inc(dist_and_fingerprint); + // bucket_idx = next(bucket_idx); + // } + + // // value is new, place the bucket and shift up until we find an empty spot + // auto value_idx = static_cast(m_values.size() - 1); + // place_and_shift_up({dist_and_fingerprint, value_idx}, bucket_idx); + + // return static_cast(value_idx); + // } template auto emplace(Args&&... args) -> std::pair { if (is_full()) { diff --git a/server/vector_type.hpp b/server/vector_type.hpp index a5a3b40..dc109b1 100644 --- a/server/vector_type.hpp +++ b/server/vector_type.hpp @@ -427,6 +427,19 @@ constexpr vector_type::vector_type(const uint32_t size, void* // } // } +// template