update structure, vis application

.gitignore

@@ -89,3 +89,5 @@ saved_procedures/**
 procedures/**
 .mypy_cache
 __pycache__
+deps/**
+*.bsc

Makefile

@@ -154,6 +154,7 @@ docker:
 	docker build -t aquery .
 
 clean:
-	rm .cached *.shm *.o dll.so server.so server.bin libaquery.a libaquery.lib -rf 2> $(NULL_DEVICE) || true
+	rm .cached *.shm *.o dll.so server.so server.bin libaquery.a libaquery.lib -rf 2> $(NULL_DEVICE) || true; \
+	rm -rf *.dSYM || true
 
 .PHONY: clean

aquery_config.py

@@ -2,7 +2,7 @@
 
 ## GLOBAL CONFIGURATION FLAGS
 
-version_string = '0.7.1a'
+version_string = '0.7.5a'
 add_path_to_ldpath = True
 rebuild_backend = False
 run_backend = True
@@ -22,7 +22,7 @@ def init_config():
     #os_platform = 'unkown'
     #msbuildroot = 'd:/gg/vs22/MSBuild/Current/Bin'
     import os
-    from engine.utils import add_dll_dir
+    from common.utils import add_dll_dir
     # os.environ['CXX'] = 'C:/Program Files/LLVM/bin/clang.exe'
     os.environ['THREADING'] = '1'
     os.environ['AQUERY_ITC_USE_SEMPH'] = '1'

build.py

@@ -6,7 +6,7 @@ import os
 import subprocess
 import hashlib
 import pickle
-from engine.utils import nullstream
+from common.utils import nullstream
 from typing import Dict, Optional, Set, Union
 
 @dataclass

common/__init__.py

@@ -1,4 +1,4 @@
-from engine.ast import Context, ast_node
+from common.ast import Context, ast_node
 saved_cxt = None
 
 def initialize(cxt = None, keep = False):

common/ast.py

@@ -1,4 +1,4 @@
-from engine.utils import base62uuid
+from common.utils import base62uuid
 from copy import copy
 from typing import *
 # replace column info with this later.
@@ -181,7 +181,7 @@ class TableInfo:
                 raise ValueError(f'Table name/alias not defined{parsedColExpr[0]}')
             else:
                 ret = datasource.parse_col_names(parsedColExpr[1], raw)
-        from engine.expr import index_expr
+        from common.expr import index_expr
         string = ret.reference() + index_expr
         if self.groupinfo is not None and ret and ret in self.groupinfo.raw_groups:
             string = f'get<{self.groupinfo.raw_groups.index(ret)}>({{y}})'
@@ -208,7 +208,7 @@ class Context:
     LOG_INFO = 'INFO'
     LOG_ERROR = 'ERROR'
     LOG_SILENT = 'SILENT'
-    from engine.types import Types
+    from common.types import Types
     type_table : Dict[str, Types] = dict()
     
     def new(self):
@@ -258,7 +258,7 @@ class Context:
         return tbl
 
     def gen_tmptable(self):
-        from engine.utils import base62uuid
+        from common.utils import base62uuid
         return f't{base62uuid(7)}'
     def reg_tmp(self, name, f):
         self.tmp_names.add(name)

common/ddl.py

@@ -1,9 +1,9 @@
 # code-gen for data decl languages
 
-from engine.orderby import orderby
-from engine.ast import ColRef, TableInfo, ast_node, Context, include
-from engine.scan import scan
-from engine.utils import base62uuid
+from common.orderby import orderby
+from common.ast import ColRef, TableInfo, ast_node, Context, include
+from common.scan import scan
+from common.utils import base62uuid
 
 class create_table(ast_node):
     name = 'create_table'

common/expr.py

@@ -1,4 +1,4 @@
-from engine.ast import ast_node, ColRef
+from common.ast import ast_node, ColRef
 start_expr = 'f"'
 index_expr = '{\'\' if x is None and y is None else f\'[{x}]\'}'
 end_expr = '"'
@@ -57,7 +57,7 @@ class expr(ast_node):
         ast_node.__init__(self, parent, node, None)
 
     def init(self, _):
-        from engine.projection import projection
+        from common.projection import projection
         parent = self.parent
         self.isvector = parent.isvector if type(parent) is expr else False
         self.is_compound = parent.is_compound if type(parent) is expr else False

common/groupby.py

@@ -1,8 +1,8 @@
-from engine.ast import ColRef, TableInfo, ast_node
-from engine.orderby import assumption
-from engine.scan import scan
-from engine.utils import base62uuid
-from engine.expr import expr
+from common.ast import ColRef, TableInfo, ast_node
+from common.orderby import assumption
+from common.scan import scan
+from common.utils import base62uuid
+from common.expr import expr
 
 class groupby(ast_node):
     name = '_groupby'

common/join.py

@@ -0,0 +1,6 @@
+from common.ast import ast_node
+
+
+class join(ast_node):
+    name='join'
+    

common/orderby.py

@@ -1,6 +1,6 @@
-from engine.ast import ColRef, TableInfo, View, ast_node, Context
-from engine.utils import base62uuid, seps
-from engine.expr import expr
+from common.ast import ColRef, TableInfo, View, ast_node, Context
+from common.utils import base62uuid, seps
+from common.expr import expr
 
 class order_item:
     def __init__(self, name, node, order = True):

common/projection.py

@@ -1,11 +1,11 @@
-from engine.ast import ColRef, TableInfo, ast_node, Context, include
-from engine.groupby import groupby
-from engine.join import join
-from engine.expr import expr
-from engine.orderby import assumption, orderby
-from engine.scan import filter
-from engine.utils import base62uuid, enlist, base62alp, has_other
-from engine.ddl import create_table, outfile
+from common.ast import ColRef, TableInfo, ast_node, Context, include
+from common.groupby import groupby
+from common.join import join
+from common.expr import expr
+from common.orderby import assumption, orderby
+from common.scan import filter
+from common.utils import base62uuid, enlist, base62alp, has_other
+from common.ddl import create_table, outfile
 import copy
 
 class projection(ast_node):

common/scan.py

@@ -1,7 +1,7 @@
 from xmlrpc.client import Boolean
-from engine.ast import ColRef, TableInfo, View, ast_node, Context
-from engine.utils import base62uuid
-from engine.expr import expr
+from common.ast import ColRef, TableInfo, View, ast_node, Context
+from common.utils import base62uuid
+from common.expr import expr
 
 class scan(ast_node):
     name = 'scan'

common/types.py

@@ -2,7 +2,7 @@ from copy import deepcopy
 from typing import Dict, List
 
 from aquery_config import have_hge
-from engine.utils import base62uuid, defval
+from common.utils import base62uuid, defval
 
 type_table: Dict[str, "Types"] = {}
 

demo/Python Integration/Makefile

@@ -0,0 +1,15 @@
+ifeq ($(PYTHON_EXEC),)
+	PYTHON_EXEC=python3
+endif
+PYTHON_CONFIG=$(PYTHON_EXEC)-config
+PYTHON_LDFLAGS=$(shell echo `$(PYTHON_CONFIG) --ldflags` | sed 's/^-L\([^[:space:]]*\).*$$/\1/')/lib$(PYTHON_EXEC)*.a
+all:
+	rm ../../libdraw.so; \
+	$(CXX) --shared app2.cpp --std=c++2a -I../.. -I/usr/local/opt/matplotplusplus/include -L../.. -L/usr/local/opt/matplotplusplus/lib/ -lmatplot -laquery -o ../../libdraw.so
+py:
+	rm ../../libdraw.so; \
+	$(CXX) --shared app.cpp --std=c++2a -I../.. `$(PYTHON_CONFIG) --cflags` -O0 -g3 $(PYTHON_LDFLAGS) ../../libaquery.a -o ../../libdraw.so
+exe:
+	$(CXX) app.cpp --std=c++2a -I../.. `$(PYTHON_CONFIG) --cflags` -O0 -g3 $(PYTHON_LDFLAGS) ../../libaquery.a -o a.out
+info:
+	$(info $(PYTHON_LDFLAGS))

demo/Python Integration/app.cpp

@@ -0,0 +1,47 @@
+#include <Python.h>
+#include <random>
+#include <ctime>
+#include <cstdio>
+#include "server/vector_type.hpp"
+#include "sdk/aquery.h"
+#include "unistd.h"
+__AQEXPORT__(bool)
+draw(vector_type<int> x, vector_type<int> y) {
+    puts("sibal!");
+    auto pid = fork();
+    int status = 0;
+    if (pid == 0) {
+        //PyOS_AfterFork();
+        Py_InitializeEx
+        Py_Initialize();
+
+        PyRun_SimpleString("print(globals())");
+        PyRun_SimpleString("import os");
+        PyRun_SimpleString("sys.path.append(os.getcwd()+'/demo/Python Integration')");
+        //PyErr_Print();
+        PyRun_SimpleString("print('fuck')");
+        auto py_strapp = PyUnicode_DecodeFSDefault("app");
+        auto py_module = PyImport_Import(py_strapp);
+        // Py_DECREF(py_strapp);
+        auto py_entrypt = PyObject_GetAttrString(py_module, "draw");
+
+        auto mvx = PyMemoryView_FromMemory((char*)x.container, x.size * sizeof(int), PyBUF_WRITE), 
+            mvy = PyMemoryView_FromMemory((char*)y.container, y.size * sizeof(int), PyBUF_WRITE);
+        PyObject_CallObject(py_entrypt, PyTuple_Pack(2, mvx, mvy));
+        // Py_DECREF(mvx);
+        // Py_DECREF(mvy);
+        // Py_DECREF(py_entrypt);
+        // Py_DECREF(py_module);
+        return 0;
+    }
+    else {
+        while(wait(&status) > 0);
+        //getchar();
+    }
+    return true;
+    //return Py_FinalizeEx() >= 0;
+}
+
+int main(){
+    draw({1,2,3}, {4,5,6});
+}

demo/Python Integration/app.py

@@ -0,0 +1,9 @@
+import matplotlib.pyplot as plt
+
+def draw(x, y):
+    plt.figure()
+    
+    plt.plot(x.cast('i', shape=(len(x)//4, )), 
+             y.cast('i', shape=(len(y)//4, )))
+    plt.show()
+ 

demo/Python Integration/app2.cpp

@@ -0,0 +1,12 @@
+#include "server/vector_type.hpp"
+#include "sdk/aquery.h"
+#include "matplot/matplot.h"
+
+__AQEXPORT__(bool)
+draw(vector_type<int> x, vector_type<int> y) {
+    using namespace matplot;
+    auto plt = gca();
+    plt->plot(vector_type_std{x}, vector_type_std{y});
+    show();
+    return true;
+}

demo/README.md

@@ -23,6 +23,6 @@ This folder contains a demo workflow for the two types of triggers.
   - Create an Interval-based Trigger that executes payload `demoi` every 5 seconds
   - Create a Conditional Trigger that executes payload `democa` whenever condition `democq` returns a true. While condition `democq` is tested every time new data is inserted to table `source`.
   - Loads test data by running [demo/test.a](/demo/test.a)
-  - Use query `select predict(x) from test` to get predictions of the test data from current random forest.
-    - In AQuery prompt, an extra `exec` command after the query is needed to execute the query.
-  - Use query `select test(x, y) from test` will also calculate l2 error.
+- Use query `select predict(x) from test` to get predictions of the test data from current random forest.
+  - In AQuery prompt, an extra `exec` command after the query is needed to execute the query.
+- Use query `select test(x, y) from test` will also calculate l2 error.

engine/join.py

@@ -1,6 +0,0 @@
-from engine.ast import ast_node
-
-
-class join(ast_node):
-    name='join'
-    

monetdb/msvc/exception_buffer.h

@@ -0,0 +1,28 @@
+/*
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0.  If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright 1997 - July 2008 CWI, August 2008 - 2022 MonetDB B.V.
+ */
+
+#ifndef EXCEPTION_BUFFER_H
+#define EXCEPTION_BUFFER_H
+
+#include "monetdb_config.h"
+#include <setjmp.h>
+
+typedef struct exception_buffer {
+	jmp_buf state;
+	int code;
+	char *msg;
+	int enabled;
+} exception_buffer;
+
+extern exception_buffer *eb_init( exception_buffer *eb );
+
+/* != 0 on when we return to the savepoint */
+#define eb_savepoint(eb) ((eb)->enabled=1,setjmp((eb)->state))
+extern _Noreturn void eb_error( exception_buffer *eb, char *msg, int val );
+
+#endif /* EXCEPTION_BUFFER_H */

monetdb/msvc/gdk_atoms.h

@@ -0,0 +1,463 @@
+/*
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0.  If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright 1997 - July 2008 CWI, August 2008 - 2022 MonetDB B.V.
+ */
+
+#ifndef _GDK_ATOMS_H_
+#define _GDK_ATOMS_H_
+
+/* atomFromStr returns the number of bytes of the input string that
+ * were processed.  atomToStr returns the length of the string
+ * produced.  Both functions return -1 on (any kind of) failure.  If
+ * *dst is not NULL, *len specifies the available space.  If there is
+ * not enough space, or if *dst is NULL, *dst will be freed (if not
+ * NULL) and a new buffer will be allocated and returned in *dst.
+ * *len will be set to reflect the actual size allocated.  If
+ * allocation fails, *dst will be NULL on return and *len is
+ * undefined.  In any case, if the function returns, *buf is either
+ * NULL or a valid pointer and then *len is the size of the area *buf
+ * points to.
+ *
+ * atomCmp returns a value less than zero/equal to zero/greater than
+ * zer if the first argument points to a values which is deemed
+ * smaller/equal to/larger than the value pointed to by the second
+ * argument.
+ *
+ * atomHash calculates a hash function for the value pointed to by the
+ * argument.
+ */
+
+#define IDLENGTH	64	/* maximum BAT id length */
+
+typedef struct {
+	/* simple attributes */
+	char name[IDLENGTH];
+	uint8_t storage;	/* stored as another type? */
+	bool linear;		/* atom can be ordered linearly */
+	uint16_t size;		/* fixed size of atom */
+
+	/* automatically generated fields */
+	const void *atomNull;	/* global nil value */
+
+	/* generic (fixed + varsized atom) ADT functions */
+	ssize_t (*atomFromStr) (const char *src, size_t *len, void **dst, bool external);
+	ssize_t (*atomToStr) (char **dst, size_t *len, const void *src, bool external);
+	void *(*atomRead) (void *dst, size_t *dstlen, stream *s, size_t cnt);
+	gdk_return (*atomWrite) (const void *src, stream *s, size_t cnt);
+	int (*atomCmp) (const void *v1, const void *v2);
+	BUN (*atomHash) (const void *v);
+	/* optional functions */
+	gdk_return (*atomFix) (const void *atom);
+	gdk_return (*atomUnfix) (const void *atom);
+
+	/* varsized atom-only ADT functions */
+	var_t (*atomPut) (BAT *, var_t *off, const void *src);
+	void (*atomDel) (Heap *, var_t *atom);
+	size_t (*atomLen) (const void *atom);
+	gdk_return (*atomHeap) (Heap *, size_t);
+} atomDesc;
+
+#define MAXATOMS	128
+
+gdk_export atomDesc BATatoms[MAXATOMS];
+gdk_export int GDKatomcnt;
+
+gdk_export int ATOMallocate(const char *nme);
+gdk_export int ATOMindex(const char *nme);
+
+gdk_export str ATOMname(int id);
+gdk_export size_t ATOMlen(int id, const void *v);
+gdk_export void *ATOMnil(int id)
+	__attribute__((__malloc__));
+gdk_export int ATOMprint(int id, const void *val, stream *fd);
+gdk_export char *ATOMformat(int id, const void *val);
+
+gdk_export void *ATOMdup(int id, const void *val);
+
+/*
+ * @- maximum atomic string lengths
+ */
+#define bitStrlen	8
+#define bteStrlen	8
+#define shtStrlen	12
+#define intStrlen	24
+#if SIZEOF_OID == SIZEOF_INT
+#define oidStrlen	24
+#else
+#define oidStrlen	48
+#endif
+#if SIZEOF_PTR == SIZEOF_INT
+#define ptrStrlen	24
+#else
+#define ptrStrlen	48
+#endif
+#define lngStrlen	48
+#ifdef HAVE_HGE
+#define hgeStrlen	96
+#endif
+#define fltStrlen	48
+#define dblStrlen	96
+
+/*
+ * The system comes with the traditional atomic types: int (4 bytes),
+ * bool(1 byte) and str (variable). In addition, we support the notion
+ * of an OID type, which ensures uniqueness of its members.  This
+ * leads to the following type descriptor table.
+ */
+
+#ifdef HAVE_HGE
+gdk_export ssize_t hgeFromStr(const char *src, size_t *len, hge **dst, bool external);
+gdk_export ssize_t hgeToStr(str *dst, size_t *len, const hge *src, bool external);
+#endif
+gdk_export ssize_t lngFromStr(const char *src, size_t *len, lng **dst, bool external);
+gdk_export ssize_t lngToStr(str *dst, size_t *len, const lng *src, bool external);
+gdk_export ssize_t intFromStr(const char *src, size_t *len, int **dst, bool external);
+gdk_export ssize_t intToStr(str *dst, size_t *len, const int *src, bool external);
+gdk_export ssize_t batFromStr(const char *src, size_t *len, bat **dst, bool external);
+gdk_export ssize_t batToStr(str *dst, size_t *len, const bat *src, bool external);
+gdk_export ssize_t ptrFromStr(const char *src, size_t *len, ptr **dst, bool external);
+gdk_export ssize_t ptrToStr(str *dst, size_t *len, const ptr *src, bool external);
+gdk_export ssize_t bitFromStr(const char *src, size_t *len, bit **dst, bool external);
+gdk_export ssize_t bitToStr(str *dst, size_t *len, const bit *src, bool external);
+gdk_export ssize_t OIDfromStr(const char *src, size_t *len, oid **dst, bool external);
+gdk_export ssize_t OIDtoStr(str *dst, size_t *len, const oid *src, bool external);
+gdk_export ssize_t shtFromStr(const char *src, size_t *len, sht **dst, bool external);
+gdk_export ssize_t shtToStr(str *dst, size_t *len, const sht *src, bool external);
+gdk_export ssize_t bteFromStr(const char *src, size_t *len, bte **dst, bool external);
+gdk_export ssize_t bteToStr(str *dst, size_t *len, const bte *src, bool external);
+gdk_export ssize_t fltFromStr(const char *src, size_t *len, flt **dst, bool external);
+gdk_export ssize_t fltToStr(str *dst, size_t *len, const flt *src, bool external);
+gdk_export ssize_t dblFromStr(const char *src, size_t *len, dbl **dst, bool external);
+gdk_export ssize_t dblToStr(str *dst, size_t *len, const dbl *src, bool external);
+gdk_export ssize_t GDKstrFromStr(unsigned char *restrict dst, const unsigned char *restrict src, ssize_t len);
+gdk_export ssize_t strFromStr(const char *restrict src, size_t *restrict len, str *restrict dst, bool external);
+gdk_export size_t escapedStrlen(const char *restrict src, const char *sep1, const char *sep2, int quote);
+gdk_export size_t escapedStr(char *restrict dst, const char *restrict src, size_t dstlen, const char *sep1, const char *sep2, int quote);
+/*
+ * @- nil values
+ * All types have a single value designated as a NIL value. It
+ * designates a missing value and it is ignored (forbidden) in several
+ * primitives.  The current policy is to use the smallest value in any
+ * ordered domain.  The routine atomnil returns a pointer to the nil
+ * value representation.
+ */
+#define GDK_bit_max ((bit) 1)
+#define GDK_bit_min ((bit) 0)
+#define GDK_bte_max ((bte) INT8_MAX)
+#define GDK_bte_min ((bte) INT8_MIN+1)
+#define GDK_sht_max ((sht) INT16_MAX)
+#define GDK_sht_min ((sht) INT16_MIN+1)
+#define GDK_int_max ((int) INT32_MAX)
+#define GDK_int_min ((int) INT32_MIN+1)
+#define GDK_lng_max ((lng) INT64_MAX)
+#define GDK_lng_min ((lng) INT64_MIN+1)
+#ifdef HAVE_HGE
+#define GDK_hge_max ((((hge) 1) << 126) - 1 + (((hge) 1) << 126))
+#define GDK_hge_min (-GDK_hge_max)
+#endif
+#define GDK_flt_max ((flt) FLT_MAX)
+#define GDK_flt_min ((flt) -FLT_MAX)
+#define GDK_dbl_max ((dbl) DBL_MAX)
+#define GDK_dbl_min ((dbl) -DBL_MAX)
+#define GDK_oid_max (((oid) 1 << ((8 * SIZEOF_OID) - 1)) - 1)
+#define GDK_oid_min ((oid) 0)
+/* representation of the nil */
+gdk_export const bte bte_nil;
+gdk_export const sht sht_nil;
+gdk_export const int int_nil;
+#ifdef NAN_CANNOT_BE_USED_AS_INITIALIZER
+/* Definition of NAN is seriously broken on Intel compiler (at least
+ * in some versions), so we work around it. */
+union _flt_nil_t {
+	uint32_t l;
+	flt f;
+};
+gdk_export const union _flt_nil_t _flt_nil_;
+#define flt_nil (_flt_nil_.f)
+union _dbl_nil_t {
+	uint64_t l;
+	dbl d;
+};
+gdk_export const union _dbl_nil_t _dbl_nil_;
+#define dbl_nil (_dbl_nil_.d)
+#else
+gdk_export const flt flt_nil;
+gdk_export const dbl dbl_nil;
+#endif
+gdk_export const lng lng_nil;
+#ifdef HAVE_HGE
+gdk_export const hge hge_nil;
+#endif
+gdk_export const oid oid_nil;
+gdk_export const char str_nil[2];
+gdk_export const ptr ptr_nil;
+gdk_export const uuid uuid_nil;
+
+/* derived NIL values - OIDDEPEND */
+#define bit_nil	((bit) bte_nil)
+#define bat_nil	((bat) int_nil)
+
+#define void_nil	oid_nil
+
+#define is_bit_nil(v)	((v) == GDK_bte_min-1)
+#define is_bte_nil(v)	((v) == GDK_bte_min-1)
+#define is_sht_nil(v)	((v) == GDK_sht_min-1)
+#define is_int_nil(v)	((v) == GDK_int_min-1)
+#define is_lng_nil(v)	((v) == GDK_lng_min-1)
+#ifdef HAVE_HGE
+#define is_hge_nil(v)	((v) == GDK_hge_min-1)
+#endif
+#define is_oid_nil(v)	((v) == ((oid) 1 << ((8 * SIZEOF_OID) - 1)))
+#define is_flt_nil(v)	isnan(v)
+#define is_dbl_nil(v)	isnan(v)
+#define is_bat_nil(v)	(((v) & 0x7FFFFFFF) == 0) /* v == bat_nil || v == 0 */
+
+#include <math.h>
+
+#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) && _MSC_VER < 1800
+#include <float.h>
+#define isnan(x)	_isnan(x)
+#define isinf(x)	(_fpclass(x) & (_FPCLASS_NINF | _FPCLASS_PINF))
+#define isfinite(x)	_finite(x)
+#endif
+
+#ifdef HAVE_HGE
+#define is_uuid_nil(x)	((x).h == 0)
+#else
+#ifdef HAVE_UUID
+#define is_uuid_nil(x)	uuid_is_null((x).u)
+#else
+#define is_uuid_nil(x)	(memcmp((x).u, uuid_nil.u, UUID_SIZE) == 0)
+#endif
+#endif
+
+/*
+ * @- Derived types
+ * In all algorithms across GDK, you will find switches on the types
+ * (bte, sht, int, flt, dbl, lng, hge, str). They respectively
+ * represent an octet, a 16-bit int, a 32-bit int, a 32-bit float, a
+ * 64-bit double, a 64-bit int, a 128-bit int, and a pointer-sized location
+ * of a char-buffer (ended by a zero char).
+ *
+ * In contrast, the types (bit, ptr, bat, oid) are derived types. They
+ * do not occur in the switches. The ATOMstorage macro maps them
+ * respectively onto a @code{ bte}, @code{ int} (pointers are 32-bit),
+ * @code{ int}, and @code{ int}. OIDs are 32-bit.
+ *
+ * This approach makes it tractable to switch to 64-bits OIDs, or to a
+ * fully 64-bits OS easily. One only has to map the @code{ oid} and
+ * @code{ ptr} types to @code{ lng} instead of @code{ int}.
+ *
+ * Derived types mimic their fathers in many ways. They inherit the
+ * @code{ size}, @code{ linear}, and @code{ null}
+ * properties of their father.  The same goes for the
+ * ADT functions HASH, CMP, PUT, NULL, DEL, LEN, and HEAP. So, a
+ * derived type differs in only two ways from its father:
+ * @table @code
+ * @item [string representation]
+ * the only two ADT operations specific for a derived type are FROMSTR
+ * and TOSTR.
+ * @item [identity]
+ * (a @code{ bit} is really of a different type than @code{ bte}). The
+ * set of operations on derived type values or BATs of such types may
+ * differ from the sets of operations on the father type.
+ * @end table
+ */
+/* use "do ... while(0)" so that lhs can safely be used in if statements */
+#define ATOMstorage(t)		BATatoms[t].storage
+#define ATOMsize(t)		BATatoms[t].size
+#define ATOMfromstr(t,s,l,src,ext)	BATatoms[t].atomFromStr(src,l,s,ext)
+#define ATOMnilptr(t)		BATatoms[t].atomNull
+#define ATOMcompare(t)		BATatoms[t].atomCmp
+#define ATOMcmp(t,l,r)		((*ATOMcompare(t))(l, r))
+#define ATOMhash(t,src)		BATatoms[t].atomHash(src)
+#define ATOMdel(t,hp,src)	do if (BATatoms[t].atomDel) BATatoms[t].atomDel(hp,src); while (0)
+#define ATOMvarsized(t)		(BATatoms[t].atomPut != NULL)
+#define ATOMlinear(t)		BATatoms[t].linear
+#define ATOMtype(t)		((t) == TYPE_void ? TYPE_oid : (t))
+#define ATOMfix(t,v)		(BATatoms[t].atomFix ? BATatoms[t].atomFix(v) : GDK_SUCCEED)
+#define ATOMunfix(t,v)		(BATatoms[t].atomUnfix ? BATatoms[t].atomUnfix(v) : GDK_SUCCEED)
+
+/* The base type is the storage type if the comparison function, the
+ * hash function, and the nil value are the same as those of the
+ * storage type; otherwise it is the type itself. */
+#define ATOMbasetype(t)	((t) != ATOMstorage(t) &&			\
+			 ATOMnilptr(t) == ATOMnilptr(ATOMstorage(t)) && \
+			 ATOMcompare(t) == ATOMcompare(ATOMstorage(t)) && \
+			 BATatoms[t].atomHash == BATatoms[ATOMstorage(t)].atomHash ? \
+			 ATOMstorage(t) : (t))
+
+/*
+ * In case that atoms are added to a bat, their logical reference
+ * count should be incremented (and decremented if deleted). Notice
+ * that BATs with atomic types that have logical references (e.g. BATs
+ * of BATs but also BATs of ODMG odSet) can never be persistent, as
+ * this would make the commit tremendously complicated.
+ */
+
+static inline gdk_return __attribute__((__warn_unused_result__))
+ATOMputVAR(BAT *b, var_t *dst, const void *src)
+{
+	assert(BATatoms[b->ttype].atomPut != NULL);
+	if ((*BATatoms[b->ttype].atomPut)(b, dst, src) == (var_t) -1)
+		return GDK_FAIL;
+	return GDK_SUCCEED;
+}
+
+
+static inline gdk_return __attribute__((__warn_unused_result__))
+ATOMputFIX(int type, void *dst, const void *src)
+{
+	gdk_return rc;
+
+	assert(BATatoms[type].atomPut == NULL);
+	rc = ATOMfix(type, src);
+	if (rc != GDK_SUCCEED)
+		return rc;
+	switch (ATOMsize(type)) {
+	case 0:		/* void */
+		break;
+	case 1:
+		* (bte *) dst = * (bte *) src;
+		break;
+	case 2:
+		* (sht *) dst = * (sht *) src;
+		break;
+	case 4:
+		* (int *) dst = * (int *) src;
+		break;
+	case 8:
+		* (lng *) dst = * (lng *) src;
+		break;
+	case 16:
+#ifdef HAVE_HGE
+		* (hge *) dst = * (hge *) src;
+#else
+		* (uuid *) dst = * (uuid *) src;
+#endif
+		break;
+	default:
+		memcpy(dst, src, ATOMsize(type));
+		break;
+	}
+	return GDK_SUCCEED;
+}
+
+static inline gdk_return __attribute__((__warn_unused_result__))
+ATOMreplaceVAR(BAT *b, var_t *dst, const void *src)
+{
+	var_t loc = *dst;
+	int type = b->ttype;
+
+	assert(BATatoms[type].atomPut != NULL);
+	if ((*BATatoms[type].atomPut)(b, &loc, src) == (var_t) -1)
+		return GDK_FAIL;
+	if (ATOMunfix(type, dst) != GDK_SUCCEED)
+		return GDK_FAIL;
+	ATOMdel(type, b->tvheap, dst);
+	*dst = loc;
+	return ATOMfix(type, src);
+}
+
+/* string heaps:
+ * - strings are 8 byte aligned
+ * - start with a 1024 bucket hash table
+ * - heaps < 64KiB are fully duplicate eliminated with this hash tables
+ * - heaps >= 64KiB are opportunistically (imperfect) duplicate
+ *   eliminated as only the last 128KiB chunk is considered and there
+ *   is no linked list
+ * - buckets and next pointers are unsigned short "indices"
+ * - indices should be multiplied by 8 and takes from ELIMBASE to get
+ *   an offset
+ * Note that a 64KiB chunk of the heap contains at most 8K 8-byte
+ * aligned strings. The 1K bucket list means that in worst load, the
+ * list length is 8 (OK).
+ */
+#define GDK_STRHASHTABLE	(1<<10)	/* 1024 */
+#define GDK_STRHASHMASK		(GDK_STRHASHTABLE-1)
+#define GDK_STRHASHSIZE		(GDK_STRHASHTABLE * sizeof(stridx_t))
+#define GDK_ELIMPOWER		16	/* 64KiB is the threshold */
+#define GDK_ELIMDOUBLES(h)	((h)->free < GDK_ELIMLIMIT)
+#define GDK_ELIMLIMIT		(1<<GDK_ELIMPOWER)	/* equivalently: ELIMBASE == 0 */
+#define GDK_ELIMBASE(x)		(((x) >> GDK_ELIMPOWER) << GDK_ELIMPOWER)
+#define GDK_VAROFFSET		((var_t) GDK_STRHASHSIZE)
+
+/*
+ * @- String Comparison, NILs and UTF-8
+ *
+ * Using the char* type for strings is handy as this is the type of
+ * any constant strings in a C/C++ program. Therefore, MonetDB uses
+ * this definition for str.  However, different compilers and
+ * platforms use either signed or unsigned characters for the char
+ * type.  It is required that string ordering in MonetDB is consistent
+ * over platforms though.
+ *
+ * As for the choice how strings should be ordered, our support for
+ * UTF-8 actually imposes that it should follow 'unsigned char'
+ * doctrine (like in the AIX native compiler). In this semantics,
+ * though we have to take corrective action to ensure that str(nil) is
+ * the smallest value of the domain.
+ */
+static inline bool __attribute__((__pure__))
+strEQ(const char *l, const char *r)
+{
+	return strcmp(l, r) == 0;
+}
+
+static inline bool __attribute__((__pure__))
+strNil(const char *s)
+{
+	return s == NULL || (s[0] == '\200' && s[1] == '\0');
+}
+
+static inline size_t __attribute__((__pure__))
+strLen(const char *s)
+{
+	return strNil(s) ? 2 : strlen(s) + 1;
+}
+
+static inline int __attribute__((__pure__))
+strCmp(const char *l, const char *r)
+{
+	return strNil(r)
+		? !strNil(l)
+		: strNil(l) ? -1 : strcmp(l, r);
+}
+
+static inline size_t
+VarHeapVal(const void *b, BUN p, int w)
+{
+	switch (w) {
+	case 1:
+		return (size_t) ((const uint8_t *) b)[p] + GDK_VAROFFSET;
+	case 2:
+		return (size_t) ((const uint16_t *) b)[p] + GDK_VAROFFSET;
+#if SIZEOF_VAR_T == 8
+	case 4:
+		return (size_t) ((const uint32_t *) b)[p];
+#endif
+	default:
+		return (size_t) ((const var_t *) b)[p];
+	}
+}
+
+static inline BUN __attribute__((__pure__))
+strHash(const char *key)
+{
+	BUN y = 0;
+
+	for (BUN i = 0; key[i]; i++) {
+		y += key[i];
+		y += (y << 10);
+		y ^= (y >> 6);
+	}
+	y += (y << 3);
+	y ^= (y >> 11);
+	y += (y << 15);
+	return y;
+}
+
+#endif /* _GDK_ATOMS_H_ */

monetdb/msvc/gdk_bbp.h

@@ -0,0 +1,97 @@
+/*
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0.  If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright 1997 - July 2008 CWI, August 2008 - 2022 MonetDB B.V.
+ */
+
+#ifndef _GDK_BBP_H_
+#define _GDK_BBP_H_
+
+#define BBPLOADED	1	/* set if bat in memory */
+#define BBPSWAPPED	2	/* set if dirty bat is not in memory */
+#define BBPTMP		4	/* set if non-persistent bat has image on disk */
+
+/* These 4 symbols indicate what the persistence state is of a bat.
+ * - If the bat was persistent at the last commit (or at startup
+ *   before the first commit), BBPEXISTING or BBPDELETED is set.
+ * - If the bat is to be persistent after the next commit, BBPEXISTING
+ *   or BBPNEW is set (i.e. (status&BBPPERSISTENT) != 0).
+ * - If the bat was transient at the last commit (or didn't exist),
+ *   BBPNEW is set, or none of these flag values is set.
+ * - If the bat is to be transient at the next commit, BBPDELETED is
+ *   set, or none of these flag values is set.
+ * BATmode() switches between BBPDELETED and BBPEXISTING (bat was
+ * persistent at last commit), or between BBPNEW and 0 (bat was
+ * transient or didn't exist at last commit).
+ * Committing a bat switches from BBPNEW to BBPEXISTING, or turns off
+ * BBPDELETED.
+ * In any case, only at most one of BBPDELETED, BBPEXISTING, and
+ * BBPNEW may be set at any one time.
+ *
+ * In short,
+ * BBPEXISTING -- bat was and should remain persistent;
+ * BBPDELETED -- bat was persistent at last commit and should be transient;
+ * BBPNEW -- bat was transient at last commit and should be persistent;
+ * none of the above -- bat was and should remain transient.
+ */
+#define BBPDELETED	16	/* set if bat persistent at last commit is now transient */
+#define BBPEXISTING	32	/* set if bat was already persistent at end of last commit */
+#define BBPNEW		64	/* set if bat has become persistent since last commit */
+#define BBPPERSISTENT	(BBPEXISTING|BBPNEW)	/* mask for currently persistent bats */
+
+#define BBPSTATUS	127
+
+#define BBPUNLOADING	128	/* set while we are unloading */
+#define BBPLOADING	256	/* set while we are loading */
+#define BBPSAVING       512	/* set while we are saving */
+#define BBPRENAMED	1024	/* set when bat is renamed in this transaction */
+#define BBPDELETING	2048	/* set while we are deleting (special case in module unload) */
+#define BBPHOT		4096	/* bat is "hot", i.e. is still in active use */
+#define BBPSYNCING	8192	/* bat between creating backup and saving */
+
+#define BBPUNSTABLE	(BBPUNLOADING|BBPDELETING)	/* set while we are unloading */
+#define BBPWAITING      (BBPUNLOADING|BBPLOADING|BBPSAVING|BBPDELETING|BBPSYNCING)
+
+#define BBPTRIM_ALL	(((size_t)1) << (sizeof(size_t)*8 - 2))	/* very large positive size_t */
+
+gdk_export bat getBBPsize(void); /* current occupied size of BBP array */
+gdk_export lng getBBPlogno(void); /* two lng of extra info in BBP.dir */
+gdk_export lng getBBPtransid(void);
+
+/* global calls */
+gdk_export gdk_return BBPaddfarm(const char *dirname, uint32_t rolemask, bool logerror);
+
+/* update interface */
+gdk_export int BBPreclaim(BAT *b);
+gdk_export gdk_return BBPsave(BAT *b);
+gdk_export int BBPrename(bat bid, const char *nme);
+
+/* query interface */
+gdk_export bat BBPindex(const char *nme);
+gdk_export BAT *BBPdescriptor(bat b);
+
+/* swapping interface */
+gdk_export gdk_return BBPsync(int cnt, bat *restrict subcommit, BUN *restrict sizes, lng logno, lng transid);
+gdk_export int BBPfix(bat b);
+gdk_export int BBPunfix(bat b);
+gdk_export int BBPretain(bat b);
+gdk_export int BBPrelease(bat b);
+gdk_export void BBPkeepref(bat i);
+gdk_export void BBPshare(bat b);
+gdk_export void BBPcold(bat i);
+
+#define BBP_status_set(bid, mode)			\
+	ATOMIC_SET(&BBP_record(bid).status, mode)
+
+#define BBP_status_on(bid, flags)			\
+	ATOMIC_OR(&BBP_record(bid).status, flags)
+
+#define BBP_status_off(bid, flags)			\
+	ATOMIC_AND(&BBP_record(bid).status, ~(flags))
+
+#define BBPswappable(b) ((b) && (b)->batCacheid && BBP_refs((b)->batCacheid) == 0)
+#define BBPtrimmable(b) (BBPswappable(b) && isVIEW(b) == 0 && (BBP_status((b)->batCacheid)&BBPWAITING) == 0)
+
+#endif /* _GDK_BBP_H_ */

monetdb/msvc/gdk_calc.h

@@ -0,0 +1,168 @@
+/*
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0.  If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright 1997 - July 2008 CWI, August 2008 - 2022 MonetDB B.V.
+ */
+
+/* included from gdk.h */
+
+gdk_export BAT *BATcalcnegate(BAT *b, BAT *s);
+gdk_export BAT *BATcalcabsolute(BAT *b, BAT *s);
+gdk_export BAT *BATcalcincr(BAT *b, BAT *s, bool abort_on_error);
+gdk_export BAT *BATcalcdecr(BAT *b, BAT *s, bool abort_on_error);
+gdk_export BAT *BATcalciszero(BAT *b, BAT *s);
+gdk_export BAT *BATcalcsign(BAT *b, BAT *s);
+gdk_export BAT *BATcalcisnil(BAT *b, BAT *s);
+gdk_export BAT *BATcalcisnotnil(BAT *b, BAT *s);
+gdk_export BAT *BATcalcnot(BAT *b, BAT *s);
+gdk_export BAT *BATcalcmin(BAT *b1, BAT *b2, BAT *s1, BAT *s2);
+gdk_export BAT *BATcalcmin_no_nil(BAT *b1, BAT *b2, BAT *s1, BAT *s2);
+gdk_export BAT *BATcalcmincst(BAT *b, const ValRecord *v, BAT *s);
+gdk_export BAT *BATcalcmincst_no_nil(BAT *b, const ValRecord *v, BAT *s);
+gdk_export BAT *BATcalccstmin(const ValRecord *v, BAT *b, BAT *s);
+gdk_export BAT *BATcalccstmin_no_nil(const ValRecord *v, BAT *b, BAT *s);
+gdk_export BAT *BATcalcmax(BAT *b1, BAT *b2, BAT *s1, BAT *s2);
+gdk_export BAT *BATcalcmax_no_nil(BAT *b1, BAT *b2, BAT *s1, BAT *s2);
+gdk_export BAT *BATcalcmaxcst(BAT *b, const ValRecord *v, BAT *s);
+gdk_export BAT *BATcalcmaxcst_no_nil(BAT *b, const ValRecord *v, BAT *s);
+gdk_export BAT *BATcalccstmax(const ValRecord *v, BAT *b, BAT *s);
+gdk_export BAT *BATcalccstmax_no_nil(const ValRecord *v, BAT *b, BAT *s);
+gdk_export BAT *BATcalcadd(BAT *b1, BAT *b2, BAT *s1, BAT *s2, int tp, bool abort_on_error);
+gdk_export BAT *BATcalcaddcst(BAT *b, const ValRecord *v, BAT *s, int tp, bool abort_on_error);
+gdk_export BAT *BATcalccstadd(const ValRecord *v, BAT *b, BAT *s, int tp, bool abort_on_error);
+gdk_export BAT *BATcalcsub(BAT *b1, BAT *b2, BAT *s1, BAT *s2, int tp, bool abort_on_error);
+gdk_export BAT *BATcalcsubcst(BAT *b, const ValRecord *v, BAT *s, int tp, bool abort_on_error);
+gdk_export BAT *BATcalccstsub(const ValRecord *v, BAT *b, BAT *s, int tp, bool abort_on_error);
+gdk_export BAT *BATcalcmul(BAT *b1, BAT *b2, BAT *s1, BAT *s2, int tp, bool abort_on_error);
+gdk_export BAT *BATcalcmulcst(BAT *b, const ValRecord *v, BAT *s, int tp, bool abort_on_error);
+gdk_export BAT *BATcalccstmul(const ValRecord *v, BAT *b, BAT *s, int tp, bool abort_on_error);
+gdk_export BAT *BATcalcdiv(BAT *b1, BAT *b2, BAT *s1, BAT *s2, int tp, bool abort_on_error);
+gdk_export BAT *BATcalcdivcst(BAT *b, const ValRecord *v, BAT *s, int tp, bool abort_on_error);
+gdk_export BAT *BATcalccstdiv(const ValRecord *v, BAT *b, BAT *s, int tp, bool abort_on_error);
+gdk_export BAT *BATcalcmod(BAT *b1, BAT *b2, BAT *s1, BAT *s2, int tp, bool abort_on_error);
+gdk_export BAT *BATcalcmodcst(BAT *b, const ValRecord *v, BAT *s, int tp, bool abort_on_error);
+gdk_export BAT *BATcalccstmod(const ValRecord *v, BAT *b, BAT *s, int tp, bool abort_on_error);
+gdk_export BAT *BATcalcxor(BAT *b1, BAT *b2, BAT *s1, BAT *s2);
+gdk_export BAT *BATcalcxorcst(BAT *b, const ValRecord *v, BAT *s);
+gdk_export BAT *BATcalccstxor(const ValRecord *v, BAT *b, BAT *s);
+gdk_export BAT *BATcalcor(BAT *b1, BAT *b2, BAT *s1, BAT *s2);
+gdk_export BAT *BATcalcorcst(BAT *b, const ValRecord *v, BAT *s);
+gdk_export BAT *BATcalccstor(const ValRecord *v, BAT *b, BAT *s);
+gdk_export BAT *BATcalcand(BAT *b1, BAT *b2, BAT *s1, BAT *s2);
+gdk_export BAT *BATcalcandcst(BAT *b, const ValRecord *v, BAT *s);
+gdk_export BAT *BATcalccstand(const ValRecord *v, BAT *b, BAT *s);
+gdk_export BAT *BATcalclsh(BAT *b1, BAT *b2, BAT *s1, BAT *s2, bool abort_on_error);
+gdk_export BAT *BATcalclshcst(BAT *b, const ValRecord *v, BAT *s, bool abort_on_error);
+gdk_export BAT *BATcalccstlsh(const ValRecord *v, BAT *b, BAT *s, bool abort_on_error);
+gdk_export BAT *BATcalcrsh(BAT *b1, BAT *b2, BAT *s1, BAT *s2, bool abort_on_error);
+gdk_export BAT *BATcalcrshcst(BAT *b, const ValRecord *v, BAT *s, bool abort_on_error);
+gdk_export BAT *BATcalccstrsh(const ValRecord *v, BAT *b, BAT *s, bool abort_on_error);
+gdk_export BAT *BATcalclt(BAT *b1, BAT *b2, BAT *s1, BAT *s2);
+gdk_export BAT *BATcalcltcst(BAT *b, const ValRecord *v, BAT *s);
+gdk_export BAT *BATcalccstlt(const ValRecord *v, BAT *b, BAT *s);
+gdk_export BAT *BATcalcle(BAT *b1, BAT *b2, BAT *s1, BAT *s2);
+gdk_export BAT *BATcalclecst(BAT *b, const ValRecord *v, BAT *s);
+gdk_export BAT *BATcalccstle(const ValRecord *v, BAT *b, BAT *s);
+gdk_export BAT *BATcalcgt(BAT *b1, BAT *b2, BAT *s1, BAT *s2);
+gdk_export BAT *BATcalcgtcst(BAT *b, const ValRecord *v, BAT *s);
+gdk_export BAT *BATcalccstgt(const ValRecord *v, BAT *b, BAT *s);
+gdk_export BAT *BATcalcge(BAT *b1, BAT *b2, BAT *s1, BAT *s2);
+gdk_export BAT *BATcalcgecst(BAT *b, const ValRecord *v, BAT *s);
+gdk_export BAT *BATcalccstge(const ValRecord *v, BAT *b, BAT *s);
+gdk_export BAT *BATcalceq(BAT *b1, BAT *b2, BAT *s1, BAT *s2, bool nil_matches);
+gdk_export BAT *BATcalceqcst(BAT *b, const ValRecord *v, BAT *s, bool nil_matches);
+gdk_export BAT *BATcalccsteq(const ValRecord *v, BAT *b, BAT *s, bool nil_matches);
+gdk_export BAT *BATcalcne(BAT *b1, BAT *b2, BAT *s1, BAT *s2, bool nil_matches);
+gdk_export BAT *BATcalcnecst(BAT *b, const ValRecord *v, BAT *s, bool nil_matches);
+gdk_export BAT *BATcalccstne(const ValRecord *v, BAT *b, BAT *s, bool nil_matches);
+gdk_export BAT *BATcalccmp(BAT *b1, BAT *b2, BAT *s1, BAT *s2);
+gdk_export BAT *BATcalccmpcst(BAT *b, const ValRecord *v, BAT *s);
+gdk_export BAT *BATcalccstcmp(const ValRecord *v, BAT *b, BAT *s);
+gdk_export BAT *BATcalcbetween(BAT *b, BAT *lo, BAT *hi, BAT *s, BAT *slo, BAT *shi, bool symmetric, bool linc, bool hinc, bool nils_false, bool anti);
+gdk_export BAT *BATcalcbetweencstcst(BAT *b, const ValRecord *lo, const ValRecord *hi, BAT *s, bool symmetric, bool linc, bool hinc, bool nils_false, bool anti);
+gdk_export BAT *BATcalcbetweenbatcst(BAT *b, BAT *lo, const ValRecord *hi, BAT *s, BAT *slo, bool symmetric, bool linc, bool hinc, bool nils_false, bool anti);
+gdk_export BAT *BATcalcbetweencstbat(BAT *b, const ValRecord *lo, BAT *hi, BAT *s, BAT *shi, bool symmetric, bool linc, bool hinc, bool nils_false, bool anti);
+gdk_export gdk_return VARcalcbetween(ValPtr ret, const ValRecord *v, const ValRecord *lo, const ValRecord *hi, bool symmetric, bool linc, bool hinc, bool nils_false, bool anti);
+gdk_export BAT *BATcalcifthenelse(BAT *b, BAT *b1, BAT *b2);
+gdk_export BAT *BATcalcifthenelsecst(BAT *b, BAT *b1, const ValRecord *c2);
+gdk_export BAT *BATcalcifthencstelse(BAT *b, const ValRecord *c1, BAT *b2);
+gdk_export BAT *BATcalcifthencstelsecst(BAT *b, const ValRecord *c1, const ValRecord *c2);
+
+gdk_export gdk_return VARcalcnot(ValPtr ret, const ValRecord *v);
+gdk_export gdk_return VARcalcnegate(ValPtr ret, const ValRecord *v);
+gdk_export gdk_return VARcalcabsolute(ValPtr ret, const ValRecord *v);
+gdk_export gdk_return VARcalcincr(ValPtr ret, const ValRecord *v, bool abort_on_error);
+gdk_export gdk_return VARcalcdecr(ValPtr ret, const ValRecord *v, bool abort_on_error);
+gdk_export gdk_return VARcalciszero(ValPtr ret, const ValRecord *v);
+gdk_export gdk_return VARcalcsign(ValPtr ret, const ValRecord *v);
+gdk_export gdk_return VARcalcisnil(ValPtr ret, const ValRecord *v);
+gdk_export gdk_return VARcalcisnotnil(ValPtr ret, const ValRecord *v);
+gdk_export gdk_return VARcalcadd(ValPtr ret, const ValRecord *lft, const ValRecord *rgt, bool abort_on_error);
+gdk_export gdk_return VARcalcsub(ValPtr ret, const ValRecord *lft, const ValRecord *rgt, bool abort_on_error);
+gdk_export gdk_return VARcalcmul(ValPtr ret, const ValRecord *lft, const ValRecord *rgt, bool abort_on_error);
+gdk_export gdk_return VARcalcdiv(ValPtr ret, const ValRecord *lft, const ValRecord *rgt, bool abort_on_error);
+gdk_export gdk_return VARcalcmod(ValPtr ret, const ValRecord *lft, const ValRecord *rgt, bool abort_on_error);
+gdk_export gdk_return VARcalcxor(ValPtr ret, const ValRecord *lft, const ValRecord *rgt);
+gdk_export gdk_return VARcalcor(ValPtr ret, const ValRecord *lft, const ValRecord *rgt);
+gdk_export gdk_return VARcalcand(ValPtr ret, const ValRecord *lft, const ValRecord *rgt);
+gdk_export gdk_return VARcalclsh(ValPtr ret, const ValRecord *lft, const ValRecord *rgt, bool abort_on_error);
+gdk_export gdk_return VARcalcrsh(ValPtr ret, const ValRecord *lft, const ValRecord *rgt, bool abort_on_error);
+gdk_export gdk_return VARcalclt(ValPtr ret, const ValRecord *lft, const ValRecord *rgt);
+gdk_export gdk_return VARcalcgt(ValPtr ret, const ValRecord *lft, const ValRecord *rgt);
+gdk_export gdk_return VARcalcle(ValPtr ret, const ValRecord *lft, const ValRecord *rgt);
+gdk_export gdk_return VARcalcge(ValPtr ret, const ValRecord *lft, const ValRecord *rgt);
+gdk_export gdk_return VARcalceq(ValPtr ret, const ValRecord *lft, const ValRecord *rgt, bool nil_matches);
+gdk_export gdk_return VARcalcne(ValPtr ret, const ValRecord *lft, const ValRecord *rgt, bool nil_matches);
+gdk_export gdk_return VARcalccmp(ValPtr ret, const ValRecord *lft, const ValRecord *rgt);
+gdk_export BAT *BATconvert(BAT *b, BAT *s, int tp, bool abort_on_error, uint8_t scale1, uint8_t scale2, uint8_t precision);
+gdk_export gdk_return VARconvert(ValPtr ret, const ValRecord *v, bool abort_on_error, uint8_t scale1, uint8_t scale2, uint8_t precision);
+gdk_export gdk_return BATcalcavg(BAT *b, BAT *s, dbl *avg, BUN *vals, int scale);
+
+gdk_export BAT *BATgroupsum(BAT *b, BAT *g, BAT *e, BAT *s, int tp, bool skip_nils, bool abort_on_error);
+gdk_export BAT *BATgroupprod(BAT *b, BAT *g, BAT *e, BAT *s, int tp, bool skip_nils, bool abort_on_error);
+gdk_export gdk_return BATgroupavg(BAT **bnp, BAT **cntsp, BAT *b, BAT *g, BAT *e, BAT *s, int tp, bool skip_nils, bool abort_on_error, int scale);
+gdk_export gdk_return BATgroupavg3(BAT **avgp, BAT **remp, BAT **cntp, BAT *b, BAT *g, BAT *e, BAT *s, bool skip_nils);
+gdk_export BAT *BATgroupavg3combine(BAT *avg, BAT *rem, BAT *cnt, BAT *g, BAT *e, bool skip_nils);
+gdk_export BAT *BATgroupcount(BAT *b, BAT *g, BAT *e, BAT *s, int tp, bool skip_nils, bool abort_on_error);
+gdk_export BAT *BATgroupsize(BAT *b, BAT *g, BAT *e, BAT *s, int tp, bool skip_nils, bool abort_on_error);
+gdk_export BAT *BATgroupmin(BAT *b, BAT *g, BAT *e, BAT *s, int tp, bool skip_nils, bool abort_on_error);
+gdk_export BAT *BATgroupmax(BAT *b, BAT *g, BAT *e, BAT *s, int tp, bool skip_nils, bool abort_on_error);
+gdk_export BAT *BATgroupmedian(BAT *b, BAT *g, BAT *e, BAT *s, int tp, bool skip_nils, bool abort_on_error);
+gdk_export BAT *BATgroupquantile(BAT *b, BAT *g, BAT *e, BAT *s, int tp, double quantile, bool skip_nils, bool abort_on_error);
+gdk_export BAT *BATgroupmedian_avg(BAT *b, BAT *g, BAT *e, BAT *s, int tp, bool skip_nils, bool abort_on_error);
+gdk_export BAT *BATgroupquantile_avg(BAT *b, BAT *g, BAT *e, BAT *s, int tp, double quantile, bool skip_nils, bool abort_on_error);
+
+/* helper function for grouped aggregates */
+gdk_export const char *BATgroupaggrinit(
+	BAT *b, BAT *g, BAT *e, BAT *s,
+	/* outputs: */
+	oid *minp, oid *maxp, BUN *ngrpp,
+	struct canditer *ci, BUN *ncand);
+
+gdk_export gdk_return BATsum(void *res, int tp, BAT *b, BAT *s, bool skip_nils, bool abort_on_error, bool nil_if_empty);
+gdk_export gdk_return BATprod(void *res, int tp, BAT *b, BAT *s, bool skip_nils, bool abort_on_error, bool nil_if_empty);
+gdk_export void *BATmax(BAT *b, void *aggr);
+gdk_export void *BATmin(BAT *b, void *aggr);
+gdk_export void *BATmax_skipnil(BAT *b, void *aggr, bit skipnil);
+gdk_export void *BATmin_skipnil(BAT *b, void *aggr, bit skipnil);
+
+gdk_export dbl BATcalcstdev_population(dbl *avgp, BAT *b);
+gdk_export dbl BATcalcstdev_sample(dbl *avgp, BAT *b);
+gdk_export BAT *BATgroupstdev_sample(BAT *b, BAT *g, BAT *e, BAT *s, int tp, bool skip_nils, bool abort_on_error);
+gdk_export BAT *BATgroupstdev_population(BAT *b, BAT *g, BAT *e, BAT *s, int tp, bool skip_nils, bool abort_on_error);
+gdk_export dbl BATcalcvariance_population(dbl *avgp, BAT *b);
+gdk_export dbl BATcalcvariance_sample(dbl *avgp, BAT *b);
+gdk_export BAT *BATgroupvariance_sample(BAT *b, BAT *g, BAT *e, BAT *s, int tp, bool skip_nils, bool abort_on_error);
+gdk_export BAT *BATgroupvariance_population(BAT *b, BAT *g, BAT *e, BAT *s, int tp, bool skip_nils, bool abort_on_error);
+gdk_export dbl BATcalccovariance_sample(BAT *b1, BAT *b2);
+gdk_export dbl BATcalccovariance_population(BAT *b1, BAT *b2);
+gdk_export dbl BATcalccorrelation(BAT *b1, BAT *b2);
+gdk_export BAT *BATgroupcovariance_sample(BAT *b1, BAT *b2, BAT *g, BAT *e, BAT *s, int tp, bool skip_nils, bool abort_on_error);
+gdk_export BAT *BATgroupcovariance_population(BAT *b1, BAT *b2, BAT *g, BAT *e, BAT *s, int tp, bool skip_nils, bool abort_on_error);
+gdk_export BAT *BATgroupcorrelation(BAT *b1, BAT *b2, BAT *g, BAT *e, BAT *s, int tp, bool skip_nils, bool abort_on_error);
+
+gdk_export BAT *BATgroupstr_group_concat(BAT *b, BAT *g, BAT *e, BAT *s, BAT *sep, bool skip_nils, bool abort_on_error, const char *restrict separator);
+gdk_export gdk_return BATstr_group_concat(ValPtr res, BAT *b, BAT *s, BAT *sep, bool skip_nils, bool abort_on_error, bool nil_if_empty, const char *restrict separator);
+gdk_export gdk_return GDKanalytical_str_group_concat(BAT *r, BAT *p, BAT *o, BAT *b, BAT *sep, BAT *s, BAT *e, const char *restrict separator, int frame_type);

monetdb/msvc/gdk_cand.h

@@ -0,0 +1,214 @@
+/*
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0.  If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright 1997 - July 2008 CWI, August 2008 - 2022 MonetDB B.V.
+ */
+
+#ifndef _GDK_CAND_H_
+#define _GDK_CAND_H_
+
+/* candidates by design are ordered oid lists, besides native oid bats
+ * there are
+ *	void bats for dense oid lists,
+ *	negative oid lists
+ *	masked oid lists
+ */
+
+#define CAND_NEGOID 0
+#define CAND_MSK 1
+
+typedef struct {
+	uint64_t
+		type:1,
+//		mask:1,
+		firstbit:48;
+} ccand_t;
+
+#define CCAND(b)	((ccand_t *) (b)->tvheap->base)
+#define complex_cand(b)	((b)->ttype == TYPE_void && (b)->tvheap != NULL)
+#define negoid_cand(b)	(complex_cand(b) && CCAND(b)->type == CAND_NEGOID)
+#define mask_cand(b)	(complex_cand(b) && CCAND(b)->type == CAND_MSK)
+#define ccand_first(b)	((b)->tvheap->base + sizeof(ccand_t))
+#define ccand_free(b)	((b)->tvheap->free - sizeof(ccand_t))
+
+struct canditer {
+	BAT *s;			/* candidate BAT the iterator is based on */
+	union {
+		struct {	/* for all except cand_mask */
+			const oid *oids; /* candidate or exceptions for non-dense */
+			BUN offset;	/* how much of candidate list BAT we skipped */
+			oid add;	/* value to add because of exceptions seen */
+		};
+		struct {	/* only for cand_mask */
+			const uint32_t *mask; /* bitmask */
+			BUN nextmsk;
+			oid mskoff;
+			uint8_t nextbit;
+			uint8_t firstbit;
+			uint8_t lastbit;
+		};
+	};
+	oid seq;		/* first candidate */
+	oid hseq;		/* hseqbase from s/b for first candidate */
+	BUN nvals;		/* number of values in .oids/.mask */
+	BUN ncand;		/* number of candidates */
+	BUN next;		/* next BUN to return value for */
+	enum {
+		cand_dense,	/* simple dense BAT, i.e. no look ups */
+		cand_materialized, /* simple materialized OID list */
+		cand_except,	/* list of exceptions in vheap */
+		cand_mask,	/* bitmask (TYPE_msk) bat as candidate list */
+	} tpe;
+};
+
+/* returns the position of the lowest order bit in x, i.e. the
+ * smallest n such that (x & (1<<n)) != 0; must not be called with 0 */
+static inline int __attribute__((__const__))
+candmask_lobit(uint32_t x)
+{
+	assert(x != 0);
+#if defined(__GNUC__)
+	return __builtin_ctz(x) /* ffs(x) - 1 */;
+#elif defined(_MSC_VER)
+	unsigned long idx;
+	if (_BitScanForward(&idx, x))
+		return (int) idx;
+	return -1;
+#else
+	/* use binary search for the lowest set bit */
+	int n = 1;
+	if ((x & 0x0000FFFF) == 0) { n += 16; x >>= 16; }
+	if ((x & 0x000000FF) == 0) { n +=  8; x >>=  8; }
+	if ((x & 0x0000000F) == 0) { n +=  4; x >>=  4; }
+	if ((x & 0x00000003) == 0) { n +=  2; x >>=  2; }
+	return n - (x & 1);
+#endif
+}
+
+/* population count: count number of 1 bits in a value */
+static inline uint32_t __attribute__((__const__))
+candmask_pop(uint32_t x)
+{
+#if defined(__GNUC__)
+	return (uint32_t) __builtin_popcount(x);
+#elif defined(_MSC_VER)
+	return (uint32_t) __popcnt((unsigned int) (x));
+#else
+	/* divide and conquer implementation (the two versions are
+	 * essentially equivalent, but the first version is written a
+	 * bit smarter) */
+#if 1
+	x -= (x >> 1) & ~0U/3 /* 0x55555555 */; /* 3-1=2; 2-1=1; 1-0=1; 0-0=0 */
+	x = (x & ~0U/5) + ((x >> 2) & ~0U/5) /* 0x33333333 */;
+	x = (x + (x >> 4)) & ~0UL/0x11 /* 0x0F0F0F0F */;
+	x = (x + (x >> 8)) & ~0UL/0x101 /* 0x00FF00FF */;
+	x = (x + (x >> 16)) & 0xFFFF /* ~0UL/0x10001 */;
+#else
+	x = (x & 0x55555555) + ((x >>  1) & 0x55555555);
+	x = (x & 0x33333333) + ((x >>  2) & 0x33333333);
+	x = (x & 0x0F0F0F0F) + ((x >>  4) & 0x0F0F0F0F);
+	x = (x & 0x00FF00FF) + ((x >>  8) & 0x00FF00FF);
+	x = (x & 0x0000FFFF) + ((x >> 16) & 0x0000FFFF);
+#endif
+	return x;
+#endif
+}
+
+#define canditer_next_dense(ci)		((ci)->seq + (ci)->next++)
+#define canditer_next_mater(ci)		((ci)->oids[(ci)->next++])
+static inline oid
+canditer_next_except(struct canditer *ci)
+{
+	oid o = ci->seq + ci->add + ci->next++;
+	while (ci->add < ci->nvals && o == ci->oids[ci->add]) {
+		ci->add++;
+		o++;
+	}
+	return o;
+}
+static inline oid
+canditer_next_mask(struct canditer *ci)
+{
+	/* since .next < .ncand, we know there must be another
+	 * candidate */
+	while ((ci->mask[ci->nextmsk] >> ci->nextbit) == 0) {
+		ci->nextmsk++;
+		ci->nextbit = 0;
+	}
+	ci->nextbit += candmask_lobit(ci->mask[ci->nextmsk] >> ci->nextbit);
+	oid o = ci->mskoff + ci->nextmsk * 32 + ci->nextbit;
+	if (++ci->nextbit == 32) {
+		ci->nextbit = 0;
+		ci->nextmsk++;
+	}
+	ci->next++;
+	return o;
+}
+
+static inline oid
+canditer_next(struct canditer *ci)
+{
+	if (ci->next == ci->ncand)
+		return oid_nil;
+	switch (ci->tpe) {
+	case cand_dense:
+		return canditer_next_dense(ci);
+	case cand_materialized:
+		assert(ci->next < ci->nvals);
+		return canditer_next_mater(ci);
+	case cand_except:
+		return canditer_next_except(ci);
+	case cand_mask:
+		/* work around compiler error: control reaches end of
+		 * non-void function */
+		break;
+	}
+	assert(ci->tpe == cand_mask);
+	return canditer_next_mask(ci);
+}
+
+#define canditer_search_dense(ci, o, next) ((o) < (ci)->seq ? next ? 0 : BUN_NONE : (o) >= (ci)->seq + (ci)->ncand ? next ? (ci)->ncand : BUN_NONE : (o) - (ci)->seq)
+
+gdk_export BUN canditer_init(struct canditer *ci, BAT *b, BAT *s);
+gdk_export oid canditer_peek(struct canditer *ci);
+gdk_export oid canditer_last(const struct canditer *ci);
+gdk_export oid canditer_prev(struct canditer *ci);
+gdk_export oid canditer_peekprev(struct canditer *ci);
+gdk_export oid canditer_idx(const struct canditer *ci, BUN p);
+#define canditer_idx_dense(ci, p) ((p >= (ci)->ncand)?oid_nil:((ci)->seq + p))
+gdk_export void canditer_setidx(struct canditer *ci, BUN p);
+gdk_export void canditer_reset(struct canditer *ci);
+gdk_export BUN canditer_search(const struct canditer *ci, oid o, bool next);
+static inline bool
+canditer_contains(struct canditer *ci, oid o)
+{
+	if (ci->tpe == cand_mask) {
+		if (o < ci->mskoff)
+			return false;
+		o -= ci->mskoff;
+		BUN p = o / 32;
+		if (p >= ci->nvals)
+			return false;
+		o %= 32;
+		if (p == ci->nvals - 1 && o >= ci->lastbit)
+			return false;
+		return ci->mask[p] & (1U << o);
+	}
+	return canditer_search(ci, o, false) != BUN_NONE;
+}
+gdk_export oid canditer_mask_next(const struct canditer *ci, oid o, bool next);
+gdk_export BAT *canditer_slice(const struct canditer *ci, BUN lo, BUN hi);
+gdk_export BAT *canditer_sliceval(const struct canditer *ci, oid lo, oid hi);
+gdk_export BAT *canditer_slice2(const struct canditer *ci, BUN lo1, BUN hi1, BUN lo2, BUN hi2);
+gdk_export BAT *canditer_slice2val(const struct canditer *ci, oid lo1, oid hi1, oid lo2, oid hi2);
+gdk_export BAT *BATnegcands(BUN nr, BAT *odels);
+gdk_export BAT *BATmaskedcands(oid hseq, BUN nr, BAT *masked, bool selected);
+gdk_export BAT *BATunmask(BAT *b);
+
+gdk_export BAT *BATmergecand(BAT *a, BAT *b);
+gdk_export BAT *BATintersectcand(BAT *a, BAT *b);
+gdk_export BAT *BATdiffcand(BAT *a, BAT *b);
+
+#endif	/* _GDK_CAND_H_ */

monetdb/msvc/gdk_delta.h

@@ -0,0 +1,45 @@
+/*
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0.  If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright 1997 - July 2008 CWI, August 2008 - 2022 MonetDB B.V.
+ */
+
+#ifndef _GDK_DELTA_H_
+#define _GDK_DELTA_H_
+
+/*
+ * We make sure here that the BUNs section of a BAT at least starts 4
+ * bytes from the BUN start.  This ensures that the first data item of
+ * e.g. a BAT[void,bit] is (at least) integer aligned.  This optimizes
+ * processing on such BATs (DDBENCH).
+ */
+static inline void
+DELTAinit(BAT *b)
+{
+	BATsetcount(b, 0);
+	b->theap->free = 0;
+	b->batInserted = 0;
+	b->tshift = ATOMelmshift(Tsize(b));
+	TRC_DEBUG(DELTA,
+		  "%s free %zu ins " BUNFMT " base %p\n",
+		  BBP_logical(b->batCacheid),
+		  b->theap->free,
+		  b->batInserted,
+		  b->theap->base);
+}
+
+/*
+ * Upon saving a BAT, we should convert the delta marker BUN pointers
+ * into indexes and convert them back into pointers upon reload.
+ *
+ * The BATdirty(b) tells you whether a BAT's main memory
+ * representation differs from its saved image on stable storage. But
+ * *not* whether it has changed since last transaction commit (it can
+ * be storage-clean, but transaction-dirty). For this we have
+ * DELTAdirty(b).
+ */
+#define DELTAdirty(b)	((b)->batInserted < BUNlast(b))
+
+#endif /* _GDK_DELTA_H_ */

monetdb/msvc/gdk_hash.h

@@ -0,0 +1,299 @@
+/*
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0.  If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright 1997 - July 2008 CWI, August 2008 - 2022 MonetDB B.V.
+ */
+
+#ifndef _GDK_SEARCH_H_
+#define _GDK_SEARCH_H_
+
+struct Hash {
+	int type;		/* type of index entity */
+	uint8_t width;		/* width of hash entries */
+	BUN mask1;		/* .mask1 < .nbucket <= .mask2 */
+	BUN mask2;		/* ... both are power-of-two minus one */
+	BUN nbucket;		/* number of valid hash buckets */
+	BUN nunique;		/* number of unique values */
+	BUN nheads;		/* number of chain heads */
+	void *Bckt;		/* hash buckets, points into .heapbckt */
+	void *Link;		/* collision list, points into .heaplink */
+	Heap heaplink;		/* heap where the hash links are stored */
+	Heap heapbckt;		/* heap where the hash buckets are stored */
+};
+
+static inline BUN
+HASHbucket(const Hash *h, BUN v)
+{
+	return (v &= h->mask2) < h->nbucket ? v : v & h->mask1;
+}
+
+gdk_export gdk_return BAThash(BAT *b);
+gdk_export void HASHdestroy(BAT *b);
+gdk_export BUN HASHprobe(const Hash *h, const void *v);
+gdk_export BUN HASHlist(Hash *h, BUN i);
+
+#define BUN2 2
+#define BUN4 4
+#if SIZEOF_BUN == 8
+#define BUN8 8
+#endif
+#ifdef BUN2
+typedef uint16_t BUN2type;
+#endif
+typedef uint32_t BUN4type;
+#if SIZEOF_BUN > 4
+typedef uint64_t BUN8type;
+#endif
+#ifdef BUN2
+#define BUN2_NONE ((BUN2type) UINT16_C(0xFFFF))
+#endif
+#define BUN4_NONE ((BUN4type) UINT32_C(0xFFFFFFFF))
+#ifdef BUN8
+#define BUN8_NONE ((BUN8type) UINT64_C(0xFFFFFFFFFFFFFFFF))
+#endif
+
+/* play around with h->Bckt[i] and h->Link[j] */
+
+static inline void
+HASHput(Hash *h, BUN i, BUN v)
+{
+	/* if v == BUN_NONE, assigning the value to a BUN2type
+	 * etc. automatically converts to BUN2_NONE etc. */
+	switch (h->width) {
+#ifdef BUN2
+	case BUN2:
+		((BUN2type *) h->Bckt)[i] = (BUN2type) v;
+		break;
+#endif
+	default:		/* BUN4 */
+		((BUN4type *) h->Bckt)[i] = (BUN4type) v;
+		break;
+#ifdef BUN8
+	case BUN8:
+		((BUN8type *) h->Bckt)[i] = (BUN8type) v;
+		break;
+#endif
+	}
+}
+
+static inline void
+HASHputlink(Hash *h, BUN i, BUN v)
+{
+	/* if v == BUN_NONE, assigning the value to a BUN2type
+	 * etc. automatically converts to BUN2_NONE etc. */
+	switch (h->width) {
+#ifdef BUN2
+	case BUN2:
+		assert(v == BUN_NONE || v == BUN2_NONE || v < i);
+		((BUN2type *) h->Link)[i] = (BUN2type) v;
+		break;
+#endif
+	default:		/* BUN4 */
+		assert(v == BUN_NONE || v == BUN4_NONE || v < i);
+		((BUN4type *) h->Link)[i] = (BUN4type) v;
+		break;
+#ifdef BUN8
+	case BUN8:
+		assert(v == BUN_NONE || v == BUN8_NONE || v < i);
+		((BUN8type *) h->Link)[i] = (BUN8type) v;
+		break;
+#endif
+	}
+}
+
+static inline BUN __attribute__((__pure__))
+HASHget(const Hash *h, BUN i)
+{
+	switch (h->width) {
+#ifdef BUN2
+	case BUN2:
+		i = (BUN) ((BUN2type *) h->Bckt)[i];
+		return i == BUN2_NONE ? BUN_NONE : i;
+#endif
+	default:		/* BUN4 */
+		i = (BUN) ((BUN4type *) h->Bckt)[i];
+		return i == BUN4_NONE ? BUN_NONE : i;
+#ifdef BUN8
+	case BUN8:
+		i = (BUN) ((BUN8type *) h->Bckt)[i];
+		return i == BUN8_NONE ? BUN_NONE : i;
+#endif
+	}
+}
+
+static inline BUN __attribute__((__pure__))
+HASHgetlink(const Hash *h, BUN i)
+{
+	switch (h->width) {
+#ifdef BUN2
+	case BUN2:
+		i = (BUN) ((BUN2type *) h->Link)[i];
+		return i == BUN2_NONE ? BUN_NONE : i;
+#endif
+	default:		/* BUN4 */
+		i = (BUN) ((BUN4type *) h->Link)[i];
+		return i == BUN4_NONE ? BUN_NONE : i;
+#ifdef BUN8
+	case BUN8:
+		i = (BUN) ((BUN8type *) h->Link)[i];
+		return i == BUN8_NONE ? BUN_NONE : i;
+#endif
+	}
+}
+
+/* mix_bte(0x80) == 0x80 */
+#define mix_bte(X)	((unsigned int) (unsigned char) (X))
+/* mix_sht(0x8000) == 0x8000 */
+#define mix_sht(X)	((unsigned int) (unsigned short) (X))
+/* mix_int(0x81060038) == 0x80000000 */
+#define mix_int(X)	(((unsigned int) (X) >> 7) ^	\
+			 ((unsigned int) (X) >> 13) ^	\
+			 ((unsigned int) (X) >> 21) ^	\
+			 (unsigned int) (X))
+/* mix_lng(0x810600394347424F) == 0x8000000000000000 */
+#define mix_lng(X)	(((ulng) (X) >> 7) ^	\
+			 ((ulng) (X) >> 13) ^	\
+			 ((ulng) (X) >> 21) ^	\
+			 ((ulng) (X) >> 31) ^	\
+			 ((ulng) (X) >> 38) ^	\
+			 ((ulng) (X) >> 46) ^	\
+			 ((ulng) (X) >> 56) ^	\
+			 (ulng) (X))
+#ifdef HAVE_HGE
+/* mix_hge(0x810600394347424F90AC1429D6BFCC57) ==
+ * 0x80000000000000000000000000000000 */
+#define mix_hge(X)	(((uhge) (X) >> 7) ^	\
+			 ((uhge) (X) >> 13) ^	\
+			 ((uhge) (X) >> 21) ^	\
+			 ((uhge) (X) >> 31) ^	\
+			 ((uhge) (X) >> 38) ^	\
+			 ((uhge) (X) >> 46) ^	\
+			 ((uhge) (X) >> 56) ^	\
+			 ((uhge) (X) >> 65) ^	\
+			 ((uhge) (X) >> 70) ^	\
+			 ((uhge) (X) >> 78) ^	\
+			 ((uhge) (X) >> 85) ^	\
+			 ((uhge) (X) >> 90) ^	\
+			 ((uhge) (X) >> 98) ^	\
+			 ((uhge) (X) >> 107) ^	\
+			 ((uhge) (X) >> 116) ^	\
+			 (uhge) (X))
+#endif
+#define hash_loc(H,V)	hash_any(H,V)
+#define hash_var(H,V)	hash_any(H,V)
+#define hash_any(H,V)	HASHbucket(H, ATOMhash((H)->type, (V)))
+#define hash_bte(H,V)	(assert((H)->nbucket >= 256), (BUN) mix_bte(*(const unsigned char*) (V)))
+#define hash_sht(H,V)	(assert((H)->nbucket >= 65536), (BUN) mix_sht(*(const unsigned short*) (V)))
+#define hash_int(H,V)	HASHbucket(H, (BUN) mix_int(*(const unsigned int *) (V)))
+/* XXX return size_t-sized value for 8-byte oid? */
+#define hash_lng(H,V)	HASHbucket(H, (BUN) mix_lng(*(const ulng *) (V)))
+#ifdef HAVE_HGE
+#define hash_hge(H,V)	HASHbucket(H, (BUN) mix_hge(*(const uhge *) (V)))
+#endif
+#if SIZEOF_OID == SIZEOF_INT
+#define hash_oid(H,V)	hash_int(H,V)
+#else
+#define hash_oid(H,V)	hash_lng(H,V)
+#endif
+
+#define hash_flt(H,V)	hash_int(H,V)
+#define hash_dbl(H,V)	hash_lng(H,V)
+
+static inline BUN __attribute__((__const__))
+mix_uuid(const uuid *u)
+{
+	ulng u1, u2;
+
+	u1 = (ulng) (uint8_t) u->u[0] << 56 |
+		(ulng) (uint8_t) u->u[1] << 48 |
+		(ulng) (uint8_t) u->u[2] << 40 |
+		(ulng) (uint8_t) u->u[3] << 32 |
+		(ulng) (uint8_t) u->u[4] << 24 |
+		(ulng) (uint8_t) u->u[5] << 16 |
+		(ulng) (uint8_t) u->u[6] << 8 |
+		(ulng) (uint8_t) u->u[7];
+	u2 = (ulng) (uint8_t) u->u[8] << 56 |
+		(ulng) (uint8_t) u->u[9] << 48 |
+		(ulng) (uint8_t) u->u[10] << 40 |
+		(ulng) (uint8_t) u->u[11] << 32 |
+		(ulng) (uint8_t) u->u[12] << 24 |
+		(ulng) (uint8_t) u->u[13] << 16 |
+		(ulng) (uint8_t) u->u[14] << 8 |
+		(ulng) (uint8_t) u->u[15];
+	/* we're not using mix_hge since this way we get the same result
+	 * on systems with and without 128 bit integer support */
+	return (BUN) (mix_lng(u1) ^ mix_lng(u2));
+}
+#define hash_uuid(H,V)	HASHbucket(H, mix_uuid((const uuid *) (V)))
+
+/*
+ * @- hash-table supported loop over BUNs The first parameter `bi' is
+ * a BAT iterator, the second (`h') should point to the Hash
+ * structure, and `v' a pointer to an atomic value (corresponding to
+ * the head column of `b'). The 'hb' is an BUN index, pointing out the
+ * `hb'-th BUN.
+ */
+#define HASHloop(bi, h, hb, v)					\
+	for (hb = HASHget(h, HASHprobe(h, v));			\
+	     hb != BUN_NONE;					\
+	     hb = HASHgetlink(h, hb))				\
+		if (ATOMcmp(h->type, v, BUNtail(bi, hb)) == 0)
+#define HASHloop_str(bi, h, hb, v)				\
+	for (hb = HASHget(h, HASHbucket(h, strHash(v)));	\
+	     hb != BUN_NONE;					\
+	     hb = HASHgetlink(h, hb))				\
+		if (strEQ(v, BUNtvar(bi, hb)))
+
+#define HASHlooploc(bi, h, hb, v)				\
+	for (hb = HASHget(h, HASHprobe(h, v));			\
+	     hb != BUN_NONE;					\
+	     hb = HASHgetlink(h, hb))				\
+		if (ATOMcmp(h->type, v, BUNtloc(bi, hb)) == 0)
+#define HASHloopvar(bi, h, hb, v)				\
+	for (hb = HASHget(h, HASHprobe(h, v));			\
+	     hb != BUN_NONE;					\
+	     hb = HASHgetlink(h, hb))				\
+		if (ATOMcmp(h->type, v, BUNtvar(bi, hb)) == 0)
+
+#define HASHloop_TYPE(bi, h, hb, v, TYPE)				\
+	for (hb = HASHget(h, hash_##TYPE(h, v));			\
+	     hb != BUN_NONE;						\
+	     hb = HASHgetlink(h,hb))					\
+		if (* (const TYPE *) (v) == * (const TYPE *) BUNtloc(bi, hb))
+
+/* need to take special care comparing nil floating point values */
+#define HASHloop_fTYPE(bi, h, hb, v, TYPE)				\
+	for (hb = HASHget(h, hash_##TYPE(h, v));			\
+	     hb != BUN_NONE;						\
+	     hb = HASHgetlink(h,hb))					\
+		if (is_##TYPE##_nil(* (const TYPE *) (v))		\
+		    ? is_##TYPE##_nil(* (const TYPE *) BUNtloc(bi, hb)) \
+		    : * (const TYPE *) (v) == * (const TYPE *) BUNtloc(bi, hb))
+
+#define HASHloop_bte(bi, h, hb, v)	HASHloop_TYPE(bi, h, hb, v, bte)
+#define HASHloop_sht(bi, h, hb, v)	HASHloop_TYPE(bi, h, hb, v, sht)
+#define HASHloop_int(bi, h, hb, v)	HASHloop_TYPE(bi, h, hb, v, int)
+#define HASHloop_lng(bi, h, hb, v)	HASHloop_TYPE(bi, h, hb, v, lng)
+#ifdef HAVE_HGE
+#define HASHloop_hge(bi, h, hb, v)	HASHloop_TYPE(bi, h, hb, v, hge)
+#endif
+#define HASHloop_flt(bi, h, hb, v)	HASHloop_fTYPE(bi, h, hb, v, flt)
+#define HASHloop_dbl(bi, h, hb, v)	HASHloop_fTYPE(bi, h, hb, v, dbl)
+#ifdef HAVE_HGE
+#define HASHloop_uuid(bi, hsh, hb, v)					\
+	for (hb = HASHget(hsh, hash_uuid(hsh, v));			\
+	     hb != BUN_NONE;						\
+	     hb = HASHgetlink(hsh,hb))					\
+		if (((const uuid *) (v))->h == ((const uuid *) BUNtloc(bi, hb))->h)
+#else
+#define HASHloop_uuid(bi, h, hb, v)					\
+	for (hb = HASHget(h, hash_uuid(h, v));				\
+	     hb != BUN_NONE;						\
+	     hb = HASHgetlink(h,hb))					\
+		if (memcmp((const uuid *) (v), (const uuid *) BUNtloc(bi, hb), 16) == 0)
+//		if (((const uuid *) (v))->l[0] == ((const uuid *) BUNtloc(bi, hb))->l[0] && ((const uuid *) (v))->l[1] == ((const uuid *) BUNtloc(bi, hb))->l[1])
+#endif
+
+#endif /* _GDK_SEARCH_H_ */

monetdb/msvc/gdk_posix.h

@@ -0,0 +1,197 @@
+/*
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0.  If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright 1997 - July 2008 CWI, August 2008 - 2022 MonetDB B.V.
+ */
+
+#ifndef GDK_POSIX_H
+#define GDK_POSIX_H
+
+#include <sys/types.h>
+
+#include <time.h>
+
+#ifdef HAVE_FTIME
+#include <sys/timeb.h>		/* ftime */
+#endif
+#ifdef HAVE_SYS_TIME_H
+#include <sys/time.h>		/* gettimeofday */
+#endif
+
+#ifndef HAVE_SYS_SOCKET_H
+#ifdef HAVE_WINSOCK_H
+#include <winsock.h>		/* for timeval */
+#endif
+#endif
+
+#include "gdk_system.h" /* gdk_export */
+
+#ifdef NATIVE_WIN32
+#include <io.h>
+#include <direct.h>
+#endif
+
+/*
+ * @- virtual memory
+ */
+#define MT_VMUNITLOG	16
+#define MT_VMUNITSIZE	(1 << MT_VMUNITLOG)
+
+/* make sure POSIX_MADV_* and posix_madvise() are defined somehow */
+#ifdef HAVE_SYS_MMAN_H
+# ifndef __USE_BSD
+#  define __USE_BSD
+# endif
+# include <sys/mman.h>
+#endif
+
+#ifdef __linux__
+/* on Linux, posix_madvise does not seem to work, fall back to classic
+ * madvise */
+#undef HAVE_POSIX_MADVISE
+#undef HAVE_POSIX_FADVISE
+#undef POSIX_MADV_NORMAL
+#undef POSIX_MADV_RANDOM
+#undef POSIX_MADV_SEQUENTIAL
+#undef POSIX_MADV_WILLNEED
+#undef POSIX_MADV_DONTNEED
+#endif
+
+#ifndef HAVE_POSIX_MADVISE
+# ifdef HAVE_MADVISE
+#  define posix_madvise madvise
+#  define HAVE_POSIX_MADVISE 1
+#  ifndef MADV_RANDOM
+#   define MADV_RANDOM	0
+#  endif
+#  ifndef POSIX_MADV_NORMAL
+#   define POSIX_MADV_NORMAL     MADV_NORMAL
+#   define POSIX_MADV_RANDOM     MADV_RANDOM
+#   define POSIX_MADV_SEQUENTIAL MADV_SEQUENTIAL
+#   define POSIX_MADV_WILLNEED   MADV_WILLNEED
+#   define POSIX_MADV_DONTNEED   MADV_DONTNEED
+#  endif
+# else
+#  define posix_madvise(x,y,z)	0
+#  ifndef POSIX_MADV_NORMAL
+#   define POSIX_MADV_NORMAL     0
+#   define POSIX_MADV_RANDOM     0
+#   define POSIX_MADV_SEQUENTIAL 0
+#   define POSIX_MADV_WILLNEED   0
+#   define POSIX_MADV_DONTNEED   0
+#  endif
+# endif
+#endif
+
+/* in case they are still not defined, define these values as
+ * something that doesn't do anything */
+#ifndef POSIX_MADV_NORMAL
+#define POSIX_MADV_NORMAL 0
+#endif
+#ifndef POSIX_MADV_RANDOM
+#define POSIX_MADV_RANDOM 0
+#endif
+#ifndef POSIX_MADV_SEQUENTIAL
+#define POSIX_MADV_SEQUENTIAL 0
+#endif
+#ifndef POSIX_MADV_WILLNEED
+#define POSIX_MADV_WILLNEED 0
+#endif
+#ifndef POSIX_MADV_DONTNEED
+#define POSIX_MADV_DONTNEED 0
+#endif
+
+/* the new mmap modes, mimic default MADV_* madvise POSIX constants */
+#define MMAP_NORMAL	POSIX_MADV_NORMAL	/* no further special treatment */
+#define MMAP_RANDOM	POSIX_MADV_RANDOM	/* expect random page references */
+#define MMAP_SEQUENTIAL	POSIX_MADV_SEQUENTIAL	/* expect sequential page references */
+#define MMAP_WILLNEED	POSIX_MADV_WILLNEED	/* will need these pages */
+#define MMAP_DONTNEED	POSIX_MADV_DONTNEED	/* don't need these pages */
+
+#define MMAP_READ		1024	/* region is readable (default if ommitted) */
+#define MMAP_WRITE		2048	/* region may be written into */
+#define MMAP_COPY		4096	/* writable, but changes never reach file */
+#define MMAP_ASYNC		8192	/* asynchronous writes (default if ommitted) */
+#define MMAP_SYNC		16384	/* writing is done synchronously */
+
+/* in order to be sure of madvise and msync modes, pass them to mmap()
+ * call as well */
+
+gdk_export size_t MT_getrss(void);
+
+gdk_export bool MT_path_absolute(const char *path);
+
+
+/*
+ * @+ Posix under WIN32
+ * WIN32 actually supports many Posix functions directly.  Some it
+ * does not, though.  For some functionality we move in Monet from
+ * Posix calls to MT_*() calls, which translate easier to WIN32.
+ * Examples are MT_mmap() , MT_sleep_ms() and MT_path_absolute(). Why?
+ * In the case of mmap() it is much easier for WIN32 to get a filename
+ * parameter rather than a file-descriptor.  That is the reason in the
+ * case of mmap() to go for a MT_mmap() solution.
+ *
+ * For some other functionality, we do not need to abandon the Posix
+ * interface, though. Two cases can be distinguished.  Missing
+ * functions in WIN32 are directly implemented
+ * (e.g. dlopen()/dlsym()/dlclose()).  Posix functions in WIN32 whose
+ * functionality should be changed a bit. Examples are
+ * stat()/rename()/mkdir()/rmdir() who under WIN32 do not work if the
+ * path ends with a directory separator, but should work according to
+ * Posix. We remap such functions using a define to an equivalent
+ * win_*() function (which in its implementation calls through to the
+ * WIN32 function).
+ */
+gdk_export void *mdlopen(const char *library, int mode);
+
+
+#ifdef NATIVE_WIN32
+
+#define RTLD_LAZY	1
+#define RTLD_NOW	2
+#define RTLD_GLOBAL	4
+
+gdk_export void *dlopen(const char *file, int mode);
+gdk_export int dlclose(void *handle);
+gdk_export void *dlsym(void *handle, const char *name);
+gdk_export char *dlerror(void);
+
+#ifndef HAVE_GETTIMEOFDAY
+gdk_export int gettimeofday(struct timeval *tv, int *ignore_zone);
+#endif
+
+#endif	/* NATIVE_WIN32 */
+
+#ifndef HAVE_LOCALTIME_R
+gdk_export struct tm *localtime_r(const time_t *restrict, struct tm *restrict);
+#endif
+#ifndef HAVE_GMTIME_R
+gdk_export struct tm *gmtime_r(const time_t *restrict, struct tm *restrict);
+#endif
+#ifndef HAVE_ASCTIME_R
+gdk_export char *asctime_r(const struct tm *restrict, char *restrict);
+#endif
+#ifndef HAVE_CTIME_R
+gdk_export char *ctime_r(const time_t *restrict, char *restrict);
+#endif
+#ifndef HAVE_STRERROR_R
+gdk_export int strerror_r(int errnum, char *buf, size_t buflen);
+#endif
+
+static inline const char *
+GDKstrerror(int errnum, char *buf, size_t buflen)
+{
+#if !defined(_GNU_SOURCE) || ((_POSIX_C_SOURCE >= 200112L) && !_GNU_SOURCE)
+	if (strerror_r(errnum, buf, buflen) == 0)
+		return buf;
+	snprintf(buf, buflen, "Unknown error %d", errnum);
+	return buf;
+#else
+	return strerror_r(errnum, buf, buflen);
+#endif
+}
+
+#endif /* GDK_POSIX_H */

monetdb/msvc/gdk_system.h

@@ -0,0 +1,722 @@
+/*
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0.  If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright 1997 - July 2008 CWI, August 2008 - 2022 MonetDB B.V.
+ */
+
+#ifndef _GDK_SYSTEM_H_
+#define _GDK_SYSTEM_H_
+
+#ifdef WIN32
+#ifndef LIBGDK
+#define gdk_export extern __declspec(dllimport)
+#else
+#define gdk_export extern __declspec(dllexport)
+#endif
+#else
+#define gdk_export extern
+#endif
+
+/* if __has_attribute is not known to the preprocessor, we ignore
+ * attributes completely; if it is known, use it to find out whether
+ * specific attributes that we use are known */
+#ifndef __has_attribute
+#ifndef __GNUC__
+/* we can define __has_attribute as 1 since we define __attribute__ as empty */
+#define __has_attribute(attr)	1
+#ifndef __attribute__
+#define __attribute__(attr)	/* empty */
+#endif
+#else
+/* older GCC does have attributes, but not __has_attribute and not all
+ * attributes that we use are known */
+#define __has_attribute__alloc_size__ 1
+#define __has_attribute__cold__ 1
+#define __has_attribute__const__ 1
+#define __has_attribute__constructor__ 1
+#define __has_attribute__designated_init__ 0
+#define __has_attribute__format__ 1
+#define __has_attribute__malloc__ 1
+#define __has_attribute__nonnull__ 1
+#define __has_attribute__nonstring__ 0
+#define __has_attribute__pure__ 1
+#define __has_attribute__returns_nonnull__ 0
+#define __has_attribute__visibility__ 1
+#define __has_attribute__warn_unused_result__ 1
+#define __has_attribute(attr)	__has_attribute##attr
+#endif
+#endif
+#if !__has_attribute(__alloc_size__)
+#define __alloc_size__(a)
+#endif
+#if !__has_attribute(__cold__)
+#define __cold__
+#endif
+#if !__has_attribute(__const__)
+#define __const__
+#endif
+#if !__has_attribute(__constructor__)
+#define __constructor__
+#endif
+#if !__has_attribute(__designated_init__)
+#define __designated_init__
+#endif
+#if !__has_attribute(__format__)
+#define __format__(a,b,c)
+#endif
+#if !__has_attribute(__malloc__)
+#define __malloc__
+#endif
+#if !__has_attribute(__nonnull__)
+#define __nonnull__(a)
+#endif
+#if !__has_attribute(__nonstring__)
+#define __nonstring__
+#endif
+#if !__has_attribute(__pure__)
+#define __pure__
+#endif
+#if !__has_attribute(__returns_nonnull__)
+#define __returns_nonnull__
+#endif
+#if !__has_attribute(__visibility__)
+#define __visibility__(a)
+#elif defined(__CYGWIN__)
+#define __visibility__(a)
+#endif
+#if !__has_attribute(__warn_unused_result__)
+#define __warn_unused_result__
+#endif
+
+/* also see gdk.h for these */
+#define THRDMASK	(1)
+#define TEMMASK		(1<<10)
+
+/*
+ * @- pthreads Includes and Definitions
+ */
+#ifdef HAVE_PTHREAD_H
+/* don't re-include config.h; on Windows, don't redefine pid_t in an
+ * incompatible way */
+#undef HAVE_CONFIG_H
+#ifdef pid_t
+#undef pid_t
+#endif
+#include <sched.h>
+#include <pthread.h>
+#endif
+
+#ifdef HAVE_SEMAPHORE_H
+# include <semaphore.h>
+#endif
+
+#ifdef HAVE_DISPATCH_DISPATCH_H
+#include <dispatch/dispatch.h>
+#endif
+
+#ifdef HAVE_SYS_PARAM_H
+# include <sys/param.h>	   /* prerequisite of sys/sysctl on OpenBSD */
+#endif
+#ifdef BSD /* BSD macro is defined in sys/param.h */
+# include <sys/sysctl.h>
+#endif
+
+/* new pthread interface, where the thread id changed to a struct */
+#ifdef PTW32_VERSION
+#define PTW32 1
+#endif
+
+/* debug and errno integers */
+gdk_export int GDKdebug;
+gdk_export void GDKsetdebug(int debug);
+gdk_export int GDKgetdebug(void);
+
+gdk_export int GDKnr_threads;
+
+/* API */
+
+/*
+ * @- sleep
+ */
+
+gdk_export void MT_sleep_ms(unsigned int ms);
+
+/*
+ * @- MT Thread Api
+ */
+typedef size_t MT_Id;		/* thread number. will not be zero */
+
+enum MT_thr_detach { MT_THR_JOINABLE, MT_THR_DETACHED };
+#define MT_NAME_LEN	32	/* length of thread/semaphore/etc. names */
+
+#define UNKNOWN_THREAD "unknown thread"
+typedef int64_t lng;
+
+typedef struct QryCtx {
+	const lng starttime;
+	lng querytimeout;
+} QryCtx;
+
+gdk_export bool MT_thread_init(void);
+gdk_export int MT_create_thread(MT_Id *t, void (*function) (void *),
+				void *arg, enum MT_thr_detach d,
+				const char *threadname);
+gdk_export const char *MT_thread_getname(void);
+gdk_export void *MT_thread_getdata(void);
+gdk_export void MT_thread_setdata(void *data);
+gdk_export void MT_exiting_thread(void);
+gdk_export MT_Id MT_getpid(void);
+gdk_export int MT_join_thread(MT_Id t);
+gdk_export QryCtx *MT_thread_get_qry_ctx(void);
+gdk_export void MT_thread_set_qry_ctx(QryCtx *ctx);
+
+#if SIZEOF_VOID_P == 4
+/* "limited" stack size on 32-bit systems */
+/* to avoid address space fragmentation   */
+#define THREAD_STACK_SIZE	((size_t)1024*1024)
+#else
+/* "increased" stack size on 64-bit systems    */
+/* since some compilers seem to require this   */
+/* for burg-generated code in pathfinder       */
+/* and address space fragmentation is no issue */
+#define THREAD_STACK_SIZE	((size_t)2*1024*1024)
+#endif
+
+
+/*
+ * @- MT Lock API
+ */
+#include "matomic.h"
+
+/* define this to keep lock statistics (can be expensive) */
+/* #define LOCK_STATS 1 */
+
+#ifdef LOCK_STATS
+#include "gdk_tracer.h"
+
+#define _DBG_LOCK_COUNT_0(l)					\
+	do {							\
+		(void) ATOMIC_INC(&GDKlockcnt);			\
+		TRC_DEBUG(TEM, "Locking %s...\n", (l)->name); 	\
+	} while (0)
+
+#define _DBG_LOCK_LOCKER(l)				\
+	do {						\
+		(l)->locker = __func__;			\
+		(l)->thread = MT_thread_getname();	\
+	} while (0)
+
+#define _DBG_LOCK_UNLOCKER(l)					\
+	do {							\
+		(l)->locker = __func__;				\
+		(l)->thread = NULL;				\
+		TRC_DEBUG(TEM, "Unlocking %s\n", (l)->name);	\
+	} while (0)
+
+#define _DBG_LOCK_CONTENTION(l)						\
+	do {								\
+		TRC_DEBUG(TEM, "Lock %s contention\n", (l)->name); 	\
+		(void) ATOMIC_INC(&GDKlockcontentioncnt);		\
+		(void) ATOMIC_INC(&(l)->contention);			\
+	} while (0)
+
+#define _DBG_LOCK_SLEEP(l)	((void) ATOMIC_INC(&(l)->sleep))
+
+#define _DBG_LOCK_COUNT_2(l)						\
+	do {								\
+		(l)->count++;						\
+		if ((l)->next == (struct MT_Lock *) -1) {		\
+			while (ATOMIC_TAS(&GDKlocklistlock) != 0)	\
+				;					\
+			(l)->next = GDKlocklist;			\
+			(l)->prev = NULL;				\
+			if (GDKlocklist)				\
+				GDKlocklist->prev = (l);		\
+			GDKlocklist = (l);				\
+			ATOMIC_CLEAR(&GDKlocklistlock);			\
+		}							\
+		TRC_DEBUG(TEM, "Locking %s complete\n", (l)->name);	\
+	} while (0)
+
+#define _DBG_LOCK_INIT(l)						\
+	do {								\
+		(l)->count = 0;						\
+		ATOMIC_INIT(&(l)->contention, 0);			\
+		ATOMIC_INIT(&(l)->sleep, 0);				\
+		(l)->locker = NULL;					\
+		(l)->thread = NULL;					\
+		/* if name starts with "sa_" don't link in GDKlocklist */ \
+		/* since the lock is in memory that is governed by the */ \
+		/* SQL storage allocator, and hence we have no control */ \
+		/* over when the lock is destroyed and the memory freed */ \
+		if (strncmp((l)->name, "sa_", 3) != 0) {		\
+			while (ATOMIC_TAS(&GDKlocklistlock) != 0)	\
+				;					\
+			if (GDKlocklist)				\
+				GDKlocklist->prev = (l);		\
+			(l)->next = GDKlocklist;			\
+			(l)->prev = NULL;				\
+			GDKlocklist = (l);				\
+			ATOMIC_CLEAR(&GDKlocklistlock);			\
+		} else {						\
+			(l)->next = NULL;				\
+			(l)->prev = NULL;				\
+		}							\
+	} while (0)
+
+#define _DBG_LOCK_DESTROY(l)						\
+	do {								\
+		/* if name starts with "sa_" don't link in GDKlocklist */ \
+		/* since the lock is in memory that is governed by the */ \
+		/* SQL storage allocator, and hence we have no control */ \
+		/* over when the lock is destroyed and the memory freed */ \
+		if (strncmp((l)->name, "sa_", 3) != 0) {		\
+			while (ATOMIC_TAS(&GDKlocklistlock) != 0)	\
+				;					\
+			if ((l)->next)					\
+				(l)->next->prev = (l)->prev;		\
+			if ((l)->prev)					\
+				(l)->prev->next = (l)->next;		\
+			else if (GDKlocklist == (l))			\
+				GDKlocklist = (l)->next;		\
+			ATOMIC_CLEAR(&GDKlocklistlock);			\
+			ATOMIC_DESTROY(&(l)->contention);		\
+			ATOMIC_DESTROY(&(l)->sleep);			\
+		}							\
+	} while (0)
+
+#else
+
+#define _DBG_LOCK_COUNT_0(l)		((void) 0)
+#define _DBG_LOCK_CONTENTION(l)		((void) 0)
+#define _DBG_LOCK_SLEEP(l)		((void) 0)
+#define _DBG_LOCK_COUNT_2(l)		((void) 0)
+#define _DBG_LOCK_INIT(l)		((void) 0)
+#define _DBG_LOCK_DESTROY(l)		((void) 0)
+#define _DBG_LOCK_LOCKER(l)		((void) 0)
+#define _DBG_LOCK_UNLOCKER(l)		((void) 0)
+
+#endif
+
+#if !defined(HAVE_PTHREAD_H) && defined(WIN32)
+typedef struct MT_Lock {
+	CRITICAL_SECTION lock;
+	char name[MT_NAME_LEN];
+#ifdef LOCK_STATS
+	size_t count;
+	ATOMIC_TYPE contention;
+	ATOMIC_TYPE sleep;
+	struct MT_Lock *volatile next;
+	struct MT_Lock *volatile prev;
+	const char *locker;
+	const char *thread;
+#endif
+} MT_Lock;
+
+/* Windows defines read as _read and adds a deprecation warning to read
+ * if you were to still use that.  We need the token "read" here.  We
+ * cannot simply #undef read, since that messes up the deprecation
+ * stuff.  So we define _read as read to change the token back to "read"
+ * where replacement stops (recursive definitions are allowed in C and
+ * are handled well).  After our use, we remove the definition of _read
+ * so everything reverts back to the way it was.  Bonus: this also works
+ * if "read" was not defined. */
+#define _read read
+#pragma section(".CRT$XCU", read)
+#undef _read
+#ifdef _WIN64
+#define _LOCK_PREF_ ""
+#else
+#define _LOCK_PREF_ "_"
+#endif
+#define MT_LOCK_INITIALIZER(n) { 0 };					\
+static void wininit_##n(void)						\
+{									\
+	MT_lock_init(&n, #n);						\
+}									\
+__declspec(allocate(".CRT$XCU")) void (*wininit_##n##_)(void) = wininit_##n; \
+__pragma(comment(linker, "/include:" _LOCK_PREF_ "wininit_" #n "_"))
+
+#define MT_lock_init(l, n)					\
+	do {							\
+		InitializeCriticalSection(&(l)->lock);		\
+		strcpy_len((l)->name, (n), sizeof((l)->name));	\
+		_DBG_LOCK_INIT(l);				\
+	} while (0)
+
+#define MT_lock_try(l)	TryEnterCriticalSection(&(l)->lock)
+
+#define MT_lock_set(l)						\
+	do {							\
+		_DBG_LOCK_COUNT_0(l);				\
+		if (!MT_lock_try(l)) {				\
+			_DBG_LOCK_CONTENTION(l);		\
+			MT_thread_setlockwait(l);		\
+			EnterCriticalSection(&(l)->lock);	\
+			MT_thread_setlockwait(NULL);		\
+		}						\
+		_DBG_LOCK_LOCKER(l);				\
+		_DBG_LOCK_COUNT_2(l);				\
+	} while (0)
+
+#define MT_lock_unset(l)				\
+	do {						\
+		_DBG_LOCK_UNLOCKER(l);			\
+		LeaveCriticalSection(&(l)->lock);	\
+	} while (0)
+
+#define MT_lock_destroy(l)				\
+	do {						\
+		_DBG_LOCK_DESTROY(l);			\
+		DeleteCriticalSection(&(l)->lock);	\
+	} while (0)
+
+typedef struct MT_RWLock {
+	SRWLOCK lock;
+	char name[MT_NAME_LEN];
+} MT_RWLock;
+
+#define MT_RWLOCK_INITIALIZER(n)	{ .lock = SRWLOCK_INIT, .name = #n, }
+
+#define MT_rwlock_init(l, n)					\
+	do {							\
+		InitializeSRWLock(&(l)->lock);			\
+		strcpy_len((l)->name, (n), sizeof((l)->name));	\
+	 } while (0)
+
+#define MT_rwlock_destroy(l)	((void) 0)
+
+#define MT_rwlock_rdlock(l)	AcquireSRWLockShared(&(l)->lock)
+#define MT_rwlock_rdtry(l)	TryAcquireSRWLockShared(&(l)->lock)
+
+#define MT_rwlock_rdunlock(l)	ReleaseSRWLockShared(&(l)->lock)
+
+#define MT_rwlock_wrlock(l)	AcquireSRWLockExclusive(&(l)->lock)
+#define MT_rwlock_wrtry(l)	TryAcquireSRWLockExclusive(&(l)->lock)
+
+#define MT_rwlock_wrunlock(l)	ReleaseSRWLockExclusive(&(l)->lock)
+
+#else
+
+typedef struct MT_Lock {
+	pthread_mutex_t lock;
+	char name[MT_NAME_LEN];
+#ifdef LOCK_STATS
+	size_t count;
+	ATOMIC_TYPE contention;
+	ATOMIC_TYPE sleep;
+	struct MT_Lock *volatile next;
+	struct MT_Lock *volatile prev;
+	const char *locker;
+	const char *thread;
+#endif
+} MT_Lock;
+
+#ifdef LOCK_STATS
+#define MT_LOCK_INITIALIZER(n)	{ .lock = PTHREAD_MUTEX_INITIALIZER, .name = #n, .next = (struct MT_Lock *) -1, }
+#else
+#define MT_LOCK_INITIALIZER(n)	{ .lock = PTHREAD_MUTEX_INITIALIZER, .name = #n, }
+#endif
+
+#define MT_lock_init(l, n)					\
+	do {							\
+		pthread_mutex_init(&(l)->lock, 0);		\
+		strcpy_len((l)->name, (n), sizeof((l)->name));	\
+		_DBG_LOCK_INIT(l);				\
+	} while (0)
+
+#define MT_lock_try(l)		(pthread_mutex_trylock(&(l)->lock) == 0)
+
+#ifdef LOCK_STATS
+#define MT_lock_set(l)					\
+	do {						\
+		_DBG_LOCK_COUNT_0(l);			\
+		if (!MT_lock_try(l)) {			\
+			_DBG_LOCK_CONTENTION(l);	\
+			MT_thread_setlockwait(l);	\
+			pthread_mutex_lock(&(l)->lock);	\
+			MT_thread_setlockwait(NULL);	\
+		}					\
+		_DBG_LOCK_LOCKER(l);			\
+		_DBG_LOCK_COUNT_2(l);			\
+	} while (0)
+#else
+#define MT_lock_set(l)		pthread_mutex_lock(&(l)->lock)
+#endif
+
+#define MT_lock_unset(l)				\
+	do {						\
+		_DBG_LOCK_UNLOCKER(l);			\
+		pthread_mutex_unlock(&(l)->lock);	\
+	} while (0)
+
+#define MT_lock_destroy(l)				\
+	do {						\
+		_DBG_LOCK_DESTROY(l);			\
+		pthread_mutex_destroy(&(l)->lock);	\
+	} while (0)
+
+#if !defined(__GLIBC__) || __GLIBC__ > 2 || (__GLIBC__ == 2 && defined(__GLIBC_MINOR__) && __GLIBC_MINOR__ >= 30)
+/* this is the normal implementation of our pthreads-based read-write lock */
+typedef struct MT_RWLock {
+	pthread_rwlock_t lock;
+	char name[MT_NAME_LEN];
+} MT_RWLock;
+
+#define MT_RWLOCK_INITIALIZER(n)				\
+	{ .lock = PTHREAD_RWLOCK_INITIALIZER, .name = #n, }
+
+#define MT_rwlock_init(l, n)					\
+	do {							\
+		pthread_rwlock_init(&(l)->lock, NULL);		\
+		strcpy_len((l)->name, (n), sizeof((l)->name));	\
+	 } while (0)
+
+#define MT_rwlock_destroy(l)	pthread_rwlock_destroy(&(l)->lock)
+
+#define MT_rwlock_rdlock(l)	pthread_rwlock_rdlock(&(l)->lock)
+#define MT_rwlock_rdtry(l)	(pthread_rwlock_tryrdlock(&(l)->lock) == 0)
+
+#define MT_rwlock_rdunlock(l)	pthread_rwlock_unlock(&(l)->lock)
+
+#define MT_rwlock_wrlock(l)	pthread_rwlock_wrlock(&(l)->lock)
+#define MT_rwlock_wrtry(l)	(pthread_rwlock_trywrlock(&(l)->lock) == 0)
+
+#define MT_rwlock_wrunlock(l)	pthread_rwlock_unlock(&(l)->lock)
+
+#else
+/* in glibc before 2.30, there was a deadlock condition in the tryrdlock
+ * and trywrlock functions, we work around that by not using the
+ * implementation at all
+ * see https://sourceware.org/bugzilla/show_bug.cgi?id=23844 for a
+ * discussion and comment 14 for the analysis */
+typedef struct MT_RWLock {
+	pthread_mutex_t lock;
+	ATOMIC_TYPE readers;
+	char name[MT_NAME_LEN];
+} MT_RWLock;
+
+#define MT_RWLOCK_INITIALIZER(n)				\
+	{ .lock = PTHREAD_MUTEX_INITIALIZER, .readers = ATOMIC_VAR_INIT(0), .name = #n, }
+
+#define MT_rwlock_init(l, n)					\
+	do {							\
+		pthread_mutex_init(&(l)->lock, 0);		\
+		ATOMIC_INIT(&(l)->readers, 0);			\
+		strcpy_len((l)->name, (n), sizeof((l)->name));	\
+	} while (0)
+
+#define MT_rwlock_destroy(l)				\
+	do {						\
+		pthread_mutex_destroy(&(l)->lock);	\
+		ATOMIC_DESTROY(&(l)->readers);		\
+	} while (0)
+
+#define MT_rwlock_rdlock(l)				\
+	do {						\
+		pthread_mutex_lock(&(l)->lock);		\
+		(void) ATOMIC_INC(&(l)->readers);	\
+		pthread_mutex_unlock(&(l)->lock);	\
+	} while (0)
+
+static inline bool
+MT_rwlock_rdtry(MT_RWLock *l)
+{
+	if (pthread_mutex_trylock(&l->lock) != 0)
+		return false;
+	(void) ATOMIC_INC(&(l)->readers);
+	pthread_mutex_unlock(&l->lock);
+	return true;
+}
+
+#define MT_rwlock_rdunlock(l)				\
+	do {						\
+		(void) ATOMIC_DEC(&(l)->readers);	\
+	} while (0)
+
+#define MT_rwlock_wrlock(l)				\
+	do {						\
+		pthread_mutex_lock(&(l)->lock);		\
+		while (ATOMIC_GET(&(l)->readers) > 0)	\
+			MT_sleep_ms(1);			\
+	} while (0)
+
+static inline bool
+MT_rwlock_wrtry(MT_RWLock *l)
+{
+	if (pthread_mutex_trylock(&l->lock) != 0)
+		return false;
+	if (ATOMIC_GET(&l->readers) > 0) {
+		pthread_mutex_unlock(&l->lock);
+		return false;
+	}
+	return true;
+}
+
+#define MT_rwlock_wrunlock(l)  pthread_mutex_unlock(&(l)->lock);
+
+#endif
+
+#endif
+
+#ifdef LOCK_STATS
+gdk_export void GDKlockstatistics(int);
+gdk_export MT_Lock * volatile GDKlocklist;
+gdk_export ATOMIC_FLAG GDKlocklistlock;
+gdk_export ATOMIC_TYPE GDKlockcnt;
+gdk_export ATOMIC_TYPE GDKlockcontentioncnt;
+gdk_export ATOMIC_TYPE GDKlocksleepcnt;
+#endif
+
+/*
+ * @- MT Semaphore API
+ */
+#if !defined(HAVE_PTHREAD_H) && defined(WIN32)
+
+typedef struct {
+	HANDLE sema;
+	char name[MT_NAME_LEN];
+} MT_Sema;
+
+#define MT_sema_init(s, nr, n)						\
+	do {								\
+		assert((s)->sema == NULL);				\
+		strcpy_len((s)->name, (n), sizeof((s)->name));		\
+		(s)->sema = CreateSemaphore(NULL, nr, 0x7fffffff, NULL); \
+	} while (0)
+
+#define MT_sema_destroy(s)			\
+	do {					\
+		assert((s)->sema != NULL);	\
+		CloseHandle((s)->sema);		\
+		(s)->sema = NULL;		\
+	} while (0)
+
+#define MT_sema_up(s)		ReleaseSemaphore((s)->sema, 1, NULL)
+
+#define MT_sema_down(s)							\
+	do {								\
+		TRC_DEBUG(TEM, "Sema %s down...\n", (s)->name);		\
+		if (WaitForSingleObject((s)->sema, 0) != WAIT_OBJECT_0) { \
+			MT_thread_setsemawait(s);			\
+			while (WaitForSingleObject((s)->sema, INFINITE) != WAIT_OBJECT_0) \
+				;					\
+			MT_thread_setsemawait(NULL);			\
+		}							\
+		TRC_DEBUG(TEM, "Sema %s down complete\n", (s)->name);	\
+	} while (0)
+
+#elif defined(HAVE_DISPATCH_SEMAPHORE_CREATE)
+
+/* MacOS X */
+typedef struct {
+	dispatch_semaphore_t sema;
+	char name[MT_NAME_LEN];
+} MT_Sema;
+
+#define MT_sema_init(s, nr, n)						\
+	do {								\
+		strcpy_len((s)->name, (n), sizeof((s)->name));		\
+		(s)->sema = dispatch_semaphore_create((long) (nr));	\
+	} while (0)
+
+#define MT_sema_destroy(s)	dispatch_release((s)->sema)
+#define MT_sema_up(s)		dispatch_semaphore_signal((s)->sema)
+#define MT_sema_down(s)		dispatch_semaphore_wait((s)->sema, DISPATCH_TIME_FOREVER)
+
+#elif defined(_AIX) || defined(__MACH__)
+
+/* simulate semaphores using mutex and condition variable */
+
+typedef struct {
+	int cnt;
+	pthread_mutex_t mutex;
+	pthread_cond_t cond;
+	char name[MT_NAME_LEN];
+} MT_Sema;
+
+#define MT_sema_init(s, nr, n)					\
+	do {							\
+		strcpy_len((s)->name, (n), sizeof((s)->name));	\
+		(s)->cnt = (nr);				\
+		pthread_mutex_init(&(s)->mutex, 0);		\
+		pthread_cond_init(&(s)->cond, 0);		\
+	} while (0)
+
+#define MT_sema_destroy(s)				\
+	do {						\
+		pthread_mutex_destroy(&(s)->mutex);	\
+		pthread_cond_destroy(&(s)->cond);	\
+	} while (0)
+
+#define MT_sema_up(s)						\
+	do {							\
+		pthread_mutex_lock(&(s)->mutex);		\
+		if ((s)->cnt++ < 0) {				\
+			pthread_cond_signal(&(s)->cond);	\
+		}						\
+		pthread_mutex_unlock(&(s)->mutex);		\
+	} while (0)
+
+#define MT_sema_down(s)							\
+	do {								\
+		TRC_DEBUG(TEM, "Sema %s down...\n", (s)->name);		\
+		pthread_mutex_lock(&(s)->mutex);			\
+		if (--(s)->cnt < 0) {					\
+			MT_thread_setsemawait(s);			\
+			do {						\
+				pthread_cond_wait(&(s)->cond,		\
+						  &(s)->mutex);		\
+			} while ((s)->cnt < 0);				\
+			MT_thread_setsemawait(NULL);			\
+			pthread_mutex_unlock(&(s)->mutex);		\
+		}							\
+		TRC_DEBUG(TEM, "Sema %s down complete\n", (s)->name);	\
+	} while (0)
+
+#else
+
+typedef struct {
+	sem_t sema;
+	char name[MT_NAME_LEN];
+} MT_Sema;
+
+#define MT_sema_init(s, nr, n)					\
+	do {							\
+		strcpy_len((s)->name, (n), sizeof((s)->name));	\
+		sem_init(&(s)->sema, 0, nr);			\
+	} while (0)
+
+#define MT_sema_destroy(s)	sem_destroy(&(s)->sema)
+
+#define MT_sema_up(s)						\
+	do {							\
+		TRC_DEBUG(TEM, "Sema %s up\n", (s)->name);	\
+		sem_post(&(s)->sema);				\
+	} while (0)
+
+#define MT_sema_down(s)							\
+	do {								\
+		TRC_DEBUG(TEM, "Sema %s down...\n", (s)->name);		\
+		if (sem_trywait(&(s)->sema) != 0) {			\
+			MT_thread_setsemawait(s);			\
+			while (sem_wait(&(s)->sema) != 0)		\
+				;					\
+			MT_thread_setsemawait(NULL);			\
+		}							\
+		TRC_DEBUG(TEM, "Sema %s down complete\n", (s)->name);	\
+	} while (0)
+
+#endif
+
+gdk_export void MT_thread_setlockwait(MT_Lock *lock);
+gdk_export void MT_thread_setsemawait(MT_Sema *sema);
+gdk_export void MT_thread_setworking(const char *work);
+gdk_export void MT_thread_setalgorithm(const char *algo);
+gdk_export const char *MT_thread_getalgorithm(void);
+
+gdk_export int MT_check_nr_cores(void);
+
+#endif /*_GDK_SYSTEM_H_*/

monetdb/msvc/gdk_tracer.h

@@ -0,0 +1,262 @@
+/*
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0.  If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright 1997 - July 2008 CWI, August 2008 - 2022 MonetDB B.V.
+ */
+
+/* // TODO: Complete it when documentation is accepted
+ *
+ * Tracer is the general logging system for the MonetDB stack modelled
+ * after the well-known logging schemes (e.g: Python). It provides a
+ * number of logging levels and options to increase or reduce the
+ * verbosity either of individual code parts or of the codebase as a
+ * whole. It allows users to focus on logging messages related to
+ * certain steps of execution, which can be proved handy when it comes
+ * to debugging. The behavior of Tracer can be controlled at runtime
+ * using the SQL API described later on. Certain calls require an "id"
+ * to operate which can be found on the list of each section below.
+ *
+ * Internally, the logger uses a buffer to capture log messages before
+ * they are forwarded to the specific adapter.
+ *
+ * - Sets the minimum flush level that an event will trigger the
+ *   logger to flush the buffer
+ * - Produces messages to the output stream. It is also used as a
+ *   fallback mechanism in case GDKtracer fails to log for whatever
+ *   reason.
+ * - Struct buffer with allocated space etc.
+ * - Flush buffer sends the messages to the selected adapter
+ * - Write about the log structure (e.g: MT_thread_get_name + datetime
+ *   + blah blah)
+ */
+
+#ifndef _GDK_TRACER_H_
+#define _GDK_TRACER_H_
+
+#define GENERATE_ENUM(ENUM) ENUM,
+
+
+// ADAPTERS
+#define FOREACH_ADPTR(ADPTR)			\
+	ADPTR( BASIC )				\
+	ADPTR( PROFILER )			\
+	ADPTR( MBEDDED )			\
+						\
+	ADPTR( ADAPTERS_COUNT )
+
+typedef enum {
+	FOREACH_ADPTR(GENERATE_ENUM)
+} adapter_t;
+
+
+
+// LOG LEVELS
+#define FOREACH_LEVEL(LEVEL)			\
+	LEVEL( M_CRITICAL )			\
+	LEVEL( M_ERROR )			\
+	LEVEL( M_WARNING )			\
+	LEVEL( M_INFO )				\
+	LEVEL( M_DEBUG )			\
+						\
+	LEVEL( LOG_LEVELS_COUNT )
+
+typedef enum {
+	FOREACH_LEVEL(GENERATE_ENUM)
+} log_level_t;
+
+
+// LAYERS
+#define FOREACH_LAYER(LAYER)			\
+	LAYER( MDB_ALL )			\
+	LAYER( SQL_ALL )			\
+	LAYER( MAL_ALL )			\
+	LAYER( GDK_ALL )			\
+						\
+	LAYER( LAYERS_COUNT )
+
+typedef enum {
+	FOREACH_LAYER(GENERATE_ENUM)
+} layer_t;
+
+
+
+
+/*
+ *
+ * NOTE: Adding/Removing components will affect the test tracer00.mal
+ * See the test file for more details.
+ *
+ */
+// COMPONENTS
+#define FOREACH_COMP(COMP)			\
+	COMP( ACCELERATOR )			\
+	COMP( ALGO )				\
+	COMP( ALLOC )				\
+	COMP( BAT_ )				\
+	COMP( CHECK_ )				\
+	COMP( DELTA )				\
+	COMP( HEAP )				\
+	COMP( IO_ )				\
+	COMP( PAR )				\
+	COMP( PERF )				\
+	COMP( TEM )				\
+	COMP( THRD )				\
+						\
+	COMP( GEOM )				\
+	COMP( FITS )				\
+	COMP( SHP )				\
+						\
+	COMP( SQL_PARSER )			\
+	COMP( SQL_TRANS )			\
+	COMP( SQL_REWRITER )			\
+	COMP( SQL_EXECUTION )			\
+	COMP( SQL_STORE )			\
+						\
+	COMP( MAL_WLC )				\
+	COMP( MAL_REMOTE )			\
+	COMP( MAL_MAPI )			\
+	COMP( MAL_SERVER )			\
+						\
+	COMP( MAL_OPTIMIZER )			\
+						\
+	COMP( GDK )				\
+						\
+	COMP( COMPONENTS_COUNT )
+
+typedef enum {
+	FOREACH_COMP(GENERATE_ENUM)
+} component_t;
+
+
+
+/*
+ * Logging macros
+ */
+gdk_export log_level_t lvl_per_component[];
+
+// If the LOG_LEVEL of the message is one of the following: CRITICAL,
+// ERROR or WARNING it is logged no matter the component. In any other
+// case the component is taken into account
+#define GDK_TRACER_TEST(LOG_LEVEL, COMP)	\
+	(LOG_LEVEL <= M_WARNING  ||		\
+	 lvl_per_component[COMP] >= LOG_LEVEL)
+
+
+#define GDK_TRACER_LOG_BODY(LOG_LEVEL, COMP, MSG, ...)			\
+	GDKtracer_log(__FILE__, __func__, __LINE__,			\
+		      LOG_LEVEL, COMP, NULL, MSG, ##__VA_ARGS__)
+
+#ifdef __COVERITY__
+/* hide this for static code analysis: too many false positives */
+#define GDK_TRACER_LOG(LOG_LEVEL, COMP, MSG, ...)	((void) 0)
+#else
+#define GDK_TRACER_LOG(LOG_LEVEL, COMP, MSG, ...)			\
+	do {								\
+		if (GDK_TRACER_TEST(LOG_LEVEL, COMP)) {			\
+			GDK_TRACER_LOG_BODY(LOG_LEVEL, COMP, MSG,	\
+					    ## __VA_ARGS__);		\
+		}							\
+	} while (0)
+#endif
+
+
+#define TRC_CRITICAL(COMP, MSG, ...)				\
+	GDK_TRACER_LOG_BODY(M_CRITICAL, COMP, MSG, ## __VA_ARGS__)
+
+#define TRC_ERROR(COMP, MSG, ...)				\
+	GDK_TRACER_LOG_BODY(M_ERROR, COMP, MSG, ## __VA_ARGS__)
+
+#define TRC_WARNING(COMP, MSG, ...)				\
+	GDK_TRACER_LOG_BODY(M_WARNING, COMP, MSG, ## __VA_ARGS__)
+
+#define TRC_INFO(COMP, MSG, ...)				\
+	GDK_TRACER_LOG(M_INFO, COMP, MSG, ## __VA_ARGS__)
+
+#define TRC_DEBUG(COMP, MSG, ...)				\
+	GDK_TRACER_LOG(M_DEBUG, COMP, MSG, ## __VA_ARGS__)
+
+
+
+// Conditional logging - Example usage
+// NOTE: When using the macro with *_IF always use the macro with
+// *_ENDIF for logging. Avoiding to do that will result into checking
+// the LOG_LEVEL of the the COMPONENT 2 times. Also NEVER use the
+// *_ENDIF macro without before performing a check with *_IF
+// macro. Such an action will have as a consequence logging everything
+// without taking into account the LOG_LEVEL of the COMPONENT.
+/*
+    TRC_CRITICAL_IF(SQL_STORE)
+    {
+	TRC_CRITICAL_ENDIF(SQL_STORE, "Test\n")
+    }
+*/
+#define TRC_CRITICAL_IF(COMP)			\
+	/* if (GDK_TRACER_TEST(M_CRITICAL, COMP)) */
+
+#define TRC_ERROR_IF(COMP)			\
+	/* if (GDK_TRACER_TEST(M_ERROR, COMP)) */
+
+#define TRC_WARNING_IF(COMP)			\
+	/* if (GDK_TRACER_TEST(M_WARNING, COMP)) */
+
+#define TRC_INFO_IF(COMP)			\
+	if (GDK_TRACER_TEST(M_INFO, COMP))
+
+#define TRC_DEBUG_IF(COMP)			\
+	if (GDK_TRACER_TEST(M_DEBUG, COMP))
+
+
+#define TRC_CRITICAL_ENDIF(COMP, MSG, ...)				\
+	GDK_TRACER_LOG_BODY(M_CRITICAL, COMP, MSG, ## __VA_ARGS__)
+
+#define TRC_ERROR_ENDIF(COMP, MSG, ...)					\
+	GDK_TRACER_LOG_BODY(M_ERROR, COMP, MSG, ## __VA_ARGS__)
+
+#define TRC_WARNING_ENDIF(COMP, MSG, ...)				\
+	GDK_TRACER_LOG_BODY(M_WARNING, COMP, MSG, ## __VA_ARGS__)
+
+#define TRC_INFO_ENDIF(COMP, MSG, ...)					\
+	GDK_TRACER_LOG_BODY(M_INFO, COMP, MSG, ## __VA_ARGS__)
+
+#define TRC_DEBUG_ENDIF(COMP, MSG, ...)					\
+	GDK_TRACER_LOG_BODY(M_DEBUG, COMP, MSG, ## __VA_ARGS__)
+
+
+
+/*
+ * GDKtracer API
+ * For the allowed log_levels, components and layers see the
+ * LOG_LEVEL, COMPONENT and LAYER enum respectively.
+ */
+// Used for logrotate
+gdk_export void GDKtracer_reinit_basic(int sig);
+
+gdk_export gdk_return GDKtracer_set_tracefile(const char *tracefile);
+
+gdk_export gdk_return GDKtracer_stop(void);
+
+gdk_export gdk_return GDKtracer_set_component_level(const char *comp, const char *lvl);
+gdk_export const char *GDKtracer_get_component_level(const char *comp);
+gdk_export gdk_return GDKtracer_reset_component_level(const char *comp);
+
+gdk_export gdk_return GDKtracer_set_layer_level(const char *layer, const char *lvl);
+gdk_export gdk_return GDKtracer_reset_layer_level(const char *layer);
+
+gdk_export gdk_return GDKtracer_set_flush_level(const char *lvl);
+gdk_export gdk_return GDKtracer_reset_flush_level(void);
+
+gdk_export gdk_return GDKtracer_set_adapter(const char *adapter);
+gdk_export gdk_return GDKtracer_reset_adapter(void);
+
+gdk_export void GDKtracer_log(const char *file, const char *func,
+			      int lineno, log_level_t lvl,
+			      component_t comp,
+			      const char *syserr,
+			      _In_z_ _Printf_format_string_ const char *format, ...)
+	__attribute__((__format__(__printf__, 7, 8)));
+
+gdk_export gdk_return GDKtracer_flush_buffer(void);
+
+#endif /* _GDK_TRACER_H_ */

monetdb/msvc/gdk_utils.h

@@ -0,0 +1,357 @@
+/*
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0.  If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright 1997 - July 2008 CWI, August 2008 - 2022 MonetDB B.V.
+ */
+
+#ifndef _GDK_UTILS_H_
+#define _GDK_UTILS_H_
+
+#include <setjmp.h>
+
+gdk_export const char *GDKgetenv(const char *name);
+
+gdk_export bool GDKgetenv_istext(const char *name, const char* text);
+gdk_export bool GDKgetenv_isyes(const char *name);
+gdk_export bool GDKgetenv_istrue(const char *name);
+
+gdk_export int GDKgetenv_int(const char *name, int def);
+
+gdk_export gdk_return GDKsetenv(const char *name, const char *value);
+gdk_export gdk_return GDKcopyenv(BAT **key, BAT **val, bool writable);
+
+/*
+ * @+ Memory management
+ * Memory management in GDK mostly relies on the facilities offered by
+ * the underlying OS.  The below routines monitor the available memory
+ * resources which consist of physical swap space and logical vm
+ * space.  There are three kinds of memory, that affect these two
+ * resources in different ways:
+ *
+ * - memory mapping
+ *   which ask for a logical region of virtual memory space.  In
+ *   principle, no physical memory is needed to keep the system afloat
+ *   here, as the memory mapped file is swapped onto a disk object
+ *   that already exists.
+ *
+ *   Actually, there are two kings of memory mapping used in GDK,
+ *   namely read-only direct mapped and writable copy-on write. For
+ *   the dirty pages, the latter actually also consumes physical
+ *   memory resources, but that is ignored here for simplicity.
+ *
+ * - anonymous virtual memory
+ *   This is virtual memory that is mapped on the swap file. Hence,
+ *   this consumes both logical VM space resources and physical memory
+ *   space.
+ *
+ * - malloced memory
+ *   comes from the heap and directly consumes physical memory
+ *   resources.
+ *
+ * The malloc routine checks the memory consumption every 1000 calls,
+ * or for calls larger that 50000 bytes. Consequently, at least every
+ * 50MB increase, alloc memory is checked. The VM calls always check
+ * the memory consumption.
+ */
+/* default setting to administer everything */
+#define GDK_MEM_NULLALLOWED
+
+#if SIZEOF_VOID_P==8
+#define GDK_VM_MAXSIZE	LL_CONSTANT(4398046511104)	/* :-) a 64-bit OS: 4TB */
+#else
+#define GDK_VM_MAXSIZE	LL_CONSTANT(1610612736)	/* :-| a 32-bit OS: 1.5GB */
+#endif
+/* virtual memory defines */
+gdk_export size_t _MT_npages;
+gdk_export size_t _MT_pagesize;
+
+#define MT_pagesize()	_MT_pagesize
+#define MT_npages()	_MT_npages
+
+gdk_export size_t GDK_mem_maxsize;	/* max allowed size of committed memory */
+gdk_export size_t GDK_vm_maxsize;	/* max allowed size of reserved vm */
+
+gdk_export void *GDKmmap(const char *path, int mode, size_t len)
+	__attribute__((__warn_unused_result__));
+gdk_export gdk_return GDKmunmap(void *addr, size_t len);
+
+gdk_export size_t GDKmem_cursize(void);	/* RAM/swapmem that MonetDB has claimed from OS */
+gdk_export size_t GDKvm_cursize(void);	/* current MonetDB VM address space usage */
+
+gdk_export void *GDKmalloc(size_t size)
+	__attribute__((__malloc__))
+	__attribute__((__alloc_size__(1)))
+	__attribute__((__warn_unused_result__));
+gdk_export void *GDKzalloc(size_t size)
+	__attribute__((__malloc__))
+	__attribute__((__alloc_size__(1)))
+	__attribute__((__warn_unused_result__));
+gdk_export void *GDKrealloc(void *pold, size_t size)
+	__attribute__((__alloc_size__(2)))
+	__attribute__((__warn_unused_result__));
+gdk_export void GDKfree(void *blk);
+gdk_export str GDKstrdup(const char *s)
+	__attribute__((__malloc__))
+	__attribute__((__warn_unused_result__));
+gdk_export str GDKstrndup(const char *s, size_t n)
+	__attribute__((__malloc__))
+	__attribute__((__warn_unused_result__));
+gdk_export size_t GDKmallocated(const void *s);
+
+gdk_export void MT_init(void);	/*  init the package. */
+struct opt;
+gdk_export gdk_return GDKinit(struct opt *set, int setlen, bool embedded);
+
+/* used for testing only */
+gdk_export void GDKsetmallocsuccesscount(lng count);
+
+/*
+ * Upon closing the session, all persistent BATs should be saved and
+ * the transient BATs should be removed.  The buffer pool manager
+ * takes care of this.
+ */
+gdk_export void GDKexit(int status);
+gdk_export bool GDKexiting(void);
+
+gdk_export void GDKprepareExit(void);
+gdk_export void GDKreset(int status);
+/* global version number */
+gdk_export const char *GDKversion(void)
+	__attribute__((__const__));
+/* ABI version of GDK library */
+gdk_export const char *GDKlibversion(void)
+	__attribute__((__const__));
+
+// these are used in embedded mode to jump out of GDKfatal
+gdk_export jmp_buf GDKfataljump;
+gdk_export str GDKfatalmsg;
+gdk_export bit GDKfataljumpenable;
+
+/* Timers
+ * The following relative timers are available for inspection.
+ * Note that they may consume recognizable overhead.
+ *
+ */
+gdk_export lng GDKusec(void);
+gdk_export int GDKms(void);
+
+
+#if !defined(NDEBUG) && !defined(__COVERITY__)
+/* In debugging mode, replace GDKmalloc and other functions with a
+ * version that optionally prints calling information.
+ *
+ * We have two versions of this code: one using a GNU C extension, and
+ * one using traditional C.  The GNU C version also prints the name of
+ * the calling function.
+ */
+#ifdef __GNUC__
+#define GDKmalloc(s)						\
+	({							\
+		size_t _size = (s);				\
+		void *_res = GDKmalloc(_size);			\
+		TRC_DEBUG(ALLOC, "GDKmalloc(%zu) -> %p\n",	\
+			  _size, _res);				\
+		_res;						\
+	})
+#define GDKzalloc(s)						\
+	({							\
+		size_t _size = (s);				\
+		void *_res = GDKzalloc(_size);			\
+		TRC_DEBUG(ALLOC, "GDKzalloc(%zu) -> %p\n",	\
+			  _size, _res);				\
+		_res;						\
+	})
+#define GDKrealloc(p, s)					\
+	({							\
+		void *_ptr = (p);				\
+		size_t _size = (s);				\
+		char _buf[12];					\
+		snprintf(_buf, sizeof(_buf), "%p", _ptr);	\
+		void *_res = GDKrealloc(_ptr, _size);		\
+		TRC_DEBUG(ALLOC, "GDKrealloc(%s,%zu) -> %p\n",	\
+			  _buf, _size, _res);			\
+		_res;						\
+	 })
+#define GDKfree(p)							\
+	({								\
+		void *_ptr = (p);					\
+		if (_ptr)						\
+			TRC_DEBUG(ALLOC, "GDKfree(%p)\n", _ptr);	\
+		GDKfree(_ptr);						\
+	})
+#define GDKstrdup(s)							\
+	({								\
+		const char *_str = (s);					\
+		void *_res = GDKstrdup(_str);				\
+		TRC_DEBUG(ALLOC, "GDKstrdup(len=%zu) -> %p\n",		\
+			  _str ? strlen(_str) : 0, _res);		\
+		_res;							\
+	})
+#define GDKstrndup(s, n)					\
+	({							\
+		const char *_str = (s);				\
+		size_t _n = (n);				\
+		void *_res = GDKstrndup(_str, _n);		\
+		TRC_DEBUG(ALLOC, "GDKstrndup(len=%zu) -> %p\n", \
+			  _n,	_res);				\
+		_res;						\
+	})
+#define GDKmmap(p, m, l)						\
+	({								\
+		const char *_path = (p);				\
+		int _mode = (m);					\
+		size_t _len = (l);					\
+		void *_res = GDKmmap(_path, _mode, _len);		\
+		TRC_DEBUG(ALLOC, "GDKmmap(%s,0x%x,%zu) -> %p\n",	\
+			  _path ? _path : "NULL",			\
+			  (unsigned) _mode, _len,			\
+			  _res);					\
+		_res;							\
+	 })
+#define GDKmunmap(p, l)						\
+	({	void *_ptr = (p);				\
+		size_t _len = (l);				\
+		gdk_return _res = GDKmunmap(_ptr, _len);	\
+		TRC_DEBUG(ALLOC,				\
+			  "GDKmunmap(%p,%zu) -> %u\n",		\
+			  _ptr, _len, _res);			\
+		_res;						\
+	})
+#define malloc(s)					\
+	({						\
+		size_t _size = (s);			\
+		void *_res = malloc(_size);		\
+		TRC_DEBUG(ALLOC, "malloc(%zu) -> %p\n", \
+			  _size, _res);			\
+		_res;					\
+	})
+#define calloc(n, s)						\
+	({							\
+		size_t _nmemb = (n);				\
+		size_t _size = (s);				\
+		void *_res = calloc(_nmemb,_size);		\
+		TRC_DEBUG(ALLOC, "calloc(%zu,%zu) -> %p\n",	\
+			  _nmemb, _size, _res);			\
+		_res;						\
+	})
+#define realloc(p, s)						\
+	({							\
+		void *_ptr = (p);				\
+		size_t _size = (s);				\
+		char _buf[12];					\
+		snprintf(_buf, sizeof(_buf), "%p", _ptr);	\
+		void *_res = realloc(_ptr, _size);		\
+		TRC_DEBUG(ALLOC, "realloc(%s,%zu) -> %p\n",	\
+			  _buf, _size, _res);			\
+		_res;						\
+	 })
+#define free(p)						\
+	({						\
+		void *_ptr = (p);			\
+		TRC_DEBUG(ALLOC, "free(%p)\n", _ptr);	\
+		free(_ptr);				\
+	})
+#else
+static inline void *
+GDKmalloc_debug(size_t size)
+{
+	void *res = GDKmalloc(size);
+	TRC_DEBUG(ALLOC, "GDKmalloc(%zu) -> %p\n", size, res);
+	return res;
+}
+#define GDKmalloc(s)	GDKmalloc_debug((s))
+static inline void *
+GDKzalloc_debug(size_t size)
+{
+	void *res = GDKzalloc(size);
+	TRC_DEBUG(ALLOC, "GDKzalloc(%zu) -> %p\n", size, res);
+	return res;
+}
+#define GDKzalloc(s)	GDKzalloc_debug((s))
+static inline void *
+GDKrealloc_debug(void *ptr, size_t size)
+{
+	void *res = GDKrealloc(ptr, size);
+	TRC_DEBUG(ALLOC, "GDKrealloc(%p,%zu) -> %p\n", ptr, size, res);
+	return res;
+}
+#define GDKrealloc(p, s)	GDKrealloc_debug((p), (s))
+static inline void
+GDKfree_debug(void *ptr)
+{
+	TRC_DEBUG(ALLOC, "GDKfree(%p)\n", ptr);
+	GDKfree(ptr);
+}
+#define GDKfree(p)	GDKfree_debug((p))
+static inline char *
+GDKstrdup_debug(const char *str)
+{
+	void *res = GDKstrdup(str);
+	TRC_DEBUG(ALLOC, "GDKstrdup(len=%zu) -> %p\n",
+		  str ? strlen(str) : 0, res);
+	return res;
+}
+#define GDKstrdup(s)	GDKstrdup_debug((s))
+static inline char *
+GDKstrndup_debug(const char *str, size_t n)
+{
+	void *res = GDKstrndup(str, n);
+	TRC_DEBUG(ALLOC, "GDKstrndup(len=%zu) -> %p\n", n, res);
+	return res;
+}
+#define GDKstrndup(s, n)	GDKstrndup_debug((s), (n))
+static inline void *
+GDKmmap_debug(const char *path, int mode, size_t len)
+{
+	void *res = GDKmmap(path, mode, len);
+	TRC_DEBUG(ALLOC, "GDKmmap(%s,0x%x,%zu) -> %p\n",
+		  path ? path : "NULL", (unsigned) mode, len, res);
+	return res;
+}
+#define GDKmmap(p, m, l)	GDKmmap_debug((p), (m), (l))
+static inline gdk_return
+GDKmunmap_debug(void *ptr, size_t len)
+{
+	gdk_return res = GDKmunmap(ptr, len);
+	TRC_DEBUG(ALLOC, "GDKmunmap(%p,%zu) -> %d\n",
+			   	  ptr, len, (int) res);
+	return res;
+}
+#define GDKmunmap(p, l)		GDKmunmap_debug((p), (l))
+static inline void *
+malloc_debug(size_t size)
+{
+	void *res = malloc(size);
+	TRC_DEBUG(ALLOC, "malloc(%zu) -> %p\n", size, res);
+	return res;
+}
+#define malloc(s)	malloc_debug((s))
+static inline void *
+calloc_debug(size_t nmemb, size_t size)
+{
+	void *res = calloc(nmemb, size);
+	TRC_DEBUG(ALLOC, "calloc(%zu,%zu) -> %p\n", nmemb, size, res);
+	return res;
+}
+#define calloc(n, s)	calloc_debug((n), (s))
+static inline void *
+realloc_debug(void *ptr, size_t size)
+{
+	void *res = realloc(ptr, size);
+	TRC_DEBUG(ALLOC, "realloc(%p,%zu) -> %p \n", ptr, size, res);
+	return res;
+}
+#define realloc(p, s)	realloc_debug((p), (s))
+static inline void
+free_debug(void *ptr)
+{
+	TRC_DEBUG(ALLOC, "free(%p)\n", ptr);
+	free(ptr);
+}
+#define free(p)	free_debug((p))
+#endif
+#endif
+
+#endif /* _GDK_UTILS_H_ */

monetdb/msvc/mal.h

@@ -0,0 +1,240 @@
+/*
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0.  If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright 1997 - July 2008 CWI, August 2008 - 2022 MonetDB B.V.
+ */
+
+/*
+ * (c) Martin Kersten
+ *  MonetDB Basic Definitions
+ * Definitions that need to included in every file of the Monet system, as well as in user defined module implementations.
+ */
+#ifndef _MAL_H
+#define _MAL_H
+
+#include "gdk.h"
+
+#ifdef WIN32
+#ifndef LIBMONETDB5
+#define mal_export extern __declspec(dllimport)
+#else
+#define mal_export extern __declspec(dllexport)
+#endif
+#else
+#define mal_export extern
+#endif
+
+#ifdef HAVE_SYS_TIMES_H
+# include <sys/times.h>
+#endif
+
+/*
+ * MonetDB Calling Options
+ * The number of invocation arguments is kept to a minimum.
+ * See `man mserver5` or tools/mserver/mserver5.1
+ * for additional system variable settings.
+ */
+#define MAXSCRIPT 64
+mal_export lng MALdebug;
+
+/*
+ * MonetDB assumes it can use most of the machines memory,
+ * leaving a small portion for other programs.
+ */
+#define GB (((lng)1024)*1024*1024)
+#define MEMORY_THRESHOLD  (0.2 * GDK_mem_maxsize > 8 * GB?  GDK_mem_maxsize - 8 * GB: 0.8 * GDK_mem_maxsize)
+
+mal_export char     monet_cwd[FILENAME_MAX];
+mal_export char 	monet_characteristics[4096];
+mal_export stream	*maleventstream;
+
+/*
+   See gdk/gdk.h for the definition of all debug masks.
+   See `man mserver5` or tools/mserver/mserver5.1
+   for a documentation of all debug options.
+*/
+#define GRPthreads (THRDMASK | PARMASK)
+#define GRPmemory (ALLOCMASK )
+#define GRPproperties (CHECKMASK | PROPMASK | BATMASK )
+#define GRPio (IOMASK | PERFMASK )
+#define GRPheaps (HEAPMASK)
+#define GRPtransactions (TMMASK | DELTAMASK | TEMMASK)
+#define GRPmodules (LOADMASK)
+#define GRPalgorithms (ALGOMASK)
+#define GRPperformance (DEADBEEFMASK)
+#define GRPforcemito (FORCEMITOMASK | NOSYNCMASK)
+
+mal_export MT_Lock  mal_contextLock;
+mal_export MT_Lock  mal_remoteLock;
+mal_export MT_Lock  mal_profileLock ;
+mal_export MT_Lock  mal_copyLock ;
+mal_export MT_Lock  mal_delayLock ;
+
+mal_export int mal_init(char *modules[], int embedded);
+mal_export _Noreturn void mal_exit(int status);
+mal_export void mal_reset(void);
+mal_export const char *mal_version(void);
+
+/* This should be here, but cannot, as "Client" isn't known, yet ... |-(
+ * For now, we move the prototype declaration to src/mal/mal_client.c,
+ * the only place where it is currently used. Maybe, we should concider
+ * also moving the implementation there...
+ */
+
+/* Listing modes are globally known */
+#define LIST_INPUT      1       /* echo original input */
+#define LIST_MAL_NAME   2       /* show variable name */
+#define LIST_MAL_TYPE   4       /* show type resolutoin */
+#define LIST_MAL_VALUE  8       /* list bat tuple count */
+#define LIST_MAL_PROPS 16       /* show variable properties */
+#define LIST_MAL_MAPI  32       /* output Mapi compatible output */
+#define LIST_MAL_REMOTE  64       /* output MAL for remote execution */
+#define LIST_MAL_FLOW   128       /* output MAL dataflow dependencies */
+#define LIST_MAL_ALGO	256		  /* output algorithm used */
+#define LIST_MAL_CALL  (LIST_MAL_NAME | LIST_MAL_VALUE )
+#define LIST_MAL_DEBUG (LIST_MAL_NAME | LIST_MAL_VALUE | LIST_MAL_TYPE | LIST_MAL_PROPS | LIST_MAL_FLOW)
+#define LIST_MAL_ALL   (LIST_MAL_NAME | LIST_MAL_VALUE | LIST_MAL_TYPE | LIST_MAL_MAPI)
+
+/* type check status is kept around to improve type checking efficiency */
+#define TYPE_ERROR      -1
+#define TYPE_UNKNOWN     0
+#define TYPE_RESOLVED    2
+#define GARBAGECONTROL   3
+
+#define VARARGS 1				/* deal with variable arguments */
+#define VARRETS 2
+
+typedef int malType;
+typedef str (*MALfcn) ();
+
+typedef struct SYMDEF {
+	struct SYMDEF *peer;		/* where to look next */
+	struct SYMDEF *skip;		/* skip to next different symbol */
+	const char *name;
+	int kind;					/* what kind of symbol */
+	struct MALBLK *def;			/* the details of the MAL fcn */
+} *Symbol, SymRecord;
+
+typedef struct VARRECORD {
+	char name[IDLENGTH];			/* use the space for the full name */
+	char kind;				/* Could be either _, X or C to stamp the variable type */
+	malType type;				/* internal type signature */
+	bool constant:1,
+            typevar:1,
+            fixedtype:1,
+            //FREE SPOT NOW:1,
+            cleanup:1,
+            initialized:1,
+            used:1,
+            disabled:1;
+	short depth;				/* scope block depth, set to -1 if not used */
+	ValRecord value;
+	int declared;				/* pc index when it was first assigned */
+	int updated;				/* pc index when it was first updated */
+	int eolife;					/* pc index when it should be garbage collected */
+	int stc;				    /* pc index for rendering schema.table.column  */
+	BUN rowcnt;					/* estimated row count*/
+} *VarPtr, VarRecord;
+
+/* For performance analysis we keep track of the number of calls and
+ * the total time spent while executing the instruction. (See
+ * mal_profiler.c)
+ */
+
+typedef struct {
+	bte token;					/* instruction type */
+	bit barrier;				/* flow of control modifier takes:
+								   BARRIER, LEAVE, REDO, EXIT, CATCH, RAISE */
+	bit typechk;				/* type check status */
+	bte gc;						/* garbage control flags */
+	bte polymorphic;			/* complex type analysis */
+	bit varargs;				/* variable number of arguments */
+	int jump;					/* controlflow program counter */
+	int pc;						/* location in MAL plan for profiler*/
+	MALfcn fcn;					/* resolved function address */
+	struct MALBLK *blk;			/* resolved MAL function address */
+	/* inline statistics */
+	lng clock;					/* when the last call was started */
+	lng ticks;					/* total micro seconds spent in last call */
+	int calls;					/* number of calls made to this instruction */
+	lng totticks;				/* total time spent on this instruction. */
+	lng wbytes;					/* number of bytes produced in last instruction */
+	/* the core admin */
+	const char *modname;		/* module context, reference into namespace */
+	const char *fcnname;		/* function name, reference into namespace */
+	int argc, retc, maxarg;		/* total and result argument count */
+	int argv[FLEXIBLE_ARRAY_MEMBER]; /* at least a few entries */
+} *InstrPtr, InstrRecord;
+
+typedef struct MALBLK {
+	char binding[IDLENGTH];	/* related C-function */
+	str help;				/* supportive commentary */
+	str statichelp;			/* static help string should not be freed */
+	oid tag;				/* unique block tag */
+	struct MALBLK *alternative;
+	int vtop;				/* next free slot */
+	int vsize;				/* size of variable arena */
+	int vid;	 			/* generate local variable counter */
+	VarRecord *var;			/* Variable table */
+	int stop;				/* next free slot */
+	int ssize;				/* byte size of arena */
+	InstrPtr *stmt;				/* Instruction location */
+
+	bool inlineProp:1,		/* inline property */
+	     unsafeProp:1;		/* unsafe property */
+
+	str errors;				/* left over errors */
+	struct MALBLK *history;	/* of optimizer actions */
+	short keephistory;		/* do we need the history at all */
+	int maxarg;				/* keep track on the maximal arguments used */
+	ptr replica;			/* for the replicator tests */
+
+	/* During the run we keep track on the maximum number of concurrent threads and memory claim */
+	int		workers;
+	lng		memory;
+	lng starttime;			/* track when the query started, for resource management */
+	lng runtime;			/* average execution time of block in ticks */
+	int calls;				/* number of calls */
+	lng optimize;			/* total optimizer time */
+} *MalBlkPtr, MalBlkRecord;
+
+#define STACKINCR   128
+#define MAXGLOBALS  (4 * STACKINCR)
+
+typedef int (*DFhook) (void *, void *, void *, void *);
+
+typedef struct MALSTK {
+	int stksize;
+	int stktop;
+	int stkbot;			/* the first variable to be initialized */
+	int stkdepth;		/* to protect against runtime stack overflow */
+	int calldepth;		/* to protect against runtime stack overflow */
+	short keepAlive;	/* do not garbage collect when set */
+	/*
+	 * Parallel processing is mostly driven by dataflow, but within this context
+	 * there may be different schemes to take instructions into execution.
+	 * The admission scheme (and wrapup) are the necessary scheduler hooks.
+	 */
+	DFhook admit;
+	DFhook wrapup;
+
+/*
+ * It is handy to administer the timing in the stack frame
+ * for use in profiling instructions.
+ */
+	struct timeval clock;	/* time this stack was created */
+	char cmd;				/* debugger and runtime communication */
+	char status;			/* srunning 'R' suspended 'S', quiting 'Q' */
+	int pcup;				/* saved pc upon a recursive all */
+	oid tag;				/* unique invocation call tag */
+	int	workers;			/* Actual number of concurrent workers */
+	lng	memory;				/* Actual memory claims for highwater mark */
+
+	struct MALSTK *up;		/* stack trace list */
+	struct MALBLK *blk;		/* associated definition */
+	ValRecord stk[FLEXIBLE_ARRAY_MEMBER];
+} MalStack, *MalStkPtr;
+
+#endif /*  _MAL_H*/

monetdb/msvc/mal_client.h

@@ -0,0 +1,195 @@
+/*
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0.  If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright 1997 - July 2008 CWI, August 2008 - 2022 MonetDB B.V.
+ */
+
+#ifndef _MAL_CLIENT_H_
+#define _MAL_CLIENT_H_
+
+#include "mal.h"
+
+#include "mal_module.h"
+
+#define SCENARIO_PROPERTIES 8
+
+enum clientmode {
+	FREECLIENT,
+	FINISHCLIENT,
+	RUNCLIENT,
+	BLOCKCLIENT
+};
+
+/*
+ * The prompt structure is designed to simplify recognition of the
+ * language framework for interaction. For access through an API we
+ * assume the prompt is an ASCII string surrounded by a \001 character. This
+ * simplifies recognition.  The information between the prompt brackets
+ * can be used to pass the mode to the front-end. Moreover, the prompt
+ * can be dropped if a single stream of information is expected from the
+ * server (see mal_profiler.c).
+ *
+ * The user can request server-side compilation as part of the
+ * initialization string. See the documentation on Scenarios.
+ */
+typedef struct CLIENT_INPUT {
+	bstream             *fdin;
+	size_t              yycur;
+	int                 listing;
+	char                *prompt;
+	struct CLIENT_INPUT *next;
+} ClientInput;
+
+typedef struct CLIENT {
+	int idx;        /* entry in mal_clients */
+	oid user;       /* user id in the auth administration */
+	str username;	/* for event processor */
+	/*
+	 * The actions for a client is separated into several stages:
+	 * parsing, strategic optimization, tactical optimization, and
+	 * execution.  The routines to handle them are obtained once the
+	 * scenario is chosen.  Each stage carries a state descriptor, but
+	 * they share the IO state description. A backup structure is
+	 * provided to temporarily switch to another scenario.
+	 */
+	str     scenario;  /* scenario management references */
+	str     oldscenario;
+	void    *state[SCENARIO_PROPERTIES], *oldstate[SCENARIO_PROPERTIES];
+	MALfcn  phase[SCENARIO_PROPERTIES], oldphase[SCENARIO_PROPERTIES];
+	char    itrace;    /* trace execution using interactive mdb */
+						/* if set to 'S' it will put the process to sleep */
+	bit		sqlprofiler;		/* control off-line sql performance monitoring */
+	/*
+	 * Each session comes with resource limitations and predefined settings.
+	 */
+	char	optimizer[IDLENGTH];/* The optimizer pipe preferred for this session */
+	int 	workerlimit;		/* maximum number of workthreads processing a query */
+	int		memorylimit;		/* Memory claim highwater mark, 0 = no limit */
+	lng 	querytimeout;		/* query abort after x usec, 0 = no limit*/
+	lng	    sessiontimeout;		/* session abort after x usec, 0 = no limit */
+
+	time_t  login;  	/* Time when this session started */
+	lng 	session;	/* usec since start of server */
+	time_t  idle;		/* Time when the session became idle */
+
+	/*
+	 * For program debugging and performance trace we keep the actual resource claims.
+	 */
+	time_t  lastcmd;	/* set when query is received */
+
+	/* The user can request a TRACE SQL statement, calling for collecting the events locally */
+	BAT *profticks;
+	BAT *profstmt;
+	BAT *profevents;
+
+	ATOMIC_TYPE	lastprint;	/* when we last printed the query, to be depricated */
+	/*
+	 * Communication channels for the interconnect are stored here.
+	 * It is perfectly legal to have a client without input stream.
+	 * It will simply terminate after consuming the input buffer.
+	 */
+	const char *srcFile;  /* NULL for stdin, or file name */
+	bstream  *fdin;
+	size_t    yycur;    /* the scanners current position */
+	/*
+	 * Keeping track of instructions executed is a valuable tool for
+	 * script processing and debugging.  It can be changed at runtime
+	 * for individual clients using the operation clients.listing(mask).
+	 * A listing bit controls the level of detail to be generated during
+	 * program execution tracing.  The lowest level (1) simply dumps the
+	 * input, (2) also demonstrates the MAL internal structure, (4) adds
+	 * the type information.
+	 */
+	int  listing;
+	str prompt;         /* acknowledge prompt */
+	size_t promptlength;
+	ClientInput *bak;   /* used for recursive script and string execution */
+
+	stream   *fdout;    /* streams from and to user. */
+	/*
+	 * In interactive mode, reading one line at a time, we should be
+	 * aware of parsing compound structures, such as functions and
+	 * barrier blocks. The level of nesting is maintained in blkmode,
+	 * which is reset to zero upon encountering an end instruction, or
+	 * the closing bracket has been detected. Once the complete
+	 * structure has been parsed the program can be checked and
+	 * executed.  Nesting is indicated using a '+' before the prompt.
+	 */
+	int blkmode;        /* control block parsing */
+	/*
+	 * The MAL debugger uses the client record to keep track of any
+	 * pervasive debugger command. For detailed information on the
+	 * debugger features.
+	 */
+	int debug;
+	enum clientmode mode;  /* FREECLIENT..BLOCKED */
+	/*
+	 * Client records are organized into a two-level dependency tree,
+	 * where children may be created to deal with parallel processing
+	 * activities. Each client runs in its own process thread. Its
+	 * identity is retained here for access by others (=father).
+	 */
+	MT_Sema 	s;	    /* sema to (de)activate thread */
+	Thread      	mythread;
+	str     	errbuf;     /* location of GDK exceptions */
+	struct CLIENT   *father;
+	/*
+	 * Each client has a private entry point into the namespace and
+	 * object space (the global variables).  Moreover, the parser needs
+	 * some administration variables to keep track of critical elements.
+	 */
+	Module      usermodule;     /* private user scope */
+	Module		curmodule;		/* where to deliver the symbol, used by parser , only freed globally */
+	Symbol      curprg;     /* container for the malparser */
+	Symbol      backup;     /* saving the parser context for functions,commands/patterns */
+	MalStkPtr   glb;        /* global variable stack */
+
+	/*
+	 * Here are pointers to scenario backends contexts.  For the time
+	 * being just SQL.  We need a pointer for each of them, since they
+	 * have to be able to interoperate with each other, e.g.  both
+	 * contexts at the same time are in use.
+	 */
+	void *sqlcontext;
+
+	/*
+	 * The workload for replication/replay is saved initially as a MAL block.
+	 * It is split into the capturing part (wlc) and the replay part (wlr).
+	 * This allows a single server to act as both a master and a replica.
+	 */
+	int wlc_kind;	// used by master to characterise the compound transaction
+	MalBlkPtr wlc;
+
+	/*
+	 *	Errors during copy into are collected in a user specific column set
+	 */
+	BAT *error_row;
+	BAT *error_fld;
+	BAT *error_msg;
+	BAT *error_input;
+
+	size_t blocksize;
+	protocol_version protocol;
+	bool filetrans;			/* whether the client can read files for us */
+	char *handshake_options;
+	char *query;			/* string, identify whatever we're working on */
+} *Client, ClientRec;
+
+mal_export int MAL_MAXCLIENTS;
+mal_export ClientRec *mal_clients;
+
+mal_export Client  MCgetClient(int id);
+mal_export Client  MCinitClient(oid user, bstream *fin, stream *fout);
+mal_export Client  MCforkClient(Client father);
+mal_export void	   MCstopClients(Client c);
+mal_export int	   MCactiveClients(void);
+mal_export size_t  MCmemoryClaim(void);
+mal_export void    MCcloseClient(Client c);
+mal_export str     MCsuspendClient(int id);
+mal_export str     MCawakeClient(int id);
+mal_export int     MCpushClientInput(Client c, bstream *new_input, int listing, char *prompt);
+mal_export int	   MCvalid(Client c);
+
+#endif /* _MAL_CLIENT_H_ */

monetdb/msvc/mal_errors.h

@@ -0,0 +1,139 @@
+/*
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0.  If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright 1997 - July 2008 CWI, August 2008 - 2022 MonetDB B.V.
+ */
+
+/*
+ * The error strings are geared at answering the question "what happened".
+ * Optional information about "why it happened" is added
+ * as local strings in the code base with possibly runtime information.
+ * Information on "how to avoid it" is sporadically added using expected values.
+ *
+ * General considerations on error classes are summarized below:
+ * MAL_MALLOC_FAIL
+ * An operation allocates space for an object failed.
+ * Either the pre-requisites are not satisfied, or the system
+ * runs low on resources and can not accomodate the object.
+ * For failures to create BATs it sometimes indicates that an
+ * intermediate BAT size is too large.
+ * OPERATION_FAILED
+ * Mostly the module and function name are indicative enough.
+ * If possible more is said about the error context,
+ * informative references to arguments or variables,
+ * provided it is produced by the underlying implementation.
+ * GDK_EXCEPTION
+ * In general these are generated deep inside the kernel.
+ * They are captured by the MAL interpreter.
+ * SEMANTIC_*
+ * The patterns can be used to defer semantic type errors
+ * to runtime.
+ *
+ * Getting all exception strings in one place improves consistency
+ * and maintenance.
+ *
+ * At a later stage we could introduce internationalization support, i.e.
+ * use a translation table where needed.
+ */
+#ifndef MAL_ERRORS
+#define MAL_ERRORS
+
+#define SQLSTATE(sqlstate)	#sqlstate "!"
+
+#define MANUAL_HELP "See documentation for details"
+
+#define PROGRAM_GENERAL "Program contains errors."
+#define PROGRAM_NYI  "Not yet implemented"
+
+#define SYNTAX_GENERAL	"Syntax error detected."
+#define SYNTAX_SIGNATURE "Function signature missing."
+
+#define SEMANTIC_GENERAL "Semantic errors detected"
+#define SEMANTIC_PROGRAM_ERRORS "Program contains semantic errors."
+#define SEMANTIC_SIGNATURE_MISSING "Function signature missing."
+#define SEMANTIC_OPERATION_MISSING "Operation not found."
+#define SEMANTIC_TYPE_ERROR "Explicit type required"
+#define SEMANTIC_TYPE_MISMATCH "Type mismatch"
+
+#define INTERNAL_BAT_ACCESS "Internal error, can not access BAT."
+#define INTERNAL_BAT_HEAD "BAT has wrong head type"
+#define INTERNAL_OBJ_CREATE "Can not create object"
+#define INTERNAL_AUTHORIZATION "authorization BATs not empty"
+
+#define MAL_MALLOC_FAIL	"Could not allocate space"
+#define MAL_STACK_FAIL	"Running out of stack space."
+#define MAL_CALLDEPTH_FAIL	"Recursive call limit reached."
+
+#define INVCRED_ACCESS_DENIED "access denied for user"
+#define INVCRED_INVALID_USER "invalid credentials for user"
+#define INVCRED_REMOVE_USER "Can not remove user"
+#define INVCRED_WRONG_ID "Undefined client id"
+
+#define RUNTIME_IO_EOF "Attempt to read beyond end-of-file"
+#define RUNTIME_FILE_NOT_FOUND "File not found"
+#define RUNTIME_UNLINK "File could not be unlinked"
+#define RUNTIME_DIR_ERROR "Unable to open directory"
+#define RUNTIME_CREATE_ERROR "Unable to create file/directory"
+#define RUNTIME_STREAM_FAILED "Could not create stream"
+#define RUNTIME_STREAM_WRITE "Could not write to stream"
+#define RUNTIME_STREAM_INPUT "Could not read from stream"
+
+#define RUNTIME_LOAD_ERROR "Loading error"
+#define RUNTIME_OBJECT_MISSING "Object not found"
+#define RUNTIME_SIGNATURE_MISSING "The <module>.<function> not found"
+#define RUNTIME_OBJECT_UNDEFINED "Object not found"
+#define RUNTIME_UNKNOWN_INSTRUCTION "Instruction type not supported"
+#define RUNTIME_QRY_TIMEOUT "Query aborted due to timeout"
+#define RUNTIME_SESSION_TIMEOUT "Query aborted due to session timeout"
+#define OPERATION_FAILED "operation failed"
+#define _OPERATION_FAILED "Operation failed"
+
+#define BOX_CLOSED "Box is not open"
+
+#define SABAOTH_NOT_INITIALIZED "Sabaoth not initialized"
+#define SABAOTH_USE_RESTRICTION "Sabaoth was not initialized as active database"
+
+#define SCENARIO_NOT_FOUND "Scenario not initialized"
+
+#define MACRO_SYNTAX_ERROR "RETURN statement is not the last one"
+#define MACRO_DUPLICATE "Duplicate macro expansion"
+#define MACRO_TOO_DEEP "Too many macro expansions"
+
+#define OPTIMIZER_CYCLE "Too many optimization cycles"
+
+#define ILLARG_NOTNIL " NIL not allowed"
+#define ILLARG_CONSTANTS "Constant argument required"
+
+#define ILLEGAL_ARGUMENT "Illegal argument"
+#define IDENTIFIER_EXPECTED "Identifier expected"
+#define POSITIVE_EXPECTED "Argument must be positive"
+#define ARGUMENT_TOO_LARGE "Argument too large"
+#define TOO_MANY_BITS "Too many bits"
+#define DUPLICATE_DEFINITION "Duplicate definition"
+#define RANGE_ERROR "Range error"
+
+#define SERVER_STOPPED "Server stopped"
+
+#define XML_PARSE_ERROR "Document parse error"
+#define XML_COMMENT_ERROR "Comment may not contain '--'"
+#define XML_PI_ERROR "No processing instruction target specified"
+#define XML_VERSION_ERROR "Illegal XML version"
+#define XML_STANDALONE_ERROR "Illegal XML standalone value"
+#define XML_NOT_WELL_FORMED "Resulting document not well-formed"
+#define XML_ATTRIBUTE_ERROR "No attribute name specified"
+#define XML_ATTRIBUTE_INVALID "Invalid attribute name"
+#define XML_NO_ELEMENT "No element name specified"
+#define XML_NO_NAMESPACE "Namespace support not implemented"
+#define XML_ILLEGAL_NAMESPACE "Illegal namespace"
+#define XML_ILLEGAL_ATTRIBUTE "Illegal attribute"
+#define XML_ILLEGAL_CONTENT "Illegal content"
+
+/* GDK_EXCEPTION should be by itself, i.e. not concatenated with some
+ * other string; to enforce this we have the parentheses */
+#define GDK_EXCEPTION ("GDK reported error.")
+#define MAL_DEPRECATED "Deprecated MAL operation."
+
+#define TYPE_NOT_SUPPORTED "Type is not supported"
+#endif /* MAL_ERRORS */

monetdb/msvc/mal_exception.h

@@ -0,0 +1,65 @@
+/*
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0.  If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright 1997 - July 2008 CWI, August 2008 - 2022 MonetDB B.V.
+ */
+
+#ifndef _MAL_EXCEPTION_H
+#define _MAL_EXCEPTION_H
+#include "mal_instruction.h"
+
+/* These are the exceptions known, adding new ones here requires to also
+ * add the "full" name to the exceptionNames array in mal_exception.c */
+enum malexception {
+	MAL=0,
+	ILLARG,
+	OUTOFBNDS,
+	IO,
+	INVCRED,
+	OPTIMIZER,
+	STKOF,
+	SYNTAX,
+	TYPE,
+	LOADER,
+	PARSE,
+	ARITH,
+	PERMD,
+	SQL,
+	REMOTE
+};
+
+#define MAL_SUCCEED ((str) 0) /* no error */
+
+#define throw \
+	return createException
+#define rethrow(FCN, TMP, PRV) \
+	do { if ((TMP = (PRV)) != MAL_SUCCEED) return(TMP); } while(0)
+
+#if !__has_attribute(__returns_nonnull__)
+#define __returns_nonnull__
+#endif
+
+mal_export str createException(enum malexception, const char *,
+	_In_z_ _Printf_format_string_ const char *, ...)
+	__attribute__((__format__(__printf__, 3, 4)))
+	__attribute__((__returns_nonnull__));
+/*FIXmal_export str createMalException(MalBlkPtr mb, int pc, enum malexception type, const char *prev, const char *format, ...);*/
+mal_export str createMalException(MalBlkPtr , int , enum malexception ,
+	_In_z_ _Printf_format_string_ const char *, ...)
+	__attribute__((__format__(__printf__, 4, 5)))
+	__attribute__((__returns_nonnull__));
+mal_export char *concatErrors(char *err1, const char *err2)
+	__attribute__((__nonnull__(1))) __attribute__((__nonnull__(2)))
+	__attribute__((__returns_nonnull__));
+mal_export bool isExceptionVariable(const char *nme);
+
+mal_export enum malexception	getExceptionType(const char *);
+mal_export str	getExceptionPlace(const char *);
+mal_export str	getExceptionMessageAndState(const char *);
+mal_export str	getExceptionMessage(const char *);
+mal_export void freeException(str);
+
+#include "mal_errors.h"
+#endif /*  _MAL_EXCEPTION_H*/

monetdb/msvc/mal_instruction.h

@@ -0,0 +1,201 @@
+/*
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0.  If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright 1997 - July 2008 CWI, August 2008 - 2022 MonetDB B.V.
+ */
+
+#ifndef _MAL_INSTR_H
+#define _MAL_INSTR_H
+
+#include "mal_type.h"
+#include "mal_stack.h"
+#include "mal_namespace.h"
+
+#define isaSignature(P)  ((P)->token >=COMMANDsymbol || (P)->token == PATTERNsymbol)
+
+#ifdef HAVE_SYS_TIMES_H
+# include <sys/times.h>
+#endif
+
+#define DEBUG_MAL_INSTR
+#define MAXARG 8				/* was 4 BEWARE the code depends on this knowledge, where? */
+#define STMT_INCREMENT 4
+#define MAL_VAR_WINDOW  16 // was 32
+#define MAXLISTING (64*1024)
+
+/* Allocation of space assumes a rather exotic number of
+ * arguments. Access to module and function name are cast in macros to
+ * prepare for separate name space management. */
+#define getModuleId(P)		((P)->modname)
+#define setModuleId(P,S)	(P)->modname= S
+#define setModuleScope(P,S)	do {(P)->modname= (S)==NULL?NULL: (S)->name;} while (0)
+
+#define getFunctionId(P)	(P)->fcnname
+#define setFunctionId(P,S)	(P)->fcnname= S
+#define garbageControl(P)	((P)->gc & GARBAGECONTROL)
+
+#define getInstrPtr(M,I)	(M)->stmt[I]
+#define putInstrPtr(M,I,P)	(M)->stmt[I] = P
+#define getSignature(S)		getInstrPtr((S)->def,0)
+#define isMain(M)			((getInstrPtr(M,0))->fcnname== putName("main",4))
+#define getFcnName(M)		getFunctionId(getInstrPtr(M,0))
+#define getArgCount(M)		getInstrPtr(M,0)->argc
+#define getModName(M)		getModuleId(getInstrPtr(M,0))
+#define getPrgSize(M)		(M)->stop
+
+#define getVar(M,I)		(&(M)->var[I])
+#define getVarType(M,I)		((M)->var[I].type)
+mal_export char* getVarName(MalBlkPtr mb, int idx);
+
+#define getVarKind(M,I)   	((M)->var[I].kind)
+#define getVarGDKType(M,I)	getGDKType((M)->var[I].type)
+#define setVarType(M,I,V)   	((M)->var[I].type = V)
+#define setVarKind(M,I,V)   	((M)->var[I].kind = V)	/* either _, X, or C */
+
+#define clrVarFixed(M,I)	((M)->var[I].fixedtype = 0)
+#define setVarFixed(M,I)	((M)->var[I].fixedtype =1)
+#define isVarFixed(M,I)		((M)->var[I].fixedtype)
+
+#define clrVarCleanup(M,I)	((M)->var[I].cleanup = 0)
+#define setVarCleanup(M,I)	((M)->var[I].cleanup = 1)
+#define isVarCleanup(M,I)	((M)->var[I].cleanup )
+
+#define isTmpVar(M,I)		(getVarKind(M,I) == REFMARKER)
+
+#define clrVarUsed(M,I)		((M)->var[I].used = 0)
+#define setVarUsed(M,I)		((M)->var[I].used = 1)
+#define isVarUsed(M,I)		((M)->var[I].used)
+
+#define clrVarDisabled(M,I)	((M)->var[I].disabled= 0 )
+#define setVarDisabled(M,I)	((M)->var[I].disabled = 1)
+#define isVarDisabled(M,I)	((M)->var[I].disabled)
+
+#define clrVarInit(M,I)		((M)->var[I].initialized = 0)
+#define setVarInit(M,I)		((M)->var[I].initialized = 1)
+#define isVarInit(M,I)		((M)->var[I].initialized)
+
+#define clrVarTypedef(M,I)	((M)->var[I].typevar = 0)
+#define setVarTypedef(M,I)	((M)->var[I].typevar = 1)
+#define isVarTypedef(M,I)	((M)->var[I].typevar)
+
+#define clrVarConstant(M,I)	((M)->var[I].constant = 0)
+#define setVarConstant(M,I)	((M)->var[I].constant = 1)
+#define isVarConstant(M,I)	((M)->var[I].constant)
+
+#define setVarDeclared(M,I,X)	((M)->var[I].declared = X )
+#define getVarDeclared(M,I)	((M)->var[I].declared)
+
+#define setVarUpdated(M,I,X)	((M)->var[I].updated = X )
+#define getVarUpdated(M,I)	((M)->var[I].updated)
+
+#define setVarEolife(M,I,X)	((M)->var[I].eolife = X )
+#define getVarEolife(M,I)	((M)->var[I].eolife)
+
+#define setVarScope(M,I,S)	((M)->var[I].depth = S)
+#define getVarScope(M,I)	((M)->var[I].depth)
+
+#define clrVarCList(M,I)	((M)->var[I].kind = REFMARKER)
+#define setVarCList(M,I)	((M)->var[I].kind = REFMARKERC)
+#define isVarCList(M,I)		((M)->var[I].kind  == REFMARKERC)
+
+#define getVarConstant(M,I)	((M)->var[I].value)
+#define getVarValue(M,I)	VALget(&(M)->var[I].value)
+
+#define setRowCnt(M,I,C)	(M)->var[I].rowcnt = C
+#define getRowCnt(M,I)		((M)->var[I].rowcnt)
+
+#define getVarSTC(M,I)			((M)->var[I].stc)
+
+#define getDestVar(P)		(P)->argv[0]
+#define setDestVar(P,X)		(P)->argv[0]  =X
+#define setDestType(M,P,V)	setVarType((M),getDestVar(P),V)
+#define getDestType(M,P)	destinationType((M),(P))
+#define getArg(P,I)			(P)->argv[I]
+#define setArg(P,I,R)		(P)->argv[I]= R
+#define getArgName(M,P,I)	getVarName((M),(P)->argv[I])
+#define getArgType(M,P,I)	getVarType((M),(P)->argv[I])
+#define getArgGDKType(M,P,I) getVarGDKType((M),(P)->argv[I])
+#define getGDKType(T) 		( T <= TYPE_str ? T : (T == TYPE_any ? TYPE_void : findGDKtype(T)))
+
+mal_export void addMalException(MalBlkPtr mb, str msg);
+mal_export void mal_instruction_reset(void);
+mal_export InstrPtr newInstruction(MalBlkPtr mb, const char *modnme, const char *fcnnme);
+mal_export InstrPtr newInstructionArgs(MalBlkPtr mb, const char *modnme, const char *fcnnme, int args);
+mal_export InstrPtr copyInstruction(InstrPtr p);
+mal_export InstrPtr copyInstructionArgs(InstrPtr p, int args);
+mal_export void clrInstruction(InstrPtr p);
+mal_export void freeInstruction(InstrPtr p);
+mal_export void clrFunction(InstrPtr p);
+mal_export Symbol newSymbol(const char *nme, int kind);
+mal_export void freeSymbol(Symbol s);
+mal_export void freeSymbolList(Symbol s);
+mal_export void printSignature(stream *fd, Symbol s, int flg);
+
+mal_export MalBlkPtr newMalBlk(int elements);
+mal_export void resetMalBlk(MalBlkPtr mb);
+mal_export void resetMalTypes(MalBlkPtr mb, int stop);
+mal_export int newMalBlkStmt(MalBlkPtr mb, int elements);
+mal_export int resizeMalBlk(MalBlkPtr mb, int elements);
+mal_export void freeMalBlk(MalBlkPtr mb);
+mal_export MalBlkPtr copyMalBlk(MalBlkPtr mb);
+mal_export void addtoMalBlkHistory(MalBlkPtr mb);
+mal_export void removeMalBlkHistory(MalBlkPtr mb);
+mal_export MalBlkPtr getMalBlkHistory(MalBlkPtr mb, int idx);
+mal_export MalBlkPtr getMalBlkOptimized(MalBlkPtr mb, const char *name);
+mal_export void trimMalVariables(MalBlkPtr mb, MalStkPtr stk);
+mal_export void trimMalVariables_(MalBlkPtr mb, MalStkPtr glb);
+mal_export void moveInstruction(MalBlkPtr mb, int pc, int target);
+mal_export void removeInstruction(MalBlkPtr mb, InstrPtr p);
+mal_export void removeInstructionBlock(MalBlkPtr mb, int pc, int cnt);
+mal_export str operatorName(int i);
+
+mal_export int findVariable(MalBlkPtr mb, const char *name);
+mal_export int findVariableLength(MalBlkPtr mb, const char *name, int len);
+mal_export malType getType(MalBlkPtr mb, const char *nme);
+mal_export str getArgDefault(MalBlkPtr mb, InstrPtr p, int idx);
+mal_export int newVariable(MalBlkPtr mb, const char *name, size_t len, malType type);
+mal_export int cloneVariable(MalBlkPtr dst, MalBlkPtr src, int varid);
+mal_export void setVariableType(MalBlkPtr mb, const int idx, malType type);
+mal_export int newTmpVariable(MalBlkPtr mb, malType type);
+mal_export int newTypeVariable(MalBlkPtr mb, malType type);
+mal_export void freeVariable(MalBlkPtr mb, int varid);
+mal_export void clearVariable(MalBlkPtr mb, int varid);
+mal_export int cpyConstant(MalBlkPtr mb, VarPtr vr);
+mal_export int defConstant(MalBlkPtr mb, int type, ValPtr cst);
+mal_export int fndConstant(MalBlkPtr mb, const ValRecord *cst, int depth);
+mal_export str convertConstant(malType type, ValPtr vr);
+
+mal_export void pushInstruction(MalBlkPtr mb, InstrPtr p);
+mal_export InstrPtr pushArgument(MalBlkPtr mb, InstrPtr p, int varid);
+mal_export InstrPtr addArgument(MalBlkPtr mb, InstrPtr p, int varid);
+mal_export InstrPtr setArgument(MalBlkPtr mb, InstrPtr p, int idx, int varid);
+mal_export InstrPtr pushReturn(MalBlkPtr mb, InstrPtr p, int varid);
+mal_export InstrPtr pushArgumentId(MalBlkPtr mb, InstrPtr p, const char *name);
+mal_export void delArgument(InstrPtr p, int varid);
+mal_export void setArgType(MalBlkPtr mb, InstrPtr p, int i, int tpe);
+mal_export void setReturnArgument(InstrPtr p, int varid);
+mal_export malType destinationType(MalBlkPtr mb, InstrPtr p);
+mal_export void setPolymorphic(InstrPtr p, int tpe, int force);
+/* Utility macros to inspect an instruction */
+#define functionStart(X) ((X)->token == FUNCTIONsymbol || \
+              (X)->token == COMMANDsymbol || \
+              (X)->token == FACTORYsymbol )
+#define patternStart(X)  ((X)->token == PATTERNsymbol)
+#define functionExit(X)  ((X)->token == ENDsymbol)
+
+#define blockStart(X)   ((X)->barrier && (((X)->barrier == BARRIERsymbol || \
+             (X)->barrier == CATCHsymbol )))
+#define blockExit(X) ((X)->barrier == EXITsymbol)
+#define blockReturn(X) ((X)->barrier == RETURNsymbol)
+#define blockCntrl(X) ( (X)->barrier== LEAVEsymbol ||  \
+             (X)->barrier== REDOsymbol || (X)->barrier== RETURNsymbol )
+#define isLinearFlow(X)  (!(blockStart(X) || blockExit(X) || \
+				(X)->barrier== LEAVEsymbol ||  (X)->barrier== REDOsymbol ))
+
+mal_export void strBeforeCall(ValPtr v, ValPtr bak);
+mal_export void strAfterCall(ValPtr v, ValPtr bak);
+mal_export void batBeforeCall(ValPtr v, ValPtr bak);
+mal_export void batAfterCall(ValPtr v, ValPtr bak);
+#endif /*  _MAL_INSTR_H */

monetdb/msvc/mal_module.h

@@ -0,0 +1,42 @@
+/*
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0.  If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright 1997 - July 2008 CWI, August 2008 - 2022 MonetDB B.V.
+ */
+
+#ifndef _MAL_SCOPE_H_
+#define _MAL_SCOPE_H_
+
+#include "mal.h"
+/* #define MAL_SCOPE_DEBUG  */
+
+#define MAXSCOPE 256
+
+typedef struct SCOPEDEF {
+	struct SCOPEDEF   *link; /* module with same index value */
+	const char *name;			/* index in namespace */
+	Symbol *space; 			/* type dispatcher table */
+	int isAtomModule; 		/* atom module definition ? */
+	str help;   			/* short description of module functionality*/
+} *Module, ModuleRecord;
+
+mal_export Module   userModule(void);
+mal_export Module   globalModule(const char *nme);
+mal_export Module   fixModule(const char *nme);
+mal_export Module   getModule(const char *nme);
+mal_export void     freeModule(Module cur);
+mal_export void     insertSymbol(Module scope, Symbol prg);
+mal_export void     deleteSymbol(Module scope, Symbol prg);
+mal_export Module   findModule(Module scope, const char *name);
+mal_export Symbol   findSymbol(Module usermodule, const char *mod, const char *fcn);
+mal_export Symbol   findSymbolInModule(Module v, const char *fcn);
+mal_export void     getModuleList(Module** out, int* length);
+mal_export void     freeModuleList(Module* list);
+mal_export void     dumpModules(stream *out);
+mal_export BAT     *getModules(void);
+
+#define getSymbolIndex(N)  (int)(*(unsigned char*)(N))
+
+#endif /* _MAL_SCOPE_H_ */

monetdb/msvc/mal_namespace.h

@@ -0,0 +1,19 @@
+/*
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0.  If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright 1997 - July 2008 CWI, August 2008 - 2022 MonetDB B.V.
+ */
+
+#ifndef _MAL_NAMESPACE_H
+#define _MAL_NAMESPACE_H
+
+mal_export void initNamespace(void);
+mal_export const char *putName(const char *nme);
+mal_export const char *putNameLen(const char *nme, size_t len);
+mal_export const char *getName(const char *nme);
+mal_export const char *getNameLen(const char *nme, size_t len);
+mal_export void delName(const char *nme, size_t len);
+
+#endif /* _MAL_NAMESPACE_H */

monetdb/msvc/mal_stack.h

@@ -0,0 +1,25 @@
+/*
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0.  If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright 1997 - July 2008 CWI, August 2008 - 2022 MonetDB B.V.
+ */
+
+#ifndef _MAL_STACK_H_
+#define _MAL_STACK_H_
+#include "mal.h"
+
+#define stackSize(CNT) (sizeof(ValRecord)*(CNT) + offsetof(MalStack, stk))
+
+mal_export MalStkPtr newGlobalStack(int size);
+mal_export MalStkPtr reallocGlobalStack(MalStkPtr s, int cnt);
+mal_export void freeStack(MalStkPtr stk);
+
+#define getStkRecord(S,P,I) &(S)->stk[(P)->argv[I]]
+#define getStkValue(S,P,I)  ( getStkType(S,P,I)== TYPE_str? \
+					getStkRecord(S,P,I)->val.sval :\
+					getStkRecord(S,P,I)->val.pval )
+#define getStkType(S,P,I)   (S)->stk[(P)->argv[I]].vtype
+#define setStkType(S,P,I,T) (S)->stk[(P)->argv[I]].vtype = T
+#endif /* _MAL_STACK_H_ */

monetdb/msvc/mal_type.h

@@ -0,0 +1,68 @@
+/*
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0.  If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright 1997 - July 2008 CWI, August 2008 - 2022 MonetDB B.V.
+ */
+
+#ifndef MAL_TYPE_H
+#define MAL_TYPE_H
+#include "mal.h"
+
+/* #define DEBUG_MAL_TYPE 1 */
+
+#define TMPMARKER '_'
+#define REFMARKER 'X'
+#define REFMARKERC 'C'
+
+#define newBatType(T)  (1<<16 |  (T & 0377) )
+#define getBatType(X)  ((X) & 0377 )
+#define isaBatType(X)   ((1<<16) & (X) && (X)!= TYPE_any)
+
+#define isAnyExpression(X) ((X) >> 17)
+#define isPolymorphic(X) (isAnyExpression(X) || (X)== TYPE_any)
+
+#define setTypeIndex(X,I) X |= ((I & 017)<<18);
+#define getTypeIndex(X)  (((X)>>18) & 017)
+
+#define isPolyType(X) (isAnyExpression(X) && getTypeIndex(X)>0)
+/*
+ * The symbol/instruction kinds are introduced here instead of reusing the defines
+ * derived from the parser to avoid a loop in the define-structure.
+ */
+
+#define RAISEsymbol     21	/* flow of control modifiers */
+#define CATCHsymbol     22
+#define RETURNsymbol    23
+#define BARRIERsymbol   24
+#define REDOsymbol      25
+#define LEAVEsymbol     26
+#define YIELDsymbol     27
+#define EXITsymbol      29
+
+#define ASSIGNsymbol    40	/* interpreter entry points */
+#define ENDsymbol       41
+#define NOOPsymbol      43	/* no operation required */
+
+#define COMMANDsymbol   61	/* these tokens should be the last group !! */
+#define FUNCTIONsymbol  62	/* the designate the signature start */
+#define FACTORYsymbol   63	/* the co-routine infrastructure */
+#define PATTERNsymbol   64	/* the MAL self-reflection commands */
+
+#define FCNcall     50		/* internal symbols */
+#define FACcall     51
+#define CMDcall     52
+#define THRDcall    53
+#define PATcall     54		/* pattern call */
+
+#define REMsymbol     99	/* commentary to be retained */
+
+mal_export str getTypeName(malType tpe);
+mal_export str getTypeIdentifier(malType tpe);
+mal_export int getAtomIndex(const char *nme, size_t len, int deftpe);
+#define idcmp(n, m)	strcmp(n, m)
+mal_export int isIdentifier(str s);
+mal_export int findGDKtype(int type);	/* used in src/mal/mal_interpreter.c */
+
+#endif /* MAL_TYPE_H */

monetdb/msvc/mapi_querytype.h

@@ -0,0 +1,25 @@
+/*
+ * SPDX-License-Identifier: MPL-2.0
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0.  If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright 1997 - July 2008 CWI, August 2008 - 2023 MonetDB B.V.
+ */
+
+#ifndef _MAPI_QUERYTYPE_H_INCLUDED
+#define _MAPI_QUERYTYPE_H_INCLUDED 1
+
+ /* this definition is a straight copy from sql/include/sql_query.h */
+typedef enum {
+	Q_PARSE = 0,
+	Q_TABLE = 1,
+	Q_UPDATE = 2,
+	Q_SCHEMA = 3,
+	Q_TRANS = 4,
+	Q_PREPARE = 5,
+	Q_BLOCK = 6
+} mapi_query_t;
+
+#endif /* _MAPI_QUERYTYPE_H_INCLUDED */

monetdb/msvc/matomic.h

@@ -0,0 +1,495 @@
+/*
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0.  If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright 1997 - July 2008 CWI, August 2008 - 2022 MonetDB B.V.
+ */
+
+/* This file provides interfaces to perform certain atomic operations
+ * on variables.  Atomic in this sense means that an operation
+ * performed in one thread shows up in another thread either
+ * completely or not at all.
+ *
+ * The following operations are defined:
+ * ATOMIC_VAR_INIT -- initializer for the variable (not necessarily atomic!);
+ * ATOMIC_INIT -- initialize the variable (not necessarily atomic!);
+ * ATOMIC_DESTROY -- destroy the variable
+ * ATOMIC_GET -- return the value of a variable;
+ * ATOMIC_SET -- set the value of a variable;
+ * ATOMIC_XCG -- set the value of a variable, return original value;
+ * ATOMIC_CAS -- compare-and-set (see below)
+ * ATOMIC_ADD -- add a value to a variable, return original value;
+ * ATOMIC_SUB -- subtract a value from a variable, return original value;
+ * ATOMIC_INC -- increment a variable's value, return new value;
+ * ATOMIC_DEC -- decrement a variable's value, return new value;
+ * These interfaces work on variables of type ATOMIC_TYPE (uint64_t).
+ *
+ * The compare-and-set operation is based on the C11 standard: if the
+ * atomic variable equals the expected value, the atomic variable is
+ * replaced by the desired value and true is returned.  Otherwise, the
+ * expected value is replaced by the current value of the atomic
+ * variable and false is returned.
+ *
+ * Some of these are also available for pointers:
+ * ATOMIC_PTR_INIT
+ * ATOMIC_PTR_DESTROY
+ * ATOMIC_PTR_GET
+ * ATOMIC_PTR_SET
+ * ATOMIC_PTR_XCG
+ * ATOMIC_PTR_CAS
+ * To define an atomic pointer, use ATOMIC_PTR_TYPE.  This is a (void *)
+ * pointer that is appropriately adapted for atomic use.
+ *
+ * In addition, the following operations are defined:
+ * ATOMIC_TAS -- test-and-set: set variable to "true" and return old value
+ * ATOMIC_CLEAR -- set variable to "false"
+ * These two operations are only defined on variables of type
+ * ATOMIC_FLAG, and the only values defined for such a variable are
+ * "false" and "true".  The variable can be statically initialized
+ * to "false" using the ATOMIC_FLAG_INIT macro.
+ */
+
+#ifndef _MATOMIC_H_
+#define _MATOMIC_H_
+
+/* define this if you don't want to use atomic instructions */
+/* #define NO_ATOMIC_INSTRUCTIONS */
+
+/* the atomic type we export is always a 64 bit unsigned integer */
+
+/* ignore __STDC_NO_ATOMICS__ if compiling using Intel compiler on
+ * Windows since otherwise we can't compile this at all in C99 mode */
+#if defined(HAVE_STDATOMIC_H) && (!defined(__STDC_NO_ATOMICS__) || (defined(__INTEL_COMPILER) && defined(_WINDOWS))) && !defined(NO_ATOMIC_INSTRUCTIONS)
+
+#include <stdatomic.h>
+
+#if SIZEOF_LONG_LONG == 8
+typedef volatile atomic_ullong ATOMIC_TYPE;
+typedef unsigned long long ATOMIC_BASE_TYPE;
+#elif SIZEOF_LONG == 8
+typedef volatile atomic_ulong ATOMIC_TYPE;
+typedef unsigned long ATOMIC_BASE_TYPE;
+#else
+#error "we need a 64 bit atomic type"
+#endif
+
+#define ATOMIC_INIT(var, val)	atomic_init(var, (ATOMIC_BASE_TYPE) (val))
+#define ATOMIC_DESTROY(var)		((void) 0)
+#define ATOMIC_GET(var)			((ATOMIC_BASE_TYPE) atomic_load(var))
+#define ATOMIC_SET(var, val)	atomic_store(var, (ATOMIC_BASE_TYPE) (val))
+#define ATOMIC_XCG(var, val)	atomic_exchange(var, (ATOMIC_BASE_TYPE) (val))
+#define ATOMIC_CAS(var, exp, des)	atomic_compare_exchange_strong(var, exp, (ATOMIC_BASE_TYPE) (des))
+#define ATOMIC_ADD(var, val)	atomic_fetch_add(var, (ATOMIC_BASE_TYPE) (val))
+#define ATOMIC_SUB(var, val)	atomic_fetch_sub(var, (ATOMIC_BASE_TYPE) (val))
+#define ATOMIC_INC(var)			(atomic_fetch_add(var, 1) + 1)
+#define ATOMIC_DEC(var)			(atomic_fetch_sub(var, 1) - 1)
+#define ATOMIC_OR(var, val)		atomic_fetch_or(var, (ATOMIC_BASE_TYPE) (val))
+#define ATOMIC_AND(var, val)	atomic_fetch_and(var, (ATOMIC_BASE_TYPE) (val))
+
+#ifdef __INTEL_COMPILER
+typedef volatile atomic_address ATOMIC_PTR_TYPE;
+#else
+typedef void *_Atomic volatile ATOMIC_PTR_TYPE;
+#endif
+#define ATOMIC_PTR_INIT(var, val)	atomic_init(var, val)
+#define ATOMIC_PTR_DESTROY(var)		((void) 0)
+#define ATOMIC_PTR_VAR_INIT(val)	ATOMIC_VAR_INIT(val)
+#define ATOMIC_PTR_GET(var)		atomic_load(var)
+#define ATOMIC_PTR_SET(var, val)	atomic_store(var, (void *) (val))
+#define ATOMIC_PTR_XCG(var, val)	atomic_exchange(var, (void *) (val))
+#define ATOMIC_PTR_CAS(var, exp, des)	atomic_compare_exchange_strong(var, exp, (void *) (des))
+
+typedef volatile atomic_flag ATOMIC_FLAG;
+/* ATOMIC_FLAG_INIT is already defined by the include file */
+#define ATOMIC_CLEAR(var)	atomic_flag_clear(var)
+#define ATOMIC_TAS(var)		atomic_flag_test_and_set(var)
+
+#elif defined(_MSC_VER) && !defined(NO_ATOMIC_INSTRUCTIONS)
+
+typedef uint64_t ATOMIC_BASE_TYPE;
+
+#include <intrin.h>
+
+/* On Windows, with Visual Studio 2005, the compiler uses acquire
+ * semantics for read operations on volatile variables and release
+ * semantics for write operations on volatile variables.
+ *
+ * With Visual Studio 2003, volatile to volatile references are
+ * ordered; the compiler will not re-order volatile variable access.
+ * However, these operations could be re-ordered by the processor.
+ *
+ * See
+ * https://docs.microsoft.com/en-us/windows/desktop/Sync/synchronization-and-multiprocessor-issues
+ *
+ * This does not go for the Intel compiler, so there we use
+ * _InterlockedExchangeAdd to load the value of an atomic variable.
+ * There might be a better way, but it's hard to find in the
+ * documentation.
+ */
+
+typedef __declspec(align(8)) volatile ATOMIC_BASE_TYPE ATOMIC_TYPE;
+
+#define ATOMIC_VAR_INIT(val)	(val)
+#define ATOMIC_INIT(var, val)	(*(var) = (val))
+#define ATOMIC_DESTROY(var)		((void) 0)
+
+#if SIZEOF_SIZE_T == 8
+
+#ifdef __INTEL_COMPILER
+#define ATOMIC_GET(var)			((ATOMIC_BASE_TYPE) _InterlockedExchangeAdd64(var, 0))
+#else
+#define ATOMIC_GET(var)			(*(var))
+/* should we use _InterlockedExchangeAdd64(var, 0) instead? */
+#endif
+#define ATOMIC_SET(var, val)	_InterlockedExchange64(var, (ATOMIC_BASE_TYPE) (val))
+#define ATOMIC_XCG(var, val)	_InterlockedExchange64(var, (ATOMIC_BASE_TYPE) (val))
+static inline bool
+ATOMIC_CAS(ATOMIC_TYPE *var, ATOMIC_BASE_TYPE *exp, ATOMIC_BASE_TYPE des)
+{
+	ATOMIC_BASE_TYPE old;
+	old = _InterlockedCompareExchange64(var, des, *exp);
+	if (old == *exp)
+		return true;
+	*exp = old;
+	return false;
+}
+#define ATOMIC_CAS(var, exp, des)	ATOMIC_CAS(var, exp, (ATOMIC_BASE_TYPE) (des))
+#define ATOMIC_ADD(var, val)	_InterlockedExchangeAdd64(var, (ATOMIC_BASE_TYPE) (val))
+#define ATOMIC_SUB(var, val)	_InterlockedExchangeAdd64(var, -(val))
+#define ATOMIC_INC(var)			_InterlockedIncrement64(var)
+#define ATOMIC_DEC(var)			_InterlockedDecrement64(var)
+#define ATOMIC_OR(var, val)		_InterlockedOr64(var, (ATOMIC_BASE_TYPE) (val))
+#define ATOMIC_AND(var, val)	_InterlockedAnd64(var, (ATOMIC_BASE_TYPE) (val))
+
+#else
+
+#ifdef DECLSPEC_NOINITALL
+#define ATOMIC_GET(var)			((ATOMIC_BASE_TYPE) _InlineInterlockedExchangeAdd64(var, 0))
+#define ATOMIC_SET(var, val)	_InlineInterlockedExchange64(var, (ATOMIC_BASE_TYPE) (val))
+#define ATOMIC_XCG(var, val)	_InlineInterlockedExchange64(var, (ATOMIC_BASE_TYPE) (val))
+#define ATOMIC_ADD(var, val)	_InlineInterlockedExchangeAdd64(var, (ATOMIC_BASE_TYPE) (val))
+#define ATOMIC_SUB(var, val)	_InlineInterlockedExchangeAdd64(var, -(val))
+#define ATOMIC_INC(var)			_InlineInterlockedIncrement64(var)
+#define ATOMIC_DEC(var)			_InlineInterlockedDecrement64(var)
+#define ATOMIC_OR(var, val)		_InlineInterlockedOr64(var, (ATOMIC_BASE_TYPE) (val))
+#define ATOMIC_AND(var, val)	_InlineInterlockedAnd64(var, (ATOMIC_BASE_TYPE) (val))
+#else
+#define ATOMIC_GET(var)			((ATOMIC_BASE_TYPE) _InterlockedExchangeAdd64(var, 0))
+#define ATOMIC_SET(var, val)	_InterlockedExchange64(var, (ATOMIC_BASE_TYPE) (val))
+#define ATOMIC_XCG(var, val)	_InterlockedExchange64(var, (ATOMIC_BASE_TYPE) (val))
+#define ATOMIC_ADD(var, val)	_InterlockedExchangeAdd64(var, (ATOMIC_BASE_TYPE) (val))
+#define ATOMIC_SUB(var, val)	_InterlockedExchangeAdd64(var, -(val))
+#define ATOMIC_INC(var)			_InterlockedIncrement64(var)
+#define ATOMIC_DEC(var)			_InterlockedDecrement64(var)
+#define ATOMIC_OR(var, val)		_InterlockedOr64(var, (ATOMIC_BASE_TYPE) (val))
+#define ATOMIC_AND(var, val)	_InterlockedAnd64(var, (ATOMIC_BASE_TYPE) (val))
+#endif
+static inline bool
+ATOMIC_CAS(ATOMIC_TYPE *var, ATOMIC_BASE_TYPE *exp, ATOMIC_BASE_TYPE des)
+{
+	ATOMIC_BASE_TYPE old;
+	old = _InterlockedCompareExchange64(var, des, *exp);
+	if (old == *exp)
+		return true;
+	*exp = old;
+	return false;
+}
+#define ATOMIC_CAS(var, exp, des)	ATOMIC_CAS(var, exp, (ATOMIC_BASE_TYPE) (des))
+
+#endif
+
+typedef PVOID volatile ATOMIC_PTR_TYPE;
+#define ATOMIC_PTR_INIT(var, val)	(*(var) = (val))
+#define ATOMIC_PTR_DESTROY(var)		((void) 0)
+#define ATOMIC_PTR_VAR_INIT(val)	(val)
+#define ATOMIC_PTR_GET(var)		(*(var))
+#define ATOMIC_PTR_SET(var, val)	_InterlockedExchangePointer(var, (PVOID) (val))
+#define ATOMIC_PTR_XCG(var, val)	_InterlockedExchangePointer(var, (PVOID) (val))
+#pragma intrinsic(_InterlockedCompareExchangePointer)
+static inline bool
+ATOMIC_PTR_CAS(ATOMIC_PTR_TYPE *var, void **exp, void *des)
+{
+	void *old;
+	old = _InterlockedCompareExchangePointer(var, des, *exp);
+	if (old == *exp)
+		return true;
+	*exp = old;
+	return false;
+}
+#define ATOMIC_PTR_CAS(var, exp, des)	ATOMIC_PTR_CAS(var, exp, (void *) (des))
+
+typedef volatile int ATOMIC_FLAG;
+#define ATOMIC_FLAG_INIT	{ 0 }
+#define ATOMIC_CLEAR(var)	_InterlockedExchange(var, 0)
+#define ATOMIC_TAS(var)		_InterlockedCompareExchange(var, 1, 0)
+#pragma intrinsic(_InterlockedCompareExchange)
+
+#elif (defined(__GNUC__) || defined(__INTEL_COMPILER))  && defined(__ATOMIC_SEQ_CST) && !(defined(__sun__) && SIZEOF_SIZE_T == 8) && !defined(_MSC_VER) && !defined(NO_ATOMIC_INSTRUCTIONS)
+
+/* the new way of doing this according to GCC (the old way, using
+ * __sync_* primitives is not supported) */
+
+typedef uint64_t ATOMIC_BASE_TYPE;
+typedef volatile ATOMIC_BASE_TYPE ATOMIC_TYPE;
+
+#define ATOMIC_VAR_INIT(val)	(val)
+#define ATOMIC_INIT(var, val)	(*(var) = (val))
+#define ATOMIC_DESTROY(var)	((void) 0)
+
+#define ATOMIC_GET(var)			((ATOMIC_BASE_TYPE) __atomic_load_n(var, __ATOMIC_SEQ_CST))
+#define ATOMIC_SET(var, val)	__atomic_store_n(var, (ATOMIC_BASE_TYPE) (val), __ATOMIC_SEQ_CST)
+#define ATOMIC_XCG(var, val)	__atomic_exchange_n(var, (ATOMIC_BASE_TYPE) (val), __ATOMIC_SEQ_CST)
+#define ATOMIC_CAS(var, exp, des)	__atomic_compare_exchange_n(var, exp, (ATOMIC_BASE_TYPE) (des), false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)
+#define ATOMIC_ADD(var, val)	__atomic_fetch_add(var, (ATOMIC_BASE_TYPE) (val), __ATOMIC_SEQ_CST)
+#define ATOMIC_SUB(var, val)	__atomic_fetch_sub(var, (ATOMIC_BASE_TYPE) (val), __ATOMIC_SEQ_CST)
+#define ATOMIC_INC(var)			__atomic_add_fetch(var, 1, __ATOMIC_SEQ_CST)
+#define ATOMIC_DEC(var)			__atomic_sub_fetch(var, 1, __ATOMIC_SEQ_CST)
+#define ATOMIC_OR(var, val)		__atomic_fetch_or(var, (ATOMIC_BASE_TYPE) (val), __ATOMIC_SEQ_CST)
+#define ATOMIC_AND(var, val)	__atomic_fetch_and(var, (ATOMIC_BASE_TYPE) (val), __ATOMIC_SEQ_CST)
+
+typedef void *volatile ATOMIC_PTR_TYPE;
+#define ATOMIC_PTR_INIT(var, val)	(*(var) = (val))
+#define ATOMIC_PTR_VAR_INIT(val)	(val)
+#define ATOMIC_PTR_DESTROY(var)		((void) 0)
+#define ATOMIC_PTR_GET(var)		__atomic_load_n(var, __ATOMIC_SEQ_CST)
+#define ATOMIC_PTR_SET(var, val)	__atomic_store_n(var, (val), __ATOMIC_SEQ_CST)
+#define ATOMIC_PTR_XCG(var, val)	__atomic_exchange_n(var, (val), __ATOMIC_SEQ_CST)
+#define ATOMIC_PTR_CAS(var, exp, des)	__atomic_compare_exchange_n(var, exp, (void *) (des), false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)
+
+typedef volatile char ATOMIC_FLAG;
+#define ATOMIC_FLAG_INIT	{ 0 }
+#define ATOMIC_CLEAR(var)	__atomic_clear(var, __ATOMIC_SEQ_CST)
+#define ATOMIC_TAS(var)		__atomic_test_and_set(var, __ATOMIC_SEQ_CST)
+
+#else
+
+/* emulate using mutexes */
+
+typedef uint64_t ATOMIC_BASE_TYPE;
+
+#include <pthread.h> /* required for pthread_mutex_t */
+
+typedef struct {
+	ATOMIC_BASE_TYPE val;
+	pthread_mutex_t lck;
+} ATOMIC_TYPE;
+#define ATOMIC_VAR_INIT(v)	{ .val = (v), .lck = PTHREAD_MUTEX_INITIALIZER }
+
+static inline void
+ATOMIC_INIT(ATOMIC_TYPE *var, ATOMIC_BASE_TYPE val)
+{
+	pthread_mutex_init(&var->lck, 0);
+	var->val = val;
+}
+#define ATOMIC_INIT(var, val)	ATOMIC_INIT((var), (ATOMIC_BASE_TYPE) (val))
+
+#define ATOMIC_DESTROY(var)	pthread_mutex_destroy(&(var)->lck)
+
+static inline ATOMIC_BASE_TYPE
+ATOMIC_GET(ATOMIC_TYPE *var)
+{
+	ATOMIC_BASE_TYPE old;
+	pthread_mutex_lock(&var->lck);
+	old = var->val;
+	pthread_mutex_unlock(&var->lck);
+	return old;
+}
+
+static inline void
+ATOMIC_SET(ATOMIC_TYPE *var, ATOMIC_BASE_TYPE val)
+{
+	pthread_mutex_lock(&var->lck);
+	var->val = val;
+	pthread_mutex_unlock(&var->lck);
+}
+#define ATOMIC_SET(var, val)	ATOMIC_SET(var, (ATOMIC_BASE_TYPE) (val))
+
+static inline ATOMIC_BASE_TYPE
+ATOMIC_XCG(ATOMIC_TYPE *var, ATOMIC_BASE_TYPE val)
+{
+	ATOMIC_BASE_TYPE old;
+	pthread_mutex_lock(&var->lck);
+	old = var->val;
+	var->val = val;
+	pthread_mutex_unlock(&var->lck);
+	return old;
+}
+#define ATOMIC_XCG(var, val)	ATOMIC_XCG(var, (ATOMIC_BASE_TYPE) (val))
+
+static inline bool
+ATOMIC_CAS(ATOMIC_TYPE *var, ATOMIC_BASE_TYPE *exp, ATOMIC_BASE_TYPE des)
+{
+	bool ret;
+	pthread_mutex_lock(&var->lck);
+	if (var->val == *exp) {
+		var->val = des;
+		ret = true;
+	} else {
+		*exp = var->val;
+		ret = false;
+	}
+	pthread_mutex_unlock(&var->lck);
+	return ret;
+}
+#define ATOMIC_CAS(var, exp, des)	ATOMIC_CAS(var, exp, (ATOMIC_BASE_TYPE) (des))
+
+static inline ATOMIC_BASE_TYPE
+ATOMIC_ADD(ATOMIC_TYPE *var, ATOMIC_BASE_TYPE val)
+{
+	ATOMIC_BASE_TYPE old;
+	pthread_mutex_lock(&var->lck);
+	old = var->val;
+	var->val += val;
+	pthread_mutex_unlock(&var->lck);
+	return old;
+}
+#define ATOMIC_ADD(var, val)	ATOMIC_ADD(var, (ATOMIC_BASE_TYPE) (val))
+
+static inline ATOMIC_BASE_TYPE
+ATOMIC_SUB(ATOMIC_TYPE *var, ATOMIC_BASE_TYPE val)
+{
+	ATOMIC_BASE_TYPE old;
+	pthread_mutex_lock(&var->lck);
+	old = var->val;
+	var->val -= val;
+	pthread_mutex_unlock(&var->lck);
+	return old;
+}
+#define ATOMIC_SUB(var, val)	ATOMIC_SUB(var, (ATOMIC_BASE_TYPE) (val))
+
+static inline ATOMIC_BASE_TYPE
+ATOMIC_INC(ATOMIC_TYPE *var)
+{
+	ATOMIC_BASE_TYPE new;
+	pthread_mutex_lock(&var->lck);
+	new = var->val += 1;
+	pthread_mutex_unlock(&var->lck);
+	return new;
+}
+
+static inline ATOMIC_BASE_TYPE
+ATOMIC_DEC(ATOMIC_TYPE *var)
+{
+	ATOMIC_BASE_TYPE new;
+	pthread_mutex_lock(&var->lck);
+	new = var->val -= 1;
+	pthread_mutex_unlock(&var->lck);
+	return new;
+}
+
+static inline ATOMIC_BASE_TYPE
+ATOMIC_OR(ATOMIC_TYPE *var, ATOMIC_BASE_TYPE val)
+{
+	ATOMIC_BASE_TYPE old;
+	pthread_mutex_lock(&var->lck);
+	old = var->val;
+	var->val |= val;
+	pthread_mutex_unlock(&var->lck);
+	return old;
+}
+#define ATOMIC_OR(var, val)	ATOMIC_OR(var, (ATOMIC_BASE_TYPE) (val))
+
+static inline ATOMIC_BASE_TYPE
+ATOMIC_AND(ATOMIC_TYPE *var, ATOMIC_BASE_TYPE val)
+{
+	ATOMIC_BASE_TYPE old;
+	pthread_mutex_lock(&var->lck);
+	old = var->val;
+	var->val &= val;
+	pthread_mutex_unlock(&var->lck);
+	return old;
+}
+#define ATOMIC_AND(var, val)	ATOMIC_AND(var, (ATOMIC_BASE_TYPE) (val))
+
+typedef struct {
+	void *val;
+	pthread_mutex_t lck;
+} ATOMIC_PTR_TYPE;
+#define ATOMIC_PTR_VAR_INIT(v)	{ .val = (v), .lck = PTHREAD_MUTEX_INITIALIZER }
+
+static inline void
+ATOMIC_PTR_INIT(ATOMIC_PTR_TYPE *var, void *val)
+{
+	pthread_mutex_init(&var->lck, 0);
+	var->val = val;
+}
+
+#define ATOMIC_PTR_DESTROY(var)	pthread_mutex_destroy(&(var)->lck)
+
+static inline void *
+ATOMIC_PTR_GET(ATOMIC_PTR_TYPE *var)
+{
+	void *old;
+	pthread_mutex_lock(&var->lck);
+	old = var->val;
+	pthread_mutex_unlock(&var->lck);
+	return old;
+}
+
+static inline void
+ATOMIC_PTR_SET(ATOMIC_PTR_TYPE *var, void *val)
+{
+	pthread_mutex_lock(&var->lck);
+	var->val = val;
+	pthread_mutex_unlock(&var->lck);
+}
+
+static inline void *
+ATOMIC_PTR_XCG(ATOMIC_PTR_TYPE *var, void *val)
+{
+	void *old;
+	pthread_mutex_lock(&var->lck);
+	old = var->val;
+	var->val = val;
+	pthread_mutex_unlock(&var->lck);
+	return old;
+}
+
+static inline bool
+ATOMIC_PTR_CAS(ATOMIC_PTR_TYPE *var, void **exp, void *des)
+{
+	bool ret;
+	pthread_mutex_lock(&var->lck);
+	if (var->val == *exp) {
+		var->val = des;
+		ret = true;
+	} else {
+		*exp = var->val;
+		ret = false;
+	}
+	pthread_mutex_unlock(&var->lck);
+	return ret;
+}
+#define ATOMIC_PTR_CAS(var, exp, des)	ATOMIC_PTR_CAS(var, exp, (void *) (des))
+
+typedef struct {
+	bool flg;
+	pthread_mutex_t lck;
+} ATOMIC_FLAG;
+#define ATOMIC_FLAG_INIT	{ .flg = false, .lck = PTHREAD_MUTEX_INITIALIZER }
+
+static inline bool
+ATOMIC_TAS(ATOMIC_FLAG *var)
+{
+	bool old;
+	pthread_mutex_lock(&var->lck);
+	old = var->flg;
+	var->flg = true;
+	pthread_mutex_unlock(&var->lck);
+	return old;
+}
+
+static inline void
+ATOMIC_CLEAR(ATOMIC_FLAG *var)
+{
+	pthread_mutex_lock(&var->lck);
+	var->flg = false;
+	pthread_mutex_unlock(&var->lck);
+}
+
+#define USE_NATIVE_LOCKS 1		/* must use pthread locks */
+
+#endif
+
+#endif	/* _MATOMIC_H_ */

monetdb/msvc/mstring.h

@@ -0,0 +1,158 @@
+/*
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0.  If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright 1997 - July 2008 CWI, August 2008 - 2022 MonetDB B.V.
+ */
+
+#ifndef _MSTRING_H_
+#define _MSTRING_H_
+
+#include <stdarg.h>		/* va_list etc. */
+#include <string.h>		/* strlen */
+
+#if defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ > 4))
+/* not on CentOS 6 (GCC 4.4.7) */
+#define GCC_Pragma(pragma)	_Pragma(pragma)
+#else
+#define GCC_Pragma(pragma)
+#endif
+
+/* copy at most (n-1) bytes from src to dst and add a terminating NULL
+ * byte; return length of src (i.e. can be more than what is copied) */
+static inline size_t
+strcpy_len(char *restrict dst, const char *restrict src, size_t n)
+{
+	if (dst != NULL && n != 0) {
+		for (size_t i = 0; i < n; i++) {
+			if ((dst[i] = src[i]) == 0)
+				return i;
+		}
+		/* for correctness, the decrement isn't needed (just assigning 0
+		 * to dst[n-1] would be sufficient), but to work around a too
+		 * strict GNU C compiler, we do need it */
+		dst[--n] = 0;
+/* in some versions of GCC (at least gcc (Ubuntu 7.5.0-3ubuntu1~18.04)
+ * 7.5.0), the error just can't be turned off when using
+ * --enable-strict, so we just use the (more) expensive way of getting the
+ * right answer (rescan the whole string) */
+#if !defined(__GNUC__) || __GNUC__ > 7 || (__GNUC__ == 7 && __GNUC_MINOR__ > 5)
+/* This code is correct, but GCC gives a warning in certain
+ * conditions, so we disable the warning temporarily.
+ * The warning happens e.g. in
+ *   strcpy_len(buf, "fixed string", sizeof(buf))
+ * where buf is larger than the string. In that case we never get here
+ * since return is executed in the loop above, but the compiler
+ * complains anyway about reading out-of-bounds.
+ * For GCC we use _Pragma to disable the warning (and hence error).
+ * Since other compilers may warn (and hence error out) on
+ * unrecognized pragmas, we use some preprocessor trickery. */
+GCC_Pragma("GCC diagnostic push")
+GCC_Pragma("GCC diagnostic ignored \"-Warray-bounds\"")
+		return n + strlen(src + n);
+GCC_Pragma("GCC diagnostic pop")
+#endif
+	}
+	return strlen(src);
+}
+
+/* copy the NULL terminated list of src strings with a maximum of n
+ * bytes to dst; return the combined length of the src strings */
+static inline size_t
+strconcat_len(char *restrict dst, size_t n, const char *restrict src, ...)
+{
+	va_list ap;
+	size_t i = 0;
+
+	va_start(ap, src);
+	while (src) {
+		size_t l;
+		if (dst && i < n)
+			l = strcpy_len(dst + i, src, n - i);
+		else
+			l = strlen(src);
+		i += l;
+		src = va_arg(ap, const char *);
+	}
+	va_end(ap);
+	return i;
+}
+
+#ifndef __GNUC__
+/* __builtin_expect returns its first argument; it is expected to be
+ * equal to the second argument */
+#define __builtin_expect(expr, expect)	(expr)
+#endif
+
+/*
+ * UTF-8 encoding is as follows:
+ * U-00000000 - U-0000007F: 0xxxxxxx
+ * U-00000080 - U-000007FF: 110zzzzx 10xxxxxx
+ * U-00000800 - U-0000FFFF: 1110zzzz 10zxxxxx 10xxxxxx
+ * U-00010000 - U-0010FFFF: 11110zzz 10zzxxxx 10xxxxxx 10xxxxxx
+ *
+ * To be correctly coded UTF-8, the sequence should be the shortest
+ * possible encoding of the value being encoded.  This means that at
+ * least one of the z bits must be non-zero.  Also note that the four
+ * byte sequence can encode more than is allowed and that the values
+ * U+D800..U+DFFF are not allowed to be encoded.
+ */
+static inline bool
+checkUTF8(const char *v)
+{
+	/* It is unlikely that this functions returns false, because
+	 * it is likely that the string presented is a correctly coded
+	 * UTF-8 string.  So we annotate the tests that are very
+	 * unlikely to succeed, i.e. the ones that lead to a return of
+	 * false, as being expected to return 0 using the
+	 * __builtin_expect function. */
+	if (v != NULL) {
+		if (v[0] != '\200' || v[1] != '\0') {
+			/* check that string is correctly encoded UTF-8 */
+			for (size_t i = 0; v[i]; i++) {
+				/* we do not annotate all tests, only the ones
+				 * leading directly to an unlikely return
+				 * statement */
+				if ((v[i] & 0x80) == 0) {
+					;
+				} else if ((v[i] & 0xE0) == 0xC0) {
+					if (__builtin_expect(((v[i] & 0x1E) == 0), 0))
+						return false;
+					if (__builtin_expect(((v[++i] & 0xC0) != 0x80), 0))
+						return false;
+				} else if ((v[i] & 0xF0) == 0xE0) {
+					if ((v[i++] & 0x0F) == 0) {
+						if (__builtin_expect(((v[i] & 0xE0) != 0xA0), 0))
+							return false;
+					} else {
+						if (__builtin_expect(((v[i] & 0xC0) != 0x80), 0))
+							return false;
+					}
+					if (__builtin_expect(((v[++i] & 0xC0) != 0x80), 0))
+						return false;
+				} else if (__builtin_expect(((v[i] & 0xF8) == 0xF0), 1)) {
+					if ((v[i++] & 0x07) == 0) {
+						if (__builtin_expect(((v[i] & 0x30) == 0), 0))
+							return false;
+					}
+					if (__builtin_expect(((v[i] & 0xC0) != 0x80), 0))
+						return false;
+					if (__builtin_expect(((v[++i] & 0xC0) != 0x80), 0))
+						return false;
+					if (__builtin_expect(((v[++i] & 0xC0) != 0x80), 0))
+						return false;
+				} else {
+					return false;
+				}
+			}
+		}
+	}
+	return true;
+}
+
+#ifndef __GNUC__
+#undef __builtin_expect
+#endif
+
+#endif

monetdb/msvc/sql_atom.h

@@ -0,0 +1,72 @@
+/*
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0.  If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright 1997 - July 2008 CWI, August 2008 - 2022 MonetDB B.V.
+ */
+
+#ifndef _SQL_ATOM_H_
+#define _SQL_ATOM_H_
+
+#include "sql_mem.h"
+#include "sql_types.h"
+
+#define atom_null(a) (((atom*)a)->isnull)
+
+extern void atom_init( atom *a );
+extern atom *atom_bool( sql_allocator *sa, sql_subtype *tpe, bit t);
+#ifdef HAVE_HGE
+extern atom *atom_int( sql_allocator *sa, sql_subtype *tpe, hge val);
+#else
+extern atom *atom_int( sql_allocator *sa, sql_subtype *tpe, lng val);
+#endif
+extern atom *atom_float( sql_allocator *sa, sql_subtype *tpe, dbl val);
+extern atom *atom_string( sql_allocator *sa, sql_subtype *tpe, const char *val);
+extern atom *atom_general( sql_allocator *sa, sql_subtype *tpe, const char *val);
+#ifdef HAVE_HGE
+extern atom *atom_dec( sql_allocator *sa, sql_subtype *tpe, hge val);
+#else
+extern atom *atom_dec( sql_allocator *sa, sql_subtype *tpe, lng val);
+#endif
+extern atom *atom_ptr( sql_allocator *sa, sql_subtype *tpe, void *v);
+extern atom *atom_general_ptr( sql_allocator *sa, sql_subtype *tpe, void *v);
+
+extern unsigned int atom_num_digits( atom *a );
+
+/* cast atom a to type tp (success returns not NULL, fail returns NULL) */
+extern atom *atom_cast(sql_allocator *sa, atom *a, sql_subtype *tp);
+
+extern char *atom2string(sql_allocator *sa, atom *a);
+extern char *atom2sql(sql_allocator *sa, atom *a, int timezone);
+extern sql_subtype *atom_type(atom *a);
+extern atom *atom_set_type(sql_allocator *sa, atom *a, sql_subtype *t);
+
+#ifdef HAVE_HGE
+extern hge atom_get_int(atom *a);
+#else
+extern lng atom_get_int(atom *a);
+#endif
+
+extern int atom_cmp(atom *a1, atom *a2);
+
+extern atom *atom_neg(sql_allocator *sa, atom *a);
+extern atom *atom_add(sql_allocator *sa, atom *a1, atom *a2);
+extern atom *atom_sub(sql_allocator *sa, atom *a1, atom *a2);
+extern atom *atom_mul(sql_allocator *sa, atom *a1, atom *a2);
+extern atom *atom_inc(sql_allocator *sa, atom *a);
+extern int atom_is_true(atom *a);
+extern int atom_is_false(atom *a);
+extern int atom_is_zero(atom *a);
+
+#ifdef HAVE_HGE
+#define MAX_SCALE 39
+extern const hge scales[MAX_SCALE];
+#else
+#define MAX_SCALE 19
+extern const lng scales[MAX_SCALE];
+#endif
+
+extern atom *atom_zero_value(sql_allocator *sa, sql_subtype* tpe);
+extern atom *atom_max_value(sql_allocator *sa, sql_subtype *tpe);
+#endif /* _SQL_ATOM_H_ */

monetdb/msvc/sql_backend.h

@@ -0,0 +1,62 @@
+/*
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0.  If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright 1997 - July 2008 CWI, August 2008 - 2022 MonetDB B.V.
+ */
+
+#ifndef _SQL_BACKEND_H_
+#define _SQL_BACKEND_H_
+
+#include "sql_mem.h"
+#include "sql_catalog.h"
+#include "sql_relation.h"
+
+typedef void (*freecode_fptr) (const char *mod, int clientid, const char *name);
+
+typedef char *(*create_user_fptr) (ptr mvc, char *user, char *passwd, char enc, char *fullname, sqlid schema_id, char *schema_path, sqlid grantor_id);
+typedef int  (*drop_user_fptr) (ptr mvc, char *user);
+typedef int  (*find_user_fptr) (ptr mvc, char *user);
+typedef void (*create_privileges_fptr) (ptr mvc, sql_schema *s);
+typedef int  (*schema_has_user_fptr) (ptr mvc, sql_schema *s);
+typedef int  (*alter_user_fptr) (ptr mvc, str user, str passwd, char enc, sqlid schema_id, char *schema_path, str oldpasswd);
+typedef int  (*rename_user_fptr) (ptr mvc, str olduser, str newuser);
+typedef void*  (*schema_user_dependencies) (ptr mvc, int schema_id);
+typedef void  (*create_function) (ptr mvc, str name, sql_rel *rel, sql_table *t);
+typedef int  (*resolve_function) (ptr mvc, sql_func *f, const char *fimp, bit *side_effect);
+typedef int  (*has_module_function) (ptr mvc, char *name);
+typedef void *(*create_sub_backend) (void *mvc, void *client);
+
+/* backing struct for this interface */
+typedef struct _backend_functions {
+	freecode_fptr fcode;
+	create_user_fptr fcuser;
+	drop_user_fptr fduser;
+	find_user_fptr ffuser;
+	create_privileges_fptr fcrpriv;
+	schema_has_user_fptr fshuser;
+	alter_user_fptr fauser;
+	rename_user_fptr fruser;
+	schema_user_dependencies fschuserdep;
+	resolve_function fresolve_function;
+	has_module_function fhas_module_function;
+	create_sub_backend sub_backend;
+} backend_functions;
+
+extern void backend_freecode(const char *mod, int clientid, const char *name);
+
+extern char *backend_create_user(ptr mvc, char *user, char *passwd, char enc, char *fullname, sqlid defschemid, char *schema_path, sqlid grantor);
+extern int  backend_drop_user(ptr mvc, char *user);
+extern int  backend_find_user(ptr mp, char *user);
+extern void backend_create_privileges(ptr mvc, sql_schema *s);
+extern int  backend_schema_has_user(ptr mvc, sql_schema *s);
+extern int	backend_alter_user(ptr mvc, str user, str passwd, char enc, sqlid schema_id, char *schema_path, str oldpasswd);
+extern int	backend_rename_user(ptr mvc, str olduser, str newuser);
+extern void*	backend_schema_user_dependencies(ptr trans, sqlid schema_id);
+extern int	backend_resolve_function(ptr trans, sql_func *f, const char *fimp, bit *side_effect);
+extern int	backend_has_module(ptr M, char *name);
+
+extern backend_functions be_funcs;
+
+#endif /* _SQL_BACKEND_H_ */

monetdb/msvc/sql_catalog.h

@@ -0,0 +1,877 @@
+/*
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0.  If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright 1997 - July 2008 CWI, August 2008 - 2022 MonetDB B.V.
+ */
+
+#ifndef SQL_CATALOG_H
+#define SQL_CATALOG_H
+
+#include "sql_mem.h"
+#include "sql_list.h"
+#include "sql_hash.h"
+#include "mapi_querytype.h"
+#include "stream.h"
+#include "matomic.h"
+
+#define sql_shared_module_name "sql"
+#define sql_private_module_name "user"
+
+#define tr_none		1
+#define tr_readonly	2
+#define tr_writable	4
+#define tr_append 	8
+#define tr_snapshot 16
+#define tr_serializable 32
+
+#define ACT_NO_ACTION 0
+#define ACT_CASCADE 1
+#define ACT_RESTRICT 2
+#define ACT_SET_NULL 3
+#define ACT_SET_DEFAULT 4
+
+#define DROP_RESTRICT 0
+#define DROP_CASCADE 1
+#define DROP_CASCADE_START 2
+
+#define PRIV_SELECT 1
+#define PRIV_UPDATE 2
+#define PRIV_INSERT 4
+#define PRIV_DELETE 8
+#define PRIV_EXECUTE 16
+#define PRIV_GRANT 32
+#define PRIV_TRUNCATE 64
+/* global privs */
+#define PRIV_COPYFROMFILE 1
+#define PRIV_COPYINTOFILE 2
+
+typedef enum sql_dependency {
+	SCHEMA_DEPENDENCY = 1,
+	TABLE_DEPENDENCY = 2,
+	COLUMN_DEPENDENCY = 3,
+	KEY_DEPENDENCY = 4,
+	VIEW_DEPENDENCY = 5,
+	USER_DEPENDENCY = 6,
+	FUNC_DEPENDENCY = 7,
+	TRIGGER_DEPENDENCY = 8,
+	OWNER_DEPENDENCY = 9,
+	INDEX_DEPENDENCY = 10,
+	FKEY_DEPENDENCY = 11,
+	SEQ_DEPENDENCY = 12,
+	PROC_DEPENDENCY = 13,
+	BEDROPPED_DEPENDENCY = 14, /*The object must be dropped when the dependent object is dropped independently of the DROP type.*/
+	TYPE_DEPENDENCY = 15
+} sql_dependency;
+
+#define NO_DEPENDENCY 0
+#define HAS_DEPENDENCY 1
+#define CICLE_DEPENDENCY 2
+#define DEPENDENCY_CHECK_ERROR 3
+#define DEPENDENCY_CHECK_OK 0
+
+#define ROLE_PUBLIC   1
+#define ROLE_SYSADMIN 2
+#define USER_MONETDB  3
+
+#define SCALE_NONE	0
+#define SCALE_FIX	1	/* many numerical functions require equal
+                           scales/precision for all their inputs */
+#define SCALE_NOFIX	2
+#define SCALE_MUL	3	/* multiplication gives the sum of scales */
+#define SCALE_DIV	4	/* div on the other hand reduces the scales */
+#define DIGITS_ADD	5	/* some types grow under functions (concat) */
+#define INOUT		6	/* output type equals input type */
+#define SCALE_EQ	7	/* user defined functions need equal scales */
+
+#define RDONLY 0
+#define RD_INS 1
+#define RD_UPD_ID 2
+#define RD_UPD_VAL 3
+#define QUICK  4
+#define RD_EXT 5
+
+/* the following list of macros are used by rel_rankop function */
+#define UNBOUNDED_PRECEDING_BOUND 0
+#define UNBOUNDED_FOLLOWING_BOUND 1
+#define CURRENT_ROW_BOUND         2
+
+#define FRAME_ROWS  0 		/* number of rows (preceding/following) */
+#define FRAME_RANGE 1		/* logical range (based on the ordering column).
+				   Example:
+				   RANGE BETWEEN INTERVAL '1' MONTH PRECEDING
+				             AND INTERVAL '1' MONTH FOLLOWING */
+#define FRAME_GROUPS 2
+#define FRAME_UNBOUNDED_TILL_CURRENT_ROW 3
+#define FRAME_CURRENT_ROW_TILL_UNBOUNDED 4
+#define FRAME_ALL 5
+#define FRAME_CURRENT_ROW 6
+
+/* the following list of macros are used by SQLwindow_bound function */
+#define BOUND_FIRST_HALF_PRECEDING  0
+#define BOUND_FIRST_HALF_FOLLOWING  1
+#define BOUND_SECOND_HALF_PRECEDING 2
+#define BOUND_SECOND_HALF_FOLLOWING 3
+#define CURRENT_ROW_PRECEDING       4
+#define CURRENT_ROW_FOLLOWING       5
+
+#define EXCLUDE_NONE 0		/* nothing excluded (also the default) */
+#define EXCLUDE_CURRENT_ROW 1	/* exclude the current row */
+#define EXCLUDE_GROUP 2		/* exclude group */
+#define EXCLUDE_TIES 3		/* exclude group but not the current row */
+
+/* The following macros are used in properties field of sql_table */
+#define PARTITION_RANGE       1
+#define PARTITION_LIST        2
+#define PARTITION_COLUMN      4
+#define PARTITION_EXPRESSION  8
+
+#define STORAGE_MAX_VALUE_LENGTH 2048
+
+#define cur_user 1
+#define cur_role 2
+
+#define sql_max(i1,i2) ((i1)<(i2))?(i2):(i1)
+
+#define dt_schema 	"%dt%"
+#define isDeclaredSchema(s) 	(strcmp(s->base.name, dt_schema) == 0)
+
+extern const char *TID;
+
+typedef enum temp_t {
+	SQL_PERSIST = 0,
+	SQL_LOCAL_TEMP = 1,
+	SQL_GLOBAL_TEMP = 2,
+	SQL_DECLARED_TABLE = 3,	/* variable inside a stored procedure */
+	SQL_MERGE_TABLE = 4,
+	/* SQL_STREAM = 5, stream tables are not used anymore */
+	SQL_REMOTE = 6,
+	SQL_REPLICA_TABLE = 7
+} temp_t;
+
+typedef enum comp_type {
+	cmp_gt = 0,
+	cmp_gte = 1,
+	cmp_lte = 2,
+	cmp_lt = 3,
+	cmp_equal = 4,
+	cmp_notequal = 5,
+
+	cmp_filter = 6,
+	cmp_or = 7,
+	cmp_in = 8,			/* in value list */
+	cmp_notin = 9,			/* not in value list */
+
+	mark_in = 10,			/* mark joins */
+	mark_notin = 11,
+
+	/* The followin cmp_* are only used within stmt (not sql_exp) */
+	cmp_all = 12,			/* special case for crossproducts */
+	cmp_project = 13,		/* special case for projection joins */
+	cmp_joined = 14, 		/* special case already joined */
+	cmp_left_project = 15	/* last step of outer join */
+} comp_type;
+
+#define is_theta_exp(e) ((e) == cmp_gt || (e) == cmp_gte || (e) == cmp_lte ||\
+						 (e) == cmp_lt || (e) == cmp_equal || (e) == cmp_notequal)
+
+#define is_complex_exp(et) ((et) == cmp_or || (et) == cmp_in || (et) == cmp_notin || (et) == cmp_filter)
+
+#define is_equality_or_inequality_exp(et) ((et) == cmp_equal || (et) == cmp_notequal || (et) == cmp_in || \
+							 			   (et) == cmp_notin || (et) == mark_in || (et) == mark_notin)
+
+typedef enum commit_action_t {
+	CA_COMMIT, 	/* commit rows, only for persistent tables */
+	CA_DELETE, 	/* delete rows */
+	CA_PRESERVE,	/* preserve rows */
+	CA_DROP		/* drop table */
+} ca_t;
+
+typedef int sqlid;
+typedef void *sql_store;
+
+typedef struct sql_base {
+	unsigned int
+		new:1,
+		deleted:1,
+		refcnt:30;
+	sqlid id;
+	char *name;
+} sql_base;
+
+#define isNew(x)          ((x)->base.new)
+
+extern void base_init(sql_allocator *sa, sql_base * b, sqlid id, bool isnew, const char *name);
+
+typedef struct changeset {
+	sql_allocator *sa;
+	fdestroy destroy;
+	struct list *set;
+	struct list *dset;
+	node *nelm;
+} changeset;
+
+typedef struct objlist {
+	list *l;
+	sql_hash *h;
+	sql_store store;
+} objlist;
+
+struct sql_trans;
+struct sql_change;
+struct objectset;
+struct versionhead;
+struct os_iter {
+	struct objectset *os;
+	struct sql_trans *tr;
+	struct versionhead *n;
+	struct sql_hash_e *e;
+	const char *name;
+};
+
+/* transaction changes */
+typedef int (*tc_valid_fptr) (struct sql_trans *tr, struct sql_change *c/*, ulng commit_ts, ulng oldest*/);
+typedef int (*tc_log_fptr) (struct sql_trans *tr, struct sql_change *c);								/* write changes to the log */
+typedef int (*tc_commit_fptr) (struct sql_trans *tr, struct sql_change *c, ulng commit_ts, ulng oldest);/* commit/rollback changes */
+typedef int (*tc_cleanup_fptr) (sql_store store, struct sql_change *c, ulng oldest);	/* garbage collection, ie cleanup structures when possible */
+typedef void (*destroy_fptr)(sql_store store, sql_base *b);
+typedef int (*validate_fptr)(struct sql_trans *tr, sql_base *b, int delete);
+
+extern struct objectset *os_new(sql_allocator *sa, destroy_fptr destroy, bool temporary, bool unique, bool concurrent, sql_store store);
+extern struct objectset *os_dup(struct objectset *os);
+extern void os_destroy(struct objectset *os, sql_store store);
+extern int /*ok, error (name existed) and conflict (added before) */ os_add(struct objectset *os, struct sql_trans *tr, const char *name, sql_base *b);
+extern int os_del(struct objectset *os, struct sql_trans *tr, const char *name, sql_base *b);
+extern int os_size(struct objectset *os, struct sql_trans *tr);
+extern int os_empty(struct objectset *os, struct sql_trans *tr);
+extern int os_remove(struct objectset *os, struct sql_trans *tr, const char *name);
+extern sql_base *os_find_name(struct objectset *os, struct sql_trans *tr, const char *name);
+extern sql_base *os_find_id(struct objectset *os, struct sql_trans *tr, sqlid id);
+/* iterating (for example for location functinos) */
+extern void os_iterator(struct os_iter *oi, struct objectset *os, struct sql_trans *tr, const char *name /*optional*/);
+extern sql_base *oi_next(struct os_iter *oi);
+extern bool os_obj_intransaction(struct objectset *os, struct sql_trans *tr, sql_base *b);
+extern bool os_has_changes(struct objectset *os, struct sql_trans *tr);
+
+extern objlist *ol_new(sql_allocator *sa, destroy_fptr destroy, sql_store store);
+extern void ol_destroy(objlist *ol, sql_store store);
+extern int ol_add(objlist *ol, sql_base *data);
+extern void ol_del(objlist *ol, sql_store store, node *data);
+extern node *ol_find_name(objlist *ol, const char *name);
+extern node *ol_find_id(objlist *ol, sqlid id);
+extern node *ol_rehash(objlist *ol, const char *oldname, node *n);
+#define ol_length(ol) (list_length(ol->l))
+#define ol_first_node(ol) (ol->l->h)
+#define ol_last_node(ol) (ol->l->t)
+#define ol_fetch(ol,nr) (list_fetch(ol->l, nr))
+
+extern void cs_new(changeset * cs, sql_allocator *sa, fdestroy destroy);
+extern void cs_destroy(changeset * cs, void *data);
+extern void cs_add(changeset * cs, void *elm, bool isnew);
+extern void cs_del(changeset * cs, void *gdata, node *elm, bool isnew);
+extern int cs_size(changeset * cs);
+extern node *cs_find_id(changeset * cs, sqlid id);
+
+typedef void *backend_code;
+typedef size_t backend_stack;
+
+typedef struct sql_schema {
+	sql_base base;
+	sqlid auth_id;
+	sqlid owner;
+	bit system;		/* system or user schema */
+	// TODO? int type;	/* persistent, session local, transaction local */
+
+	struct objectset *tables;
+	struct objectset *types;
+	struct objectset *funcs;
+	struct objectset *seqs;
+	struct objectset *keys;		/* Names for keys, idxs, and triggers and parts are */
+	struct objectset *idxs;		/* global, but these objects are only */
+	struct objectset *triggers;	/* useful within a table */
+	struct objectset *parts;
+
+	char *internal; 	/* optional internal module name */
+	sql_store store;
+} sql_schema;
+
+typedef struct sql_catalog {
+	struct objectset *schemas;
+	struct objectset *objects;
+} sql_catalog;
+
+typedef struct sql_trans {
+	char *name;
+
+	ulng ts;			/* transaction start timestamp */
+	ulng tid;			/* transaction id */
+
+	sql_store store;	/* keep link into the global store */
+	MT_Lock lock;		/* lock protecting concurrent writes to the changes list */
+	list *changes;		/* list of changes */
+
+	list *dropped;  	/* protection against recursive cascade action*/
+	list *predicates;	/* list of read predicates logged during update transactions */
+	list *dependencies;	/* list of dependencies created (list of sqlids from the objects) */
+	list *depchanges;	/* list of dependencies changed (it would be tested for conflicts at the end of the transaction) */
+
+	int logchanges;		/* count number of changes to be applied to the wal */
+	int active;			/* is active transaction */
+	int status;			/* status of the last query */
+
+	sql_catalog *cat;
+	sql_schema *tmp;	/* each session has its own tmp schema */
+	changeset localtmps;
+	sql_allocator *sa;	/* transaction allocator */
+
+	struct sql_trans *parent;	/* multilevel transaction support */
+} sql_trans;
+
+typedef enum sql_class {
+	EC_ANY,
+	EC_TABLE,
+	EC_BIT,
+	EC_CHAR,
+	EC_STRING,
+	EC_BLOB,
+	EC_POS,
+	EC_NUM,
+	EC_MONTH,
+	EC_SEC,
+	EC_DEC,
+	EC_FLT,
+	EC_TIME,
+	EC_TIME_TZ,
+	EC_DATE,
+	EC_TIMESTAMP,
+	EC_TIMESTAMP_TZ,
+	EC_GEOM,
+	EC_EXTERNAL,
+	EC_MAX /* evaluated to the max value, should be always kept at the bottom */
+} sql_class;
+
+#define has_tz(e)			(EC_TEMP_TZ(e))
+#define type_has_tz(t)		has_tz((t)->type->eclass)
+#define EC_VARCHAR(e)		((e)==EC_CHAR||(e)==EC_STRING)
+#define EC_INTERVAL(e)		((e)==EC_MONTH||(e)==EC_SEC)
+#define EC_NUMBER(e)		((e)==EC_POS||(e)==EC_NUM||EC_INTERVAL(e)||(e)==EC_DEC||(e)==EC_FLT)
+#define EC_EXACTNUM(e)		((e)==EC_NUM||(e)==EC_DEC)
+#define EC_APPNUM(e)		((e)==EC_FLT)
+#define EC_COMPUTE(e)		((e)==EC_NUM||(e)==EC_FLT)
+#define EC_BOOLEAN(e)		((e)==EC_BIT||(e)==EC_NUM||(e)==EC_FLT)
+#define EC_TEMP_TZ(e)		((e)==EC_TIME_TZ||(e)==EC_TIMESTAMP_TZ)
+#define EC_TEMP(e)			((e)==EC_TIME||(e)==EC_DATE||(e)==EC_TIMESTAMP||EC_TEMP_TZ(e))
+#define EC_TEMP_FRAC(e)		((e)==EC_TIME||(e)==EC_TIMESTAMP||EC_TEMP_TZ(e))
+#define EC_TEMP_NOFRAC(e)	((e)==EC_TIME||(e)==EC_TIMESTAMP)
+#define EC_SCALE(e)			((e)==EC_DEC||EC_TEMP_FRAC(e)||(e)==EC_SEC)
+#define EC_BACKEND_FIXED(e)	(EC_NUMBER(e)||(e)==EC_BIT||EC_TEMP(e))
+
+typedef struct sql_type {
+	sql_base base;
+
+	char *impl; /* backend correspondent type */
+	unsigned int digits;
+	unsigned int scale;	/* indicates how scale is used in functions */
+	int localtype;		/* localtype, need for coersions */
+	unsigned char radix;
+	unsigned int bits;
+	sql_class eclass; 	/* types are grouped into equivalence classes */
+	sql_schema *s;
+} sql_type;
+
+typedef struct sql_alias {
+	char *name;
+	char *alias;
+} sql_alias;
+
+#define ARG_IN 1
+#define ARG_OUT 0
+
+typedef struct sql_subtype {
+	sql_type *type;
+	unsigned int digits;
+	unsigned int scale;
+} sql_subtype;
+
+/* sql_func need type transform rules types are equal if underlying
+ * types are equal + scale is equal if types do not mach we try type
+ * conversions which means for simple 1 arg functions
+ */
+
+typedef struct sql_arg {
+	char *name;
+	bte inout;
+	sql_subtype type;
+} sql_arg;
+
+typedef enum sql_ftype {
+	F_FUNC = 1,
+	F_PROC = 2,
+	F_AGGR = 3,
+	F_FILT = 4,
+	F_UNION = 5,
+	F_ANALYTIC = 6,
+	F_LOADER = 7
+} sql_ftype;
+
+#define IS_FUNC(f)     ((f)->type == F_FUNC)
+#define IS_PROC(f)     ((f)->type == F_PROC)
+#define IS_AGGR(f)     ((f)->type == F_AGGR)
+#define IS_FILT(f)     ((f)->type == F_FILT)
+#define IS_UNION(f)    ((f)->type == F_UNION)
+#define IS_ANALYTIC(f) ((f)->type == F_ANALYTIC)
+#define IS_LOADER(f)   ((f)->type == F_LOADER)
+
+#define FUNC_TYPE_STR(type, F, fn) \
+	switch (type) { \
+		case F_FUNC: \
+			F = "FUNCTION"; \
+			fn = "function"; \
+			break; \
+		case F_PROC: \
+			F = "PROCEDURE"; \
+			fn = "procedure"; \
+			break; \
+		case F_AGGR: \
+			F = "AGGREGATE"; \
+			fn = "aggregate"; \
+			break; \
+		case F_FILT: \
+			F = "FILTER FUNCTION"; \
+			fn = "filter function"; \
+			break; \
+		case F_UNION: \
+			F = "UNION FUNCTION"; \
+			fn = "table returning function"; \
+			break; \
+		case F_ANALYTIC: \
+			F = "WINDOW FUNCTION"; \
+			fn = "window function"; \
+			break; \
+		case F_LOADER: \
+			F = "LOADER FUNCTION"; \
+			fn = "loader function"; \
+			break; \
+		default: \
+			assert(0); \
+	}
+
+typedef enum sql_flang {
+	FUNC_LANG_INT = 0, /* internal */
+	FUNC_LANG_MAL = 1, /* create sql external mod.func */
+	FUNC_LANG_SQL = 2, /* create ... sql function/procedure */
+	FUNC_LANG_R = 3,   /* create .. language R */
+	FUNC_LANG_C = 4,   /* create .. language C */
+	FUNC_LANG_J = 5,   /* create .. language JAVASCRIPT (not implemented) */
+	/* this should probably be done in a better way */
+	FUNC_LANG_PY = 6,       /* create .. language PYTHON */
+	FUNC_LANG_MAP_PY = 7,   /* create .. language PYTHON_MAP */
+	/* values 8 and 9 were for Python 2 */
+	FUNC_LANG_PY3 = 10,     /* create .. language PYTHON3 */
+	FUNC_LANG_MAP_PY3 = 11, /* create .. language PYTHON3_MAP */
+	FUNC_LANG_CPP = 12      /* create .. language CPP */
+} sql_flang;
+
+#define LANG_EXT(l)  ((l)>FUNC_LANG_SQL)
+#define UDF_LANG(l)  ((l)>=FUNC_LANG_SQL)
+
+typedef struct sql_func {
+	sql_base base;
+
+	char *mod;
+	char *imp;
+		/*
+		Backend implementation function after it gets instantiated.
+		During instantiation 'imp' will be set, but look for the 'instantiated' value to check if it's done or not.
+		Note that functions other than SQL and MAL, don't require instantiation and 'imp' is always set.
+		*/
+	sql_ftype type;
+	list *ops;	/* param list */
+	list *res;	/* list of results */
+	sql_flang lang;
+	char *query;	/* sql code */
+	bit semantics; /*When set to true, function incorporates some kind of null semantics.*/
+	bit side_effect;
+	bit varres;	/* variable output result */
+	bit vararg;	/* variable input arguments */
+	bit system;	/* system function */
+	bit instantiated; /* if the function is instantiated */
+	int fix_scale;
+			/*
+	   		   SCALE_NOFIX/SCALE_NONE => nothing
+	   		   SCALE_FIX => input scale fixing,
+	   		   SCALE_ADD => leave inputs as is and do add scales
+	   		   example numerical multiplication
+	   		   SCALE_SUB => first input scale, fix with second scale
+	   		   result scale is equal to first input
+	   		   example numerical division
+	   		   DIGITS_ADD => result digits, sum of args
+	   		   example string concat
+	 		*/
+	sql_schema *s;
+	sql_allocator *sa;
+} sql_func;
+
+typedef struct sql_subfunc {
+	sql_func *func;
+	list *res;
+	list *coltypes; /* we need this for copy into from loader */
+	list *colnames; /* we need this for copy into from loader */
+	char *sname, *tname; /* we need this for create table from loader */
+} sql_subfunc;
+
+typedef enum key_type {
+	pkey,
+	ukey,
+	fkey
+} key_type;
+
+typedef struct sql_kc {
+	struct sql_column *c;
+	int trunc;		/* 0 not truncated, >0 colum is truncated */
+} sql_kc;
+
+typedef enum idx_type {
+	hash_idx,
+	join_idx,
+	oph_idx,		/* order preserving hash */
+	no_idx,			/* no idx, ie no storage */
+	imprints_idx,
+	ordered_idx,
+	new_idx_types
+} idx_type;
+
+#define hash_index(t) 		((t) == hash_idx || (t) == oph_idx)
+#define idx_has_column(t) 	(hash_index(t) || (t) == join_idx)
+#define oid_index(t)		((t) == join_idx)
+#define non_updatable_index(t) ((t) == ordered_idx || (t) == no_idx || !idx_has_column(t))
+
+typedef struct sql_idx {
+	sql_base base;
+	idx_type type;		/* unique */
+	struct list *columns;	/* list of sql_kc */
+	struct sql_table *t;
+	struct sql_key *key;	/* key */
+	ATOMIC_PTR_TYPE data;
+} sql_idx;
+
+/* fkey consists of two of these */
+typedef struct sql_key {	/* pkey, ukey, fkey */
+	sql_base base;
+	key_type type;		/* pkey, ukey, fkey */
+	sql_idx *idx;		/* idx to accelerate key check */
+
+	struct list *columns;	/* list of sql_kc */
+	struct sql_table *t;
+	int drop_action;	/* only needed for alter drop key */
+} sql_key;
+
+typedef struct sql_ukey {	/* pkey, ukey */
+	sql_key k;
+	//list *keys;
+} sql_ukey;
+
+typedef struct sql_fkey {	/* fkey */
+	sql_key k;
+	/* 0=no action, 1=cascade, 2=restrict (default setting), 3=set null, 4=set default */
+	int on_delete;
+	int on_update;
+	sqlid rkey;
+} sql_fkey;
+
+typedef struct sql_trigger {
+	sql_base base;
+	sht time;		/* before or after */
+	sht orientation; 	/* row or statement */
+	sht event;		/* insert, delete, update, truncate */
+	/* int action_order;	 TODO, order within the set of triggers */
+	struct list *columns;	/* update trigger on list of (sql_kc) columns */
+
+	struct sql_table *t;
+	char *old_name;		/* name referencing the old values */
+	char *new_name;		/* name referencing the new values */
+
+	char *condition; 	/* when search condition, ie query */
+	char *statement;	/* action, ie list of sql statements */
+} sql_trigger;
+
+typedef struct sql_sequence {
+	sql_base base;
+	lng start;
+	lng minvalue;
+	lng maxvalue;
+	lng increment;
+	lng cacheinc;
+	bit cycle;
+	bit bedropped;		/*Drop the SEQUENCE if you are dropping the column, e.g., SERIAL COLUMN".*/
+	sql_schema *s;
+} sql_sequence;
+
+typedef struct sql_column {
+	sql_base base;
+	sql_subtype type;
+	int colnr;
+	bit null;
+	char *def;
+	char unique; 		/* 0 NOT UNIQUE, 1 SUB_UNIQUE, 2 UNIQUE */
+	int drop_action;	/* only used for alter statements */
+	char *storage_type;
+	size_t dcount;
+	void *min;
+	void *max;
+
+	struct sql_table *t;
+	ATOMIC_PTR_TYPE data;
+} sql_column;
+
+typedef enum table_types {
+	tt_table = 0, 		/* table */
+	tt_view = 1, 		/* view */
+	tt_merge_table = 3,	/* multiple tables form one table */
+	/* tt_stream = 4, stream tables are not used anymore */
+	tt_remote = 5,		/* stored on a remote server */
+	tt_replica_table = 6	/* multiple replica of the same table */
+} table_types;
+
+#define TABLE_TYPE_DESCRIPTION(tt, properties)                                                                      \
+((tt) == tt_table)?"TABLE":((tt) == tt_view)?"VIEW":((tt) == tt_merge_table && !(properties))?"MERGE TABLE":                \
+((tt) == tt_remote)?"REMOTE TABLE":                                                  \
+((tt) == tt_merge_table && ((properties) & PARTITION_LIST) == PARTITION_LIST)?"LIST PARTITION TABLE":                   \
+((tt) == tt_merge_table && ((properties) & PARTITION_RANGE) == PARTITION_RANGE)?"RANGE PARTITION TABLE":"REPLICA TABLE"
+
+#define isTable(x)                        ((x)->type==tt_table)
+#define isView(x)                         ((x)->type==tt_view)
+#define isNonPartitionedTable(x)          ((x)->type==tt_merge_table && !(x)->properties)
+#define isRangePartitionTable(x)          ((x)->type==tt_merge_table && ((x)->properties & PARTITION_RANGE) == PARTITION_RANGE)
+#define isListPartitionTable(x)           ((x)->type==tt_merge_table && ((x)->properties & PARTITION_LIST) == PARTITION_LIST)
+#define isPartitionedByColumnTable(x)     ((x)->type==tt_merge_table && ((x)->properties & PARTITION_COLUMN) == PARTITION_COLUMN)
+#define isPartitionedByExpressionTable(x) ((x)->type==tt_merge_table && ((x)->properties & PARTITION_EXPRESSION) == PARTITION_EXPRESSION)
+#define isMergeTable(x)                   ((x)->type==tt_merge_table)
+#define isRemote(x)                       ((x)->type==tt_remote)
+#define isReplicaTable(x)                 ((x)->type==tt_replica_table)
+#define isKindOfTable(x)                  (isTable(x) || isMergeTable(x) || isRemote(x) || isReplicaTable(x))
+
+#define TABLE_WRITABLE	0
+#define TABLE_READONLY	1
+#define TABLE_APPENDONLY	2
+
+typedef struct sql_part_value {
+	ptr value;
+	size_t length;
+} sql_part_value;
+
+typedef struct sql_part {
+	sql_base base;
+	struct sql_table *t;	/* the merge table */
+	sqlid member;			/* the member of the merge table */
+	bit with_nills;			/* 0 no nills, 1 holds nills, NULL holds all values -> range FROM MINVALUE TO MAXVALUE WITH NULL */
+	union {
+		list *values;       /* partition by values/list */
+		struct sql_range {  /* partition by range */
+			ptr minvalue;
+			ptr maxvalue;
+			size_t minlength;
+			size_t maxlength;
+		} range;
+	} part;
+} sql_part;
+
+typedef struct sql_expression {
+	sql_subtype type; /* the returning sql_subtype of the expression */
+	char *exp;        /* the expression itself */
+	list *cols;       /* list of colnr of the columns of the table used in the expression */
+} sql_expression;
+
+typedef struct sql_table {
+	sql_base base;
+	sht type;		/* table, view, etc */
+	sht access;		/* writable, readonly, appendonly */
+	bit system;		/* system or user table */
+	bit bootstrap;		/* system table created during bootstrap */
+	bte properties;		/* used for merge_tables */
+	temp_t persistence;	/* persistent, global or local temporary */
+	ca_t commit_action;  	/* on commit action */
+	char *query;		/* views may require some query */
+	int  sz;
+
+	sql_ukey *pkey;
+	objlist *columns;
+	objlist *idxs;
+	objlist *keys;
+	objlist *triggers;
+	list *members;		/* member tables of merge/replica tables */
+	int drop_action;	/* only needed for alter drop table */
+
+	ATOMIC_PTR_TYPE data;
+	struct sql_schema *s;
+
+	union {
+		struct sql_column *pcol; /* If it is partitioned on a column */
+		struct sql_expression *pexp; /* If it is partitioned by an expression */
+	} part;
+} sql_table;
+
+typedef struct res_col {
+	char *tn;
+	char *name;
+	sql_subtype type;
+	bat b;
+	char mtype;
+	bool cached;
+	ptr *p;
+} res_col;
+
+typedef struct res_table {
+	int id;
+	oid query_id;
+	mapi_query_t query_type;
+	int nr_cols;
+	BUN nr_rows;
+	BUN cur_row;
+	int cur_col;
+	const char *tsep;
+	const char *rsep;
+	const char *ssep;
+	const char *ns;
+	res_col *cols;
+	bat order;
+	struct res_table *next;
+} res_table;
+
+typedef struct sql_session {
+	sql_allocator *sa;
+	sql_trans *tr; 		/* active transaction */
+
+	char *schema_name; /* transaction's schema name */
+	sql_schema *schema;
+
+	char ac_on_commit;	/* if 1, auto_commit should be enabled on
+	                           commit, rollback, etc. */
+	char auto_commit;
+	int level;		/* TRANSACTION isolation level */
+	int status;		/* status, ok/error */
+	backend_stack stk;
+} sql_session;
+
+#define sql_base_loop(l, n) for (n=l->h; n; n=n->next)
+
+extern int base_key(sql_base *b);
+extern node *list_find_name(list *l, const char *name);
+extern node *list_find_id(list *l, sqlid id);
+extern node *list_find_base_id(list *l, sqlid id);
+
+extern sql_key *find_sql_key(sql_table *t, const char *kname);
+extern sql_key *sql_trans_find_key(sql_trans *tr, sqlid id);
+
+extern sql_idx *find_sql_idx(sql_table *t, const char *kname);
+extern sql_idx *sql_trans_find_idx(sql_trans *tr, sqlid id);
+
+extern sql_column *find_sql_column(sql_table *t, const char *cname);
+
+extern sql_table *find_sql_table(sql_trans *tr, sql_schema *s, const char *tname);
+extern sql_table *find_sql_table_id(sql_trans *tr, sql_schema *s, sqlid id);
+extern sql_table *sql_trans_find_table(sql_trans *tr, sqlid id);
+
+extern sql_sequence *find_sql_sequence(sql_trans *tr, sql_schema *s, const char *sname);
+
+extern sql_schema *find_sql_schema(sql_trans *t, const char *sname);
+extern sql_schema *find_sql_schema_id(sql_trans *t, sqlid id);
+
+extern sql_type *find_sql_type(sql_trans *tr, sql_schema * s, const char *tname);
+extern sql_type *sql_trans_bind_type(sql_trans *tr, sql_schema *s, const char *name);
+extern sql_type *sql_trans_find_type(sql_trans *tr, sql_schema *s /*optional */, sqlid id);
+extern sql_func *sql_trans_find_func(sql_trans *tr, sqlid id);
+extern sql_trigger *sql_trans_find_trigger(sql_trans *tr, sqlid id);
+
+extern void find_partition_type(sql_subtype *tpe, sql_table *mt);
+extern void *sql_values_list_element_validate_and_insert(void *v1, void *v2, void *tpe, int* res);
+extern void *sql_range_part_validate_and_insert(void *v1, void *v2, void *tpe);
+extern void *sql_values_part_validate_and_insert(void *v1, void *v2, void *tpe);
+
+typedef struct {
+	BAT *b;
+	char* name;
+	void* def;
+} sql_emit_col;
+
+extern int nested_mergetable(sql_trans *tr, sql_table *t, const char *sname, const char *tname);
+extern bool is_column_unique(sql_column *c);
+sql_export sql_part *partition_find_part(sql_trans *tr, sql_table *pt, sql_part *pp);
+extern node *members_find_child_id(list *l, sqlid id);
+
+#define outside_str 1
+#define inside_str 2
+
+#define extracting_schema 1
+#define extracting_sequence 2
+
+static inline void
+extract_schema_and_sequence_name(sql_allocator *sa, char *default_value, char **schema, char **sequence)
+{
+	int status = outside_str, identifier = extracting_schema;
+	char next_identifier[1024]; /* needs one extra character for null terminator */
+	size_t bp = 0;
+
+	for (size_t i = 0; default_value[i]; i++) {
+		char next = default_value[i];
+
+		if (next == '"') {
+			if (status == inside_str && default_value[i + 1] == '"') {
+				next_identifier[bp++] = '"';
+				i++; /* has to advance two positions */
+			} else if (status == inside_str) {
+				next_identifier[bp++] = '\0';
+				if (identifier == extracting_schema) {
+					*schema = SA_STRDUP(sa, next_identifier);
+					identifier = extracting_sequence;
+				} else if (identifier == extracting_sequence) {
+					*sequence = SA_STRDUP(sa, next_identifier);
+					break; /* done extracting */
+				}
+				bp = 0;
+				status = outside_str;
+			} else {
+				assert(status == outside_str);
+				status = inside_str;
+			}
+		} else if (next == '.') {
+			if (status == outside_str && default_value[i + 1] == '"') {
+				status = inside_str;
+				i++; /* has to advance two positions */
+			} else {
+				assert(status == inside_str);
+				next_identifier[bp++] = '.'; /* used inside an identifier name */
+			}
+		} else if (status == inside_str) {
+			next_identifier[bp++] = next;
+		} else {
+			assert(status == outside_str);
+		}
+	}
+}
+
+extern void arg_destroy(sql_store store, sql_arg *a);
+extern void part_value_destroy(sql_store store, sql_part_value *pv);
+
+typedef struct atom {
+	int isnull;
+	sql_subtype tpe;
+	ValRecord data;
+} atom;
+
+/* duplicate atom */
+extern ValPtr SA_VALcopy(sql_allocator *sa, ValPtr d, const ValRecord *s);
+extern atom *atom_copy(sql_allocator *sa, atom *a);
+
+typedef struct pl {
+	sql_column *c;
+	unsigned int cmp;
+	atom *r; /* if r is NULL then a full match is required */
+	atom *f; /* make it match range expressions */
+	uint8_t
+	 anti:1,
+	 semantics:1;
+} pl;
+
+#endif /* SQL_CATALOG_H */

monetdb/msvc/sql_hash.h

@@ -0,0 +1,69 @@
+/*
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0.  If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright 1997 - July 2008 CWI, August 2008 - 2022 MonetDB B.V.
+ */
+
+#ifndef SQL_HASH_H
+#define SQL_HASH_H
+
+/* sql_hash implementation
+ * used to optimize search in expression and statement lists
+ */
+
+#include "sql_mem.h"
+
+#define HASH_MIN_SIZE 4
+
+typedef int (*fkeyvalue) (void *data);
+
+typedef struct sql_hash_e {
+	int key;
+	void *value;
+	struct sql_hash_e *chain;
+} sql_hash_e;
+
+typedef struct sql_hash {
+	sql_allocator *sa;
+	int size; /* power of 2 */
+	sql_hash_e **buckets;
+	fkeyvalue key;
+	int entries;
+} sql_hash;
+
+extern sql_hash *hash_new(sql_allocator *sa, int size, fkeyvalue key);
+extern int hash_entries(sql_hash *h);
+extern int hash_empty(sql_hash *h);
+extern void hash_del(sql_hash *ht, int key, void *value);
+extern void hash_clear(sql_hash *h);
+extern void hash_destroy(sql_hash *h);
+
+static inline sql_hash_e*
+hash_add(sql_hash *h, int key, void *value)
+{
+	sql_hash_e *e = (h->sa)?SA_NEW(h->sa, sql_hash_e):MNEW(sql_hash_e);
+
+	if (e == NULL)
+		return NULL;
+	e->chain = h->buckets[key&(h->size-1)];
+	h->buckets[key&(h->size-1)] = e;
+	e->key = key;
+	e->value = value;
+	h->entries++;
+	return e;
+}
+
+static inline unsigned int
+hash_key(const char *restrict k)
+{
+	unsigned int h = 37; /* prime number */
+	while (*k) {
+		h = (h * 54059) ^ (k[0] * 76963); /* prime numbers */
+		k++;
+	}
+	return h;
+}
+
+#endif /* SQL_HASH_H */

monetdb/msvc/sql_keyword.h

@@ -0,0 +1,32 @@
+/*
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0.  If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright 1997 - July 2008 CWI, August 2008 - 2022 MonetDB B.V.
+ */
+
+#ifndef SQL_KEYWORD_H
+#define SQL_KEYWORD_H
+
+/* we need to define these here as the parser header file is generated to late.
+ * The numbers get remapped in the scanner.
+ */
+#define KW_ALIAS 4000
+#define KW_TYPE  4001
+
+typedef struct keyword {
+	char *keyword;
+	int len;
+	int token;
+	struct keyword *next;
+} keyword;
+
+extern int keywords_insert(char *k, int token);
+extern keyword *find_keyword(char *text);
+extern int keyword_exists(char *text);
+
+extern void keyword_init(void);
+extern void keyword_exit(void);
+
+#endif /* SQL_KEYWORD_H */

monetdb/msvc/sql_list.h

@@ -0,0 +1,109 @@
+/*
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0.  If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright 1997 - July 2008 CWI, August 2008 - 2022 MonetDB B.V.
+ */
+
+#ifndef LIST_H
+#define LIST_H
+
+#include "sql_mem.h"
+#include "sql_hash.h"
+
+#define SA_LIST(sa, destroy) ((sa)?sa_list(sa):list_create(destroy))
+
+typedef struct node {
+	struct node *next;
+	void *data;
+} node;
+
+typedef void (*fdestroy) (void *gdata, void *ndata); /* gdata is passed to the list_destroy2 function */
+
+typedef struct list {
+	sql_allocator *sa;
+	sql_hash *ht;
+	MT_Lock ht_lock;	/* latch protecting ht */
+	fdestroy destroy;
+	node *h;
+	node *t;
+	int cnt;
+	int expected_cnt;
+} list;
+
+typedef int (*traverse_func) (void *clientdata, int seqnr, void *data);
+typedef int (*prop_check_func) (void *data);
+
+extern list *list_create(fdestroy destroy);
+sql_export list *sa_list(sql_allocator *sa);
+extern list *list_new(sql_allocator *sa, fdestroy destroy);
+
+extern list *sa_list_append( sql_allocator *sa, list *l, void *data);
+
+extern void list_destroy(list *l);
+extern void list_destroy2(list *l, void *data);
+sql_export int list_length(list *l);
+extern int list_empty(list *l);
+
+sql_export list *list_append(list *l, void *data);
+extern list *list_append_before(list *l, node *n, void *data);
+extern list *list_prepend(list *l, void *data);
+
+extern node *list_remove_node(list *l, void *gdata, node *n);
+extern void list_remove_data(list *l, void *gdata, void *data);
+extern void list_remove_list(list *l, void *gdata, list *data);
+extern void list_move_data(list *l, list *d, void *data);
+
+
+extern int list_traverse(list *l, traverse_func f, void *clientdata);
+
+extern int list_check_prop_all(list *l, prop_check_func f);
+
+/* the compare function gets one element from the list and a key from the
+ * as input from the find function
+ * Returns 0 if data and key are equal
+ * */
+typedef int (*fcmp) (void *data, void *key);
+typedef void *(*fcmpvalidate) (void *v1, void *v2, void *extra, int *cmp);
+typedef void *(*fvalidate) (void *v1, void *v2, void *extra);
+typedef int (*fcmp2) (void *data, void *v1, void *v2);
+typedef void *(*fdup) (void *data);
+typedef void *(*freduce) (void *v1, void *v2);
+typedef void *(*freduce2) (sql_allocator *sa, void *v1, void *v2);
+typedef void *(*fmap) (void *data, void *clientdata);
+
+extern void *list_transverse_with_validate(list *l, void *data, void *extra, fvalidate cmp);
+extern void *list_append_with_validate(list *l, void *data, void *extra, fvalidate cmp);
+extern void *list_append_sorted(list *l, void *data, void *extra, fcmpvalidate cmp);
+extern node *list_find(list *l, void *key, fcmp cmp);
+extern int  list_position(list *l, void *val);
+sql_export void *list_fetch(list *l, int pos);
+extern list *list_select(list *l, void *key, fcmp cmp, fdup dup);
+extern list *list_order(list *l, fcmp cmp, fdup dup);
+extern list *list_distinct(list *l, fcmp cmp, fdup dup);
+extern void *list_reduce(list *l, freduce red, fdup dup);
+extern void *list_reduce2(list *l, freduce2 red, sql_allocator *sa);
+extern list *list_map(list *l, void *data, fmap f);
+extern int list_cmp(list *l1, list *l2, fcmp cmp);
+/* cmp the lists in link order */
+extern int list_match(list *l1, list *l2, fcmp cmp);
+/* match the lists (in any order) */
+extern list *list_sort(list *l, fkeyvalue key, fdup dup);
+/* The sort function sorts the list using the key function, which
+ * translates the list item values into integer keyvalues. */
+/* sometimes more complex functions are needed to compute a key, then
+ * we can pass the keys via an array, to keysort */
+extern list *list_keysort(list *l, int *key, fdup dup);
+
+extern list *list_dup(list *l, fdup dup);
+extern list *list_merge(list *l, list *data, fdup dup);
+extern list *list_merge_destroy(list *l, list *data, fdup dup);
+
+extern list *list_flaten(list *l);
+
+extern void list_hash_delete(list *l, void *data, fcmp cmp);
+extern void* list_hash_add(list *l, void *data, fcmp cmp);
+extern void list_hash_clear(list *l);
+
+#endif /* LIST_H */

monetdb/msvc/sql_mem.h

@@ -0,0 +1,135 @@
+/*
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0.  If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright 1997 - July 2008 CWI, August 2008 - 2022 MonetDB B.V.
+ */
+
+#ifndef _SQL_MEM_H_
+#define _SQL_MEM_H_
+
+#include "gdk.h"
+#include "exception_buffer.h"
+
+#define SQL_OK 	1
+#define SQL_ERR 0
+#define SQL_CONFLICT 2
+
+#ifdef WIN32
+#ifdef LIBSQL
+#define sql_export extern __declspec(dllexport)
+#else
+#define sql_export extern __declspec(dllimport)
+#endif
+#else
+#define sql_export extern
+#endif
+
+#define MNEW( type ) (type*)GDKmalloc(sizeof(type) )
+#define ZNEW( type ) (type*)GDKzalloc(sizeof(type) )
+#define NEW_ARRAY( type, size ) (type*)GDKmalloc((size)*sizeof(type))
+#define ZNEW_ARRAY( type, size ) (type*)GDKzalloc((size)*sizeof(type))
+#define RENEW_ARRAY( type,ptr,size) (type*)GDKrealloc((void*)ptr,(size)*sizeof(type))
+
+#define _DELETE( ptr )	do { GDKfree(ptr); ptr = NULL; } while (0)
+#define _STRDUP( ptr )	GDKstrdup((char*)ptr)
+
+extern void c_delete( const void *p );
+
+typedef struct sql_ref {
+	int refcnt;
+} sql_ref;
+
+extern sql_ref *sql_ref_init(sql_ref *r);
+extern int sql_ref_inc(sql_ref *r);
+extern int sql_ref_dec(sql_ref *r);
+
+typedef struct sql_allocator {
+	struct sql_allocator *pa;
+	size_t size;
+	size_t nr;
+	char **blks;
+	size_t used; 	/* memory used in last block */
+	size_t usedmem;	/* used memory */
+	void *freelist;	/* list of freed blocks */
+	exception_buffer eb;
+} sql_allocator;
+
+sql_export sql_allocator *sa_create( sql_allocator *pa );
+sql_export sql_allocator *sa_reset( sql_allocator *sa );
+extern void *sa_alloc( sql_allocator *sa,  size_t sz );
+sql_export void *sa_zalloc( sql_allocator *sa,  size_t sz );
+extern void *sa_realloc( sql_allocator *sa,  void *ptr, size_t sz, size_t osz );
+extern void sa_destroy( sql_allocator *sa );
+extern char *sa_strndup( sql_allocator *sa, const char *s, size_t l);
+sql_export char *sa_strdup( sql_allocator *sa, const char *s);
+extern char *sa_strconcat( sql_allocator *sa, const char *s1, const char *s2);
+extern size_t sa_size( sql_allocator *sa );
+
+#define SA_NEW( sa, type ) (sa?((type*)sa_alloc( sa, sizeof(type))):MNEW(type))
+#define SA_ZNEW( sa, type ) (sa?((type*)sa_zalloc( sa, sizeof(type))):ZNEW(type))
+#define SA_NEW_ARRAY( sa, type, size ) (sa?(type*)sa_alloc( sa, ((size)*sizeof(type))):NEW_ARRAY(type,size))
+#define SA_ZNEW_ARRAY( sa, type, size ) (type*)sa_zalloc( sa, ((size)*sizeof(type)))
+#define SA_RENEW_ARRAY( sa, type, ptr, sz, osz ) (sa?(type*)sa_realloc( sa, ptr, ((sz)*sizeof(type)), ((osz)*sizeof(type))):RENEW_ARRAY(type,ptr,sz))
+#define SA_STRDUP( sa, s) (sa?sa_strdup(sa, s):_STRDUP(s))
+
+#define _strlen(s) (int)strlen(s)
+
+#if !defined(NDEBUG) && !defined(__COVERITY__) && defined(__GNUC__)
+#define sa_alloc(sa, sz)					\
+	({							\
+		sql_allocator *_sa = (sa);			\
+		size_t _sz = (sz);				\
+		void *_res = sa_alloc(_sa, _sz);		\
+		TRC_DEBUG(ALLOC,				\
+				"sa_alloc(%p,%zu) -> %p\n",	\
+				_sa, _sz, _res);	\
+		_res;						\
+	})
+#define sa_zalloc(sa, sz)					\
+	({							\
+		sql_allocator *_sa = (sa);			\
+		size_t _sz = (sz);				\
+		void *_res = sa_zalloc(_sa, _sz);		\
+		TRC_DEBUG(ALLOC,				\
+				"sa_zalloc(%p,%zu) -> %p\n",	\
+				_sa, _sz, _res);	\
+		_res;						\
+	})
+#define sa_realloc(sa, ptr, sz, osz)					\
+	({								\
+		sql_allocator *_sa = (sa);				\
+		void *_ptr = (ptr);					\
+		size_t _sz = (sz);					\
+		size_t _osz = (osz);					\
+		void *_res = sa_realloc(_sa, _ptr, _sz, _osz);		\
+		TRC_DEBUG(ALLOC,				\
+				"sa_realloc(%p,%p,%zu,%zu) -> %p\n",	\
+				_sa, _ptr, _sz, _osz, _res);		\
+		_res;							\
+	})
+#define sa_strdup(sa, s)					\
+	({							\
+		sql_allocator *_sa = (sa);			\
+		const char *_s = (s);				\
+		char *_res = sa_strdup(_sa, _s);		\
+		TRC_DEBUG(ALLOC,				\
+				"sa_strdup(%p,len=%zu) -> %p\n",	\
+				_sa, strlen(_s), _res);	\
+		_res;						\
+	})
+#define sa_strndup(sa, s, l)					\
+	({							\
+		sql_allocator *_sa = (sa);			\
+		const char *_s = (s);				\
+		size_t _l = (l);				\
+		char *_res = sa_strndup(_sa, _s, _l);		\
+		TRC_DEBUG(ALLOC,				\
+				"sa_strndup(%p,len=%zu) -> %p\n", 	\
+				_sa, _l, _res);		\
+		_res;						\
+	})
+#endif
+
+#endif /*_SQL_MEM_H_*/

monetdb/msvc/sql_mvc.h

@@ -0,0 +1,319 @@
+/*
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0.  If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright 1997 - July 2008 CWI, August 2008 - 2022 MonetDB B.V.
+ */
+
+/* multi version catalog */
+#ifndef _SQL_MVC_H
+#define _SQL_MVC_H
+
+#include "sql_mem.h"
+#include "gdk.h"
+#include "sql_scan.h"
+#include "sql_list.h"
+#include "sql_types.h"
+#include "sql_backend.h"
+#include "sql_catalog.h"
+#include "sql_relation.h"
+#include "sql_storage.h"
+#include "sql_keyword.h"
+#include "mapi_querytype.h"
+#include "sql_atom.h"
+#include "sql_tokens.h"
+#include "sql_symbol.h"
+
+#define ERRSIZE 8192
+#define ERR_AMBIGUOUS	050000
+#define ERR_GROUPBY		060000
+#define ERR_NOTFOUND	070000
+
+/* value vs predicate (boolean) */
+#define type_value	0
+#define type_relation	1
+
+/* cardinality expected by enclosing operator */
+#define card_none	-1	/* psm call doesn't return anything */
+#define card_value	0
+#define card_row 	1 /* needed for subqueries on single value tables (select (select 1))*/
+#define card_column 	2
+#define card_set	3 /* some operators require only a set (IN/EXISTS) */
+#define card_exists	4
+/* to be removed ie are in type (aka dimention) */
+#define card_relation 	5
+#define card_loader 	6
+
+#define CARD_VALUE(card) (card == card_value || card == card_row || card == card_column || card == card_set || card == card_exists)
+
+/* allowed to reduce (in the where and having parts we can reduce) */
+
+/* different query execution modes (emode) */
+#define m_normal 	0
+#define m_deallocate 1
+#define m_prepare 	2
+#define m_plan 		3
+
+/* special modes for function/procedure and view instantiation and
+   dependency generation */
+#define m_instantiate 	5
+#define m_deps 		6
+
+/* different query execution modifiers (emod) */
+#define mod_none 	0
+#define mod_debug 	1
+#define mod_trace 	2
+#define mod_explain 	4
+#define mod_exec 	8
+
+typedef struct sql_groupby_expression {
+	symbol *sdef;
+	tokens token;
+	sql_exp *exp;
+} sql_groupby_expression;
+
+typedef struct sql_window_definition {
+	char *name;
+	dlist *wdef;
+	bool visited; /* used for window definitions lookup */
+} sql_window_definition;
+
+typedef struct sql_local_table { /* declared tables during session */
+	sql_table *table;
+} sql_local_table;
+
+typedef struct sql_rel_view { /* CTEs */
+	char *name;
+	sql_rel *rel_view;
+} sql_rel_view;
+
+typedef struct sql_var { /* Declared variables and parameters */
+	char *sname; /* Global variables have a schema */
+	char *name;
+	atom var;
+} sql_var;
+
+typedef struct sql_frame {
+	char *name; /* frame name */
+	list *group_expressions;
+	list *windows;
+	list *tables;
+	list *rel_views;
+	list *vars;
+	int frame_number;
+} sql_frame;
+
+typedef struct mvc {
+	char errstr[ERRSIZE];
+
+	sql_allocator *sa, *ta, *pa;
+
+	struct scanner scanner;
+
+	list *params;
+	sqlid objid;	/* when replacing an existing view, it can't be seen */
+	sql_func *forward;	/* forward definitions for recursive functions */
+	list *global_vars; /* SQL declared variables on the global scope */
+	sql_frame **frames;	/* stack of frames with variables */
+	int topframes;
+	int sizeframes;
+	int frame;
+	struct symbol *sym;
+
+	int8_t use_views:1,
+		   schema_path_has_sys:1, /* speed up object search */
+		   schema_path_has_tmp:1;
+	struct qc *qc;
+	int clientid;		/* id of the owner */
+
+	/* session variables */
+	sqlid user_id;
+	sqlid role_id;
+	int timezone;		/* milliseconds west of UTC */
+	int reply_size;		/* reply size */
+	int debug;
+
+	char emode;		/* execution mode */
+	char emod;		/* execution modifier */
+
+	sql_session *session;
+	sql_store store;
+
+	/* per query context */
+	mapi_query_t type;	/* query type */
+
+	/* during query needed flags */
+	unsigned int label;	/* numbers for relational projection labels */
+	list *cascade_action;  /* protection against recursive cascade actions */
+	list *schema_path; /* schema search path for object lookup */
+	uintptr_t sp;
+} mvc;
+
+extern sql_table *mvc_init_create_view(mvc *sql, sql_schema *s, const char *name, const char *query);
+
+/* should return structure */
+extern sql_store mvc_init(int debug, store_type store, int ro, int su);
+extern void mvc_exit(sql_store store);
+
+extern void mvc_logmanager(sql_store store);
+
+extern mvc *mvc_create(sql_store *store, sql_allocator *pa, int clientid, int debug, bstream *rs, stream *ws);
+extern int mvc_reset(mvc *m, bstream *rs, stream *ws, int debug);
+extern void mvc_destroy(mvc *c);
+
+extern int mvc_status(mvc *c);
+extern int mvc_error_retry(mvc *c); // error code on errors else 0, errors AMBIGUOUS and GROUPBY will also output 0
+extern int mvc_type(mvc *c);
+extern int mvc_debug_on(mvc *m, int flag);
+extern void mvc_cancel_session(mvc *m);
+
+/* since Savepoints and transactions are related the
+ * commit function includes the savepoint creation.
+ * Rollbacks can be either full or until a given savepoint.
+ * The special mvc_release can be used to release savepoints.
+ */
+#define has_snapshots(tr) ((tr) && (tr)->parent && (tr)->parent->parent)
+
+extern int mvc_trans(mvc *c);
+sql_export str mvc_commit(mvc *c, int chain, const char *name, bool enabling_auto_commit);
+sql_export str mvc_rollback(mvc *c, int chain, const char *name, bool disabling_auto_commit);
+extern str mvc_release(mvc *c, const char *name);
+
+extern sql_type *mvc_bind_type(mvc *sql, const char *name);
+extern sql_type *schema_bind_type(mvc *sql, sql_schema * s, const char *name);
+
+sql_export sql_schema *mvc_bind_schema(mvc *c, const char *sname);
+sql_export sql_table *mvc_bind_table(mvc *c, sql_schema *s, const char *tname);
+extern sql_column *mvc_bind_column(mvc *c, sql_table *t, const char *cname);
+extern sql_column *mvc_first_column(mvc *c, sql_table *t);
+extern sql_idx *mvc_bind_idx(mvc *c, sql_schema *s, const char *iname);
+extern sql_key *mvc_bind_key(mvc *c, sql_schema *s, const char *kname);
+extern sql_key *mvc_bind_ukey(sql_table *t, list *cols);
+extern sql_trigger *mvc_bind_trigger(mvc *c, sql_schema *s, const char *tname);
+
+extern int mvc_create_type(mvc *sql, sql_schema *s, const char *sqlname, unsigned int digits, unsigned int scale, int radix, const char *impl);
+extern int mvc_drop_type(mvc *sql, sql_schema *s, sql_type *t, int drop_action);
+
+extern int mvc_create_func(sql_func **f, mvc *m, sql_allocator *sa, sql_schema *s, const char *name, list *args, list *res, sql_ftype type, sql_flang lang,
+						   const char *mod, const char *impl, const char *query, bit varres, bit vararg, bit system, bit side_effect);
+extern int mvc_drop_func(mvc *c, sql_schema *s, sql_func * func, int drop_action);
+extern int mvc_drop_all_func(mvc *c, sql_schema *s, list *list_func, int drop_action);
+
+extern int mvc_drop_schema(mvc *c, sql_schema *s, int drop_action);
+extern int mvc_create_schema(mvc *m, const char *name, sqlid auth_id, sqlid owner);
+extern BUN mvc_clear_table(mvc *m, sql_table *t);
+extern str mvc_drop_table(mvc *c, sql_schema *s, sql_table * t, int drop_action);
+extern int mvc_create_table(sql_table **t, mvc *m, sql_schema *s, const char *name, int tt, bit system, int persistence, int commit_action, int sz, bit properties);
+extern int mvc_create_view(sql_table **t, mvc *m, sql_schema *s, const char *name, int persistence, const char *sql, bit system);
+extern int mvc_create_remote(sql_table **t, mvc *m, sql_schema *s, const char *name, int persistence, const char *loc);
+
+extern int mvc_drop_column(mvc *c, sql_table *t, sql_column *col, int drop_action);
+sql_export int mvc_create_column(sql_column **col, mvc *m, sql_table *t, const char *name, sql_subtype *tpe);
+extern int mvc_create_column_(sql_column **col, mvc *m, sql_table *t, const char *name, const char *type, unsigned int digits);
+extern int mvc_null(mvc *c, sql_column *col, int flag);
+extern int mvc_default(mvc *c, sql_column *col, char *val);
+extern int mvc_drop_default(mvc *c, sql_column *col);
+extern int mvc_storage(mvc *c, sql_column *col, char *storage);
+extern int mvc_access(mvc *m, sql_table *t, sht access);
+extern int mvc_is_sorted(mvc *c, sql_column *col);
+extern int mvc_is_unique(mvc *m, sql_column *col);
+extern int mvc_is_duplicate_eliminated(mvc *c, sql_column *col);
+
+extern int mvc_create_ukey(sql_key **kres, mvc *m, sql_table *t, const char *name, key_type kt);
+extern int mvc_create_fkey(sql_fkey **kres, mvc *m, sql_table *t, const char *name, key_type kt, sql_key *rkey, int on_delete, int on_update);
+extern int mvc_create_kc(mvc *m, sql_key *k, sql_column *c);
+extern int mvc_create_fkc(mvc *m, sql_fkey *fk, sql_column *c);
+extern int mvc_create_key_done(mvc *m, sql_key *k);
+
+extern int mvc_drop_key(mvc *c, sql_schema *s, sql_key *key, int drop_action);
+
+extern int mvc_create_idx(sql_idx **i, mvc *m, sql_table *t, const char *name, idx_type it);
+extern int mvc_create_ic(mvc *m, sql_idx * i, sql_column *c);
+extern int mvc_create_idx_done(mvc *m, sql_idx *i);
+extern int mvc_drop_idx(mvc *c, sql_schema *s, sql_idx * i);
+
+extern int mvc_create_trigger(sql_trigger **tri, mvc *m, sql_table *t, const char *name, sht time, sht orientation, sht event, const char *old_name, const char *new_name, const char *condition, const char *statement);
+extern int mvc_drop_trigger(mvc *m, sql_schema *s, sql_trigger * tri);
+
+/*dependency control*/
+extern int mvc_create_dependency(mvc *m, sql_base *b, sqlid depend_id, sql_dependency depend_type);
+extern int mvc_create_dependencies(mvc *m, list *blist, sqlid depend_id, sql_dependency dep_type);
+extern int mvc_check_dependency(mvc *m, sqlid id, sql_dependency type, list *ignore_ids);
+
+/* variable management */
+extern int init_global_variables(mvc *sql);
+extern sql_var *find_global_var(mvc *sql, sql_schema *s, const char *name);
+extern sql_var *push_global_var(mvc *sql, const char *sname, const char *name, sql_subtype *type);
+extern int remove_global_var(mvc *sql, sql_schema *s, const char *name);
+
+extern sql_var* frame_push_var(mvc *sql, const char *name, sql_subtype *type);
+extern sql_local_table* frame_push_table(mvc *sql, sql_table *t);
+extern sql_rel_view* stack_push_rel_view(mvc *sql, const char *name, sql_rel *var);
+extern sql_window_definition* frame_push_window_def(mvc *sql, const char *name, dlist *wdef);
+extern dlist* frame_get_window_def(mvc *sql, const char *name, int *pos);
+extern sql_groupby_expression* frame_push_groupby_expression(mvc *sql, symbol *def, sql_exp *exp);
+extern sql_exp* frame_get_groupby_expression(mvc *sql, symbol *def);
+extern void stack_update_rel_view(mvc *sql, const char *name, sql_rel *view);
+
+extern bool frame_check_var_visited(mvc *sql, int i);
+extern void frame_set_var_visited(mvc *sql, int i);
+extern void frame_clear_visited_flag(mvc *sql);
+
+extern sql_frame *stack_push_frame(mvc *sql, const char *name);
+extern void stack_pop_frame(mvc *sql);
+extern void clear_frame(mvc *sql, sql_frame *frame);
+extern void stack_pop_until(mvc *sql, int frame);
+
+/* find variable in the stack */
+extern sql_var *stack_find_var_frame(mvc *sql, const char *name, int *level);
+extern sql_var *stack_find_var_at_level(mvc *sql, const char *name, int level);
+extern sql_table *stack_find_table(mvc *sql, const char *name);
+extern sql_table *frame_find_table(mvc *sq, const char *name);
+extern sql_rel *stack_find_rel_view(mvc *sql, const char *name);
+extern int stack_find_rel_view_projection_columns(mvc *sql, const char *name, sql_rel **res);
+
+/* find variable in the current frame */
+extern int frame_find_var(mvc *sql, const char *name);
+extern sql_rel *frame_find_rel_view(mvc *sql, const char *name);
+
+extern int stack_has_frame(mvc *sql, const char *name);
+extern int stack_nr_of_declared_tables(mvc *sql);
+
+extern atom *sqlvar_set(sql_var *var, ValRecord *v);
+extern str sqlvar_get_string(sql_var *var);
+extern str sqlvar_set_string(sql_var *var, const char *v);
+#ifdef HAVE_HGE
+extern hge val_get_number(ValRecord *val);
+extern void sqlvar_set_number(sql_var *var, hge v);
+#else
+extern lng val_get_number(ValRecord *val);
+extern void sqlvar_set_number(sql_var *var, lng v);
+#endif
+
+#define get_string_global_var(m, val) (sqlvar_get_string(find_global_var(m, mvc_bind_schema(m, "sys"), val)))
+
+extern int mvc_copy_column(mvc *m, sql_table *t, sql_column *c, sql_column **cres);
+extern int mvc_copy_key(mvc *m, sql_table *t, sql_key *k, sql_key **kres);
+extern int mvc_copy_idx(mvc *m, sql_table *t, sql_idx *i, sql_idx **ires);
+extern int mvc_copy_trigger(mvc *m, sql_table *t, sql_trigger *tr, sql_trigger **tres);
+
+extern sql_rel *sql_processrelation(mvc *sql, sql_rel *rel, int instantiate, int value_based_opt, int storage_based_opt);
+
+extern void *sql_error(mvc *sql, int error_code, _In_z_ _Printf_format_string_ char *format, ...)
+	__attribute__((__format__(__printf__, 3, 4)));
+
+extern int symbol_cmp(mvc* sql, symbol *s1, symbol *s2);
+
+static inline int mvc_highwater(mvc *sql)
+{
+	int l = 0, rc = 0;
+	uintptr_t c = (uintptr_t) (&l);
+
+	size_t diff = c < sql->sp ? sql->sp - c : c - sql->sp;
+	if (diff > THREAD_STACK_SIZE - 280 * 1024)
+		rc = 1;
+	return rc;
+}
+
+#endif /*_SQL_MVC_H*/

monetdb/msvc/sql_parser.h

@@ -0,0 +1,29 @@
+/*
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0.  If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright 1997 - July 2008 CWI, August 2008 - 2022 MonetDB B.V.
+ */
+
+#ifndef _SQL_PARSER_H_
+#define _SQL_PARSER_H_
+
+#include "sql_tokens.h"
+#include "sql_mvc.h"
+
+/* the next define makes the parser output more specific error messages,
+ * instead of only a dull 'parse error' */
+#define YYERROR_VERBOSE 1
+/* enable this to get an idea of what the parser is doing on stdout
+#define YYDEBUG 1
+*/
+
+extern int find_subgeometry_type(mvc *m, char*);
+extern char *token2string(tokens token);
+extern int sqlparse(mvc *m);
+
+#include "mal_errors.h"		/* for SQLSTATE() */
+
+#endif /*_SQL_PARSER_H_*/
+

monetdb/msvc/sql_relation.h

@@ -0,0 +1,301 @@
+/*
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0.  If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright 1997 - July 2008 CWI, August 2008 - 2022 MonetDB B.V.
+ */
+
+#ifndef SQL_RELATION_H
+#define SQL_RELATION_H
+
+#include "sql_catalog.h"
+
+typedef enum expression_type {
+	e_atom,
+	e_column,
+	e_cmp,
+	e_func,
+	e_aggr,
+	e_convert,
+	e_psm
+} expression_type;
+
+#define CARD_ATOM 1
+#define CARD_AGGR 2
+#define CARD_MULTI 3
+
+typedef struct sql_exp_name {
+	unsigned int label;
+	const char *name;
+	const char *rname;
+} sql_exp_name;
+
+typedef struct sql_var_name {
+	const char *name;
+	const char *sname;
+} sql_var_name;
+
+typedef struct expression {
+	expression_type type;	/* atom, cmp, func/aggr */
+	sql_exp_name alias;
+	void *l;
+	void *r;
+	void *f;	/* func's and aggr's, also e_cmp may have have 2 arguments */
+	unsigned int flag; /* cmp types, PSM types/level */
+	unsigned int
+	 card:2,	/* card (0 truth value!) (1 atoms) (2 aggr) (3 multi value) */
+	 freevar:4,	/* free variable, ie binds to the upper dependent join */
+	 intern:1,
+	 anti:1,
+	 ascending:1,	/* order direction */
+	 nulls_last:1,	/* return null after all other rows */
+	 zero_if_empty:1, 	/* in case of partial aggregator computation, some aggregators need to return 0 instead of NULL */
+	 distinct:1,
+	 argument_independence:1, /* for a function expression, the arguments are independent of the inner project relation. */
+
+	 semantics:1,	/* is vs = semantics (nil = nil vs unknown != unknown), ranges with or without nil, aggregation with or without nil */
+	 need_no_nil:1,
+	 has_no_nil:1,
+	 unique:1,	/* expression has unique values, but it may have multiple NULL values! */
+
+	 base:1,
+	 ref:1,		/* used to indicate an other expression may reference this one */
+	 used:1,	/* used for quick dead code removal */
+	 symmetric:1; /* compare between symmetric */
+	sql_subtype	tpe;
+	void *p;	/* properties for the optimizer */
+} sql_exp;
+
+#define TABLE_PROD_FUNC		1
+#define TABLE_FROM_RELATION	2
+#define TRIGGER_WRAPPER		4
+
+#define IS_TABLE_PROD_FUNC(X)  ((X & TABLE_PROD_FUNC) == TABLE_PROD_FUNC)
+
+/* or-ed with the above TABLE_PROD_FUNC */
+#define UPD_COMP		2
+
+#define LEFT_JOIN		4
+#define REL_PARTITION		8
+#define MERGE_LEFT		16 /* used by merge statements */
+#define OUTER_ZERO		32
+
+/* We need bit wise exclusive numbers as we merge the level also in the flag */
+#define PSM_SET 1
+#define PSM_VAR 2
+#define PSM_RETURN 4
+#define PSM_WHILE 8
+#define PSM_IF 16
+#define PSM_REL 32
+#define PSM_EXCEPTION 64
+
+#define SET_PSM_LEVEL(level)	(level<<8)
+#define GET_PSM_LEVEL(level)	(level>>8)
+
+typedef enum ddl_statement {
+	ddl_output,
+	ddl_list,
+	ddl_psm,
+	ddl_exception,
+	ddl_create_seq,
+	ddl_alter_seq,
+	ddl_drop_seq,
+	ddl_alter_table_add_range_partition,
+	ddl_alter_table_add_list_partition,
+	ddl_release,
+	ddl_commit,
+	ddl_rollback,
+	ddl_trans,
+	ddl_create_schema,
+	ddl_drop_schema,
+	ddl_create_table,
+	ddl_drop_table,
+	ddl_create_view,
+	ddl_drop_view,
+	ddl_drop_constraint,
+	ddl_alter_table,
+	ddl_create_type,
+	ddl_drop_type,
+	ddl_drop_index,
+	ddl_create_function,
+	ddl_drop_function,
+	ddl_create_trigger,
+	ddl_drop_trigger,
+	ddl_grant_roles,
+	ddl_revoke_roles,
+	ddl_grant,
+	ddl_revoke,
+	ddl_grant_func,
+	ddl_revoke_func,
+	ddl_create_user,
+	ddl_drop_user,
+	ddl_alter_user,
+	ddl_rename_user,
+	ddl_create_role,
+	ddl_drop_role,
+	ddl_alter_table_add_table,
+	ddl_alter_table_del_table,
+	ddl_alter_table_set_access,
+	ddl_comment_on,
+	ddl_rename_schema,
+	ddl_rename_table,
+	ddl_rename_column,
+	ddl_maxops /* evaluated to the max value, should be always kept at the bottom */
+} ddl_statement;
+
+typedef enum operator_type {
+	op_basetable = 0,
+	op_table,
+	op_ddl,
+	op_project,		/* includes order by */
+	op_select,
+	op_join,
+	op_left,
+	op_right,
+	op_full,
+	op_semi,
+	op_anti,
+	op_union,
+	op_inter,
+	op_except,
+	op_groupby,
+	op_topn,
+	op_sample,
+	op_insert,	/* insert(l=table, r insert expressions) */
+	op_update,	/* update(l=table, r update expressions) */
+	op_delete,	/* delete(l=table, r delete expression) */
+	op_truncate, /* truncate(l=table) */
+	op_merge
+} operator_type;
+
+#define is_atom(et) 		(et == e_atom)
+/* a simple atom is a literal or on the query stack */
+#define is_simple_atom(e) 	(is_atom((e)->type) && !(e)->r && !(e)->f)
+#define is_values(e) 		((e)->type == e_atom && (e)->f)
+#define is_func(et) 		(et == e_func)
+#define is_aggr(et) 		(et == e_aggr)
+#define is_convert(et) 		(et == e_convert)
+#define is_map_op(et) 		(et == e_func || et == e_convert)
+#define is_compare(et) 		(et == e_cmp)
+#define is_column(et)		(et != e_cmp)
+#define is_alias(et) 		(et == e_column)
+#define is_analytic(e) 		((e)->type == e_func && ((sql_subfunc*)(e)->f)->func->type == F_ANALYTIC)
+
+#define is_base(op)  		(op == op_basetable || op == op_table)
+#define is_basetable(op) 	(op == op_basetable)
+#define is_ddl(op)	 		(op == op_ddl)
+#define is_outerjoin(op) 	(op == op_left || op == op_right || op == op_full)
+#define is_left(op) 		(op == op_left)
+#define is_right(op) 		(op == op_right)
+#define is_full(op) 		(op == op_full)
+#define is_innerjoin(op)	(op == op_join)
+#define is_join(op) 		(op == op_join || is_outerjoin(op))
+#define is_semi(op) 		(op == op_semi || op == op_anti)
+#define is_joinop(op) 		(is_join(op) || is_semi(op))
+#define is_select(op) 		(op == op_select)
+#define is_set(op) 			(op == op_union || op == op_inter || op == op_except)
+#define is_union(op) 		(op == op_union)
+#define is_inter(op) 		(op == op_inter)
+#define is_except(op) 		(op == op_except)
+#define is_simple_project(op) 	(op == op_project)
+#define is_project(op) 		(op == op_project || op == op_groupby || is_set(op))
+#define is_groupby(op) 		(op == op_groupby)
+#define is_topn(op) 		(op == op_topn)
+#define is_modify(op) 	 	(op == op_insert || op == op_update || op == op_delete || op == op_truncate || op == op_merge)
+#define is_sample(op) 		(op == op_sample)
+#define is_insert(op) 		(op == op_insert)
+#define is_update(op) 		(op == op_update)
+#define is_delete(op) 		(op == op_delete)
+#define is_truncate(op) 	(op == op_truncate)
+#define is_merge(op) 		(op == op_merge)
+
+/* ZERO on empty sets, needed for sum (of counts)). */
+#define zero_if_empty(e) 	((e)->zero_if_empty)
+#define set_zero_if_empty(e) 	(e)->zero_if_empty = 1
+
+/* NO NIL semantics of aggr operations */
+#define need_no_nil(e) 		((e)->need_no_nil)
+#define set_no_nil(e) 		(e)->need_no_nil = 1
+
+/* does the expression (possibly) have nils */
+#define has_nil(e)			((e)->has_no_nil==0)
+#define set_has_no_nil(e) 	(e)->has_no_nil = 1
+#define set_has_nil(e) 		(e)->has_no_nil = 0
+
+#define is_ascending(e) 	((e)->ascending)
+#define set_ascending(e) 	((e)->ascending = 1)
+#define set_descending(e) 	((e)->ascending = 0)
+#define nulls_last(e) 		((e)->nulls_last)
+#define set_nulls_last(e) 	((e)->nulls_last=1)
+#define set_nulls_first(e) 	((e)->nulls_last=0)
+#define set_direction(e, dir) 	((e)->ascending = (dir&1), (e)->nulls_last = (dir&2)?1:0)
+
+#define is_unique(e)		((e)->unique)
+#define set_unique(e)		(e)->unique = 1
+#define set_not_unique(e)	(e)->unique = 0
+#define is_anti(e) 			((e)->anti)
+#define set_anti(e)  		(e)->anti = 1
+#define is_semantics(e) 	((e)->semantics)
+#define set_semantics(e) 	(e)->semantics = 1
+#define is_symmetric(e) 	((e)->symmetric)
+#define set_symmetric(e) 	(e)->symmetric = 1
+#define is_intern(e) 		((e)->intern)
+#define set_intern(e) 		(e)->intern = 1
+#define is_basecol(e) 		((e)->base)
+#define set_basecol(e) 		(e)->base = 1
+
+#define has_label(e)  		((e)->alias.label > 0)
+
+/* used for expressions and relations */
+#define need_distinct(er) 	((er)->distinct)
+#define set_distinct(er) 	(er)->distinct = 1
+#define set_nodistinct(er)	(er)->distinct = 0
+
+#define is_processed(rel) 	((rel)->processed)
+#define set_processed(rel) 	(rel)->processed = 1
+#define reset_processed(rel) 	(rel)->processed = 0
+#define is_dependent(rel) 	((rel)->dependent)
+#define set_dependent(rel) 	(rel)->dependent = 1
+#define reset_dependent(rel) 	(rel)->dependent = 0
+#define is_outer(rel)		((rel)->outer)
+#define set_outer(rel)		(rel)->outer = 1
+#define reset_outer(rel)	(rel)->outer = 0
+#define is_single(rel) 		((rel)->single)
+#define set_single(rel) 	(rel)->single = 1
+#define reset_single(rel) 	(rel)->single = 0
+
+#define is_freevar(e) 		((e)->freevar)
+#define set_freevar(e,level) 	(e)->freevar = level+1
+#define reset_freevar(e) 	(e)->freevar = 0
+
+#define rel_is_ref(rel)		(((sql_rel*)(rel))->ref.refcnt > 1)
+
+typedef struct relation {
+	sql_ref ref;
+
+	operator_type op;
+	void *l;
+	void *r;
+	list *exps;
+	int nrcols;	/* nr of cols */
+	unsigned int
+	 flag:16,
+	 card:4,	/* 0, 1 (row), 2 aggr, 3 */
+	 dependent:1, 	/* dependent join */
+	 distinct:1,
+	 processed:1,   /* fully processed or still in the process of building */
+	 outer:1,	/* used as outer (ungrouped) */
+	 grouped:1,	/* groupby processed all the group by exps */
+	 single:1,
+	/*
+	 * Used by rewriters at rel_unnest, rel_optimizer and rel_distribute so a relation is not modified twice
+	 * All bits are used by rel_unnest modifiers and always reset after.
+	 * The first bit is also used by rel_dce and rel_merge_select_rse optimizers.
+	 * The third bit is used by rel_remote_func only and it's not reset.
+	 */
+	 used:3;
+	void *p;	/* properties for the optimizer, distribution */
+} sql_rel;
+
+#endif /* SQL_RELATION_H */

monetdb/msvc/sql_scan.h

@@ -0,0 +1,68 @@
+/*
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0.  If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright 1997 - July 2008 CWI, August 2008 - 2022 MonetDB B.V.
+ */
+
+#ifndef _SQL_SCAN_H_
+#define _SQL_SCAN_H_
+
+#include "sql_mem.h"
+#include "sql_list.h"
+#include "stream.h"
+
+typedef enum { LINE_1, LINE_N } prot;
+
+struct scanner {
+	bstream *rs;
+	stream *ws;
+	stream *log;
+
+	int yynext;		/* next token, lr(1) isn't powerful enough for sql */
+	int yylast;		/* previous token, to detect superfluous semi-colons */
+	int yyval;		/* current token */
+	size_t yysval;		/* start of current token */
+	size_t yycur;		/* next char in the queue */
+	size_t as;		/* start of query part of view's etc */
+	char yybak;		/* sometimes it's needed to write an EOS marker */
+	int started;	/* found at least one token */
+	prot mode;		/* which mode (line (1,N), blocked) */
+	char *schema;	/* Keep schema name of create statement, needed AUTO_INCREMENT, SERIAL */
+	char *errstr;	/* error message from the bowels of the scanner */
+
+
+	/* Currently, MonetDB interprets \ specially in strings.  This is contrary
+	 * to the SQL standard. The default behavior of the reader is controlled by
+	 * ``raw_strings`` mserver5 option (``--set raw_strings=<true|false>``) and
+	 * the database property of the same name of merovingian (``monetdb set
+	 * raw_strings=yes <database>``). If it is ``true`` by default the reader
+	 * interprets strings as specified in the SQL standard, i.e. no
+	 * interpretation of the \ characters. If it is ``false`` \ characters are
+	 * used for escaping.
+	 *
+	 * The default value of this setting is false as of the Jul2021 release,
+	 * i.e. MonetDB by default interprets \ in strings.
+	 *
+	 * In case that we are interpreting \ in strings, we need state in the
+	 * scanner so that we Do The Right Thing (TM) when we get a unicode string
+	 * split up in multiple parts (i.e. U&'string1' 'string2') where the second
+	 * and subsequent string MUST NOT undergo \ interpretation.
+	 *
+	 * Because we need to be backward compatible (i.e. we need have ``select
+	 * char_length('a\nb')`` return 3) the next 2 members will probably stay for
+	 * the forseeable future.
+	 */
+	bool next_string_is_raw;
+	bool raw_string_mode;
+};
+
+#define QUERY(scanner) (scanner.rs->buf+scanner.rs->pos)
+
+extern char *query_cleaned(sql_allocator *sa, const char *query);
+extern void scanner_init(struct scanner *s, bstream *rs, stream *ws);
+sql_export void scanner_query_processed(struct scanner *s);
+
+extern int scanner_init_keywords(void);
+#endif /* _SQL_SCAN_H_ */

monetdb/msvc/sql_semantic.h

@@ -0,0 +1,76 @@
+/*
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0.  If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright 1997 - July 2008 CWI, August 2008 - 2022 MonetDB B.V.
+ */
+
+#ifndef _SQL_SEMANTIC_H_
+#define _SQL_SEMANTIC_H_
+
+#include "sql_list.h"
+#include "sql_symbol.h"
+#include "sql_parser.h"
+
+typedef struct exp_kind_t {
+	bte type;
+	bte card;
+	bit reduce;
+} exp_kind;
+
+extern sql_schema *cur_schema(mvc *sql);
+extern sql_schema *tmp_schema(mvc *sql);
+
+/* Search function for SQL objects with scoping rules. For tables an optional schema can be provided with search priority */
+sql_export sql_table *find_table_or_view_on_scope(mvc *sql, sql_schema *s, const char *sname, const char *tname, const char *error, bool isView);
+extern sql_sequence *find_sequence_on_scope(mvc *sql, const char *sname, const char *name, const char *error);
+extern sql_idx *find_idx_on_scope(mvc *sql, const char *sname, const char *name, const char *error);
+extern sql_type *find_type_on_scope(mvc *sql, const char *sname, const char *name, const char *error);
+extern sql_trigger *find_trigger_on_scope(mvc *sql, const char *sname, const char *name, const char *error);
+extern bool find_variable_on_scope(mvc *sql, const char *sname, const char *name, sql_var **var, sql_arg **a, sql_subtype **tpe, int *level, const char *error);
+
+/* These functions find catalog functions according to scoping rules */
+extern sql_subfunc *sql_find_func(mvc *sql, const char *sname, const char *fname, int nrargs, sql_ftype type, sql_subfunc *prev);
+extern sql_subfunc *sql_bind_member(mvc *sql, const char *sname, const char *fname, sql_subtype *tp, sql_ftype type, int nrargs, sql_subfunc *prev);
+extern sql_subfunc *sql_bind_func(mvc *sql, const char *sname, const char *fname, sql_subtype *tp1, sql_subtype *tp2, sql_ftype type);
+extern sql_subfunc *sql_bind_func3(mvc *sql, const char *sname, const char *fname, sql_subtype *tp1, sql_subtype *tp2, sql_subtype *tp3, sql_ftype type);
+extern sql_subfunc *sql_bind_func_result(mvc *sql, const char *sname, const char *fname, sql_ftype type, sql_subtype *res, int nargs, ...);
+extern sql_subfunc *sql_bind_func_(mvc *sql, const char *sname, const char *fname, list *ops, sql_ftype type);
+extern sql_subfunc *sql_resolve_function_with_undefined_parameters(mvc *sql, const char *sname, const char *fname, list *ops, sql_ftype type);
+
+extern list *sql_find_funcs(mvc *sql, const char *sname, const char *fname, int nrargs, sql_ftype type);
+extern list *sql_find_funcs_by_name(mvc *sql, const char *sname, const char *name, sql_ftype type);
+
+extern char *qname_schema(dlist *qname);
+extern char *qname_schema_object(dlist *qname);
+extern char *qname_catalog(dlist *qname);
+#define qname_module(qname) qname_schema(qname)
+
+extern sql_subtype *result_datatype(sql_subtype *super, sql_subtype *l, sql_subtype *r);
+extern sql_subtype *supertype(sql_subtype *super, sql_subtype *r, sql_subtype *i);
+
+typedef enum {
+	type_set,	/* set operations have very limiting coersion rules */
+	type_equal,
+	type_equal_no_any,
+	type_cast	/* also truncate */
+} check_type;
+
+/* SQL's parameters '?' (in prepare statements) and parameters of sql
+ * functions and procedures are kept in the param list.  */
+
+extern void sql_add_param(mvc *sql, const char *name, sql_subtype *st);
+extern sql_arg *sql_bind_param(mvc *sql, const char *name);
+/* once the type of the '?' parameters is known it's set using the set_type
+ * function */
+extern int set_type_param(mvc *sql, sql_subtype *type, int nr);
+extern void sql_destroy_params(mvc *sql);	/* used in backend */
+
+extern char *symbol2string(mvc *sql, symbol *s, int expression, char **err);
+//extern char *dlist2string(mvc *sql, dlist *s, int expression, char **err);
+
+extern char * toUpperCopy(char *dest, const char *src);
+
+#endif /*_SQL_SEMANTIC_H_*/
+

monetdb/msvc/sql_storage.h

@@ -0,0 +1,544 @@
+/*
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0.  If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright 1997 - July 2008 CWI, August 2008 - 2022 MonetDB B.V.
+ */
+
+#ifndef SQL_STORAGE_H
+#define SQL_STORAGE_H
+
+#include "sql_catalog.h"
+#include "store_sequence.h"
+
+#define COLSIZE	1024
+
+#define LOG_OK		0
+#define LOG_ERR		(-1)
+#define LOG_CONFLICT	(-2)
+
+#define isTempTable(x)   ((x)->persistence!=SQL_PERSIST)
+#define isGlobal(x)      ((x)->persistence!=SQL_LOCAL_TEMP && (x)->persistence!=SQL_DECLARED_TABLE)
+#define isGlobalTemp(x)  ((x)->persistence==SQL_GLOBAL_TEMP)
+#define isLocalTemp(x)   ((x)->persistence==SQL_LOCAL_TEMP)
+#define isTempSchema(x)  (strcmp((x)->base.name, "tmp") == 0)
+#define isDeclaredTable(x)  ((x)->persistence==SQL_DECLARED_TABLE)
+
+typedef enum store_type {
+	store_bat,	/* delta bats, ie multi user read/write */
+	store_tst,
+	store_mem
+} store_type;
+
+
+struct sqlstore;
+
+/* builtin functions have ids less than this */
+#define FUNC_OIDS 2000
+
+/* relational interface */
+typedef oid (*column_find_row_fptr)(sql_trans *tr, sql_column *c, const void *value, ...);
+typedef void *(*column_find_value_fptr)(sql_trans *tr, sql_column *c, oid rid);
+typedef sqlid (*column_find_sqlid_fptr)(sql_trans *tr, sql_column *c, oid rid);
+typedef bte (*column_find_bte_fptr)(sql_trans *tr, sql_column *c, oid rid);
+typedef sht (*column_find_sht_fptr)(sql_trans *tr, sql_column *c, oid rid);
+typedef int (*column_find_int_fptr)(sql_trans *tr, sql_column *c, oid rid);
+typedef lng (*column_find_lng_fptr)(sql_trans *tr, sql_column *c, oid rid);
+typedef const char * (*column_find_string_start_fptr)(sql_trans *tr, sql_column *c, oid rid, ptr *cbat);
+typedef void (*column_find_string_end_fptr)(ptr cbat);
+typedef int (*column_update_value_fptr)(sql_trans *tr, sql_column *c, oid rid, void *value);
+typedef int (*table_insert_fptr)(sql_trans *tr, sql_table *t, ...);
+typedef int (*table_delete_fptr)(sql_trans *tr, sql_table *t, oid rid);
+
+typedef res_table *(*table_orderby_fptr)(sql_trans *tr, sql_table *t,
+		sql_column *jl, sql_column *jr,
+		sql_column *jl2, sql_column *jr2 /* optional join(jl,jr,(jl2,jr2)) */, sql_column *o, ...);
+typedef void *(*table_fetch_value_fptr)(res_table *rt, sql_column *c);
+typedef void (*table_result_destroy_fptr)(res_table *rt);
+
+typedef struct rids {
+	BUN cur;
+	void *data;
+} rids;
+
+typedef struct subrids {
+	BUN pos;
+	sqlid id;
+	void *ids;
+	void *rids;
+} subrids;
+
+/* returns table rids, for the given select ranges */
+typedef rids *(*rids_select_fptr)( sql_trans *tr, sql_column *key, const void *key_value_low, const void *key_value_high, ...);
+
+/* order rids by orderby_column values */
+typedef rids *(*rids_orderby_fptr)( sql_trans *tr, rids *r, sql_column *orderby_col);
+
+typedef rids *(*rids_join_fptr)( sql_trans *tr, rids *l, sql_column *lc, rids *r, sql_column *rc);
+typedef rids *(*rids_semijoin_fptr)( sql_trans *tr, rids *l, sql_column *lc, rids *r, sql_column *rc);
+typedef rids *(*rids_diff_fptr)( sql_trans *tr, rids *l, sql_column *lc, subrids *r, sql_column *rc);
+
+/* return table rids from result of table_select, return (-1) when done */
+typedef oid (*rids_next_fptr)(rids *r);
+
+/* clean up the resources taken by the result of table_select */
+typedef void (*rids_destroy_fptr)(rids *r);
+typedef int (*rids_empty_fptr)(rids *r);
+
+typedef subrids *(*subrids_create_fptr)( sql_trans *tr, rids *l, sql_column *jc1, sql_column *jc2, sql_column *obc);
+
+/* return table rids from result of table_select, return (-1) when done */
+typedef oid (*subrids_next_fptr)(subrids *r);
+typedef sqlid (*subrids_nextid_fptr)(subrids *r);
+
+/* clean up the resources taken by the result of table_select */
+typedef void (*subrids_destroy_fptr)(subrids *r);
+
+typedef struct table_functions {
+	column_find_row_fptr column_find_row;
+	column_find_value_fptr column_find_value;
+	column_find_sqlid_fptr column_find_sqlid;
+	column_find_bte_fptr column_find_bte;
+	column_find_int_fptr column_find_int;
+	column_find_lng_fptr column_find_lng;
+	column_find_string_start_fptr column_find_string_start; /* this function returns a pointer to the heap, use it with care! */
+	column_find_string_end_fptr column_find_string_end; /* don't forget to call this function to unfix the bat descriptor! */
+
+	column_update_value_fptr column_update_value;
+	table_insert_fptr table_insert;
+	table_delete_fptr table_delete;
+	table_orderby_fptr table_orderby;
+	table_fetch_value_fptr table_fetch_value;
+	table_result_destroy_fptr table_result_destroy;
+
+	rids_select_fptr rids_select;
+	rids_orderby_fptr rids_orderby;
+	rids_join_fptr rids_join;
+	rids_semijoin_fptr rids_semijoin;
+	rids_next_fptr rids_next;
+	rids_destroy_fptr rids_destroy;
+	rids_empty_fptr rids_empty;
+
+	subrids_create_fptr subrids_create;
+	subrids_next_fptr subrids_next;
+	subrids_nextid_fptr subrids_nextid;
+	subrids_destroy_fptr subrids_destroy;
+	rids_diff_fptr rids_diff;
+} table_functions;
+
+/* delta table setup (ie readonly col + ins + upd + del)
+-- binds for column,idx (rdonly, inserts, updates) and delets
+*/
+typedef void *(*bind_col_fptr) (sql_trans *tr, sql_column *c, int access);
+typedef void *(*bind_idx_fptr) (sql_trans *tr, sql_idx *i, int access);
+typedef void *(*bind_cands_fptr) (sql_trans *tr, sql_table *t, int nr_of_parts, int part_nr);
+
+typedef int (*append_col_fptr) (sql_trans *tr, sql_column *c, BUN offset, BAT *offsets, void *d, BUN cnt, int t);
+typedef int (*append_idx_fptr) (sql_trans *tr, sql_idx *i, BUN offset, BAT *offsets, void *d, BUN cnt, int t);
+
+typedef int (*update_col_fptr) (sql_trans *tr, sql_column *c, void *tids, void *d, int t);
+typedef int (*update_idx_fptr) (sql_trans *tr, sql_idx *i, void *tids, void *d, int t);
+
+typedef int (*delete_tab_fptr) (sql_trans *tr, sql_table *t, void *d, int tpe);
+typedef int (*claim_tab_fptr) (sql_trans *tr, sql_table *t, size_t cnt, BUN *offset, BAT **offsets);
+typedef int (*tab_validate_fptr) (sql_trans *tr, sql_table *t, int uncommitted);
+
+/*
+-- count number of rows in column (excluding the deletes)
+-- check for sortedness
+ */
+typedef size_t (*count_del_fptr) (sql_trans *tr, sql_table *t, int access);
+typedef size_t (*count_col_fptr) (sql_trans *tr, sql_column *c, int access);
+typedef size_t (*count_idx_fptr) (sql_trans *tr, sql_idx *i, int access);
+typedef size_t (*dcount_col_fptr) (sql_trans *tr, sql_column *c);
+typedef int (*min_max_col_fptr) (sql_trans *tr, sql_column *c);
+typedef int (*prop_col_fptr) (sql_trans *tr, sql_column *c);
+
+/*
+-- create the necessary storage resources for columns, indices and tables
+-- returns LOG_OK, LOG_ERR
+*/
+typedef int (*create_col_fptr) (sql_trans *tr, sql_column *c);
+typedef int (*create_idx_fptr) (sql_trans *tr, sql_idx *i);
+typedef int (*create_del_fptr) (sql_trans *tr, sql_table *t);
+
+typedef void *(*col_dup_fptr) (sql_column *c);
+typedef void *(*idx_dup_fptr) (sql_idx *i);
+typedef void *(*del_dup_fptr) (sql_table *t);
+
+/*
+-- upgrade the necessary storage resources for columns, indices and tables
+-- needed for the upgrade of the logger structure (ie user tables are
+-- now stored using the object ids, no longer the names). This allows
+-- renames.
+-- returns LOG_OK, LOG_ERR
+*/
+typedef int (*upgrade_col_fptr) (sql_trans *tr, sql_column *c);
+typedef int (*upgrade_idx_fptr) (sql_trans *tr, sql_idx *i);
+typedef int (*upgrade_del_fptr) (sql_trans *tr, sql_table *t);
+typedef int (*swap_bats_fptr) (sql_trans *tr, sql_column *c, BAT *b);
+
+/*
+-- free the storage resources for columns, indices and tables
+-- returns LOG_OK, LOG_ERR
+*/
+typedef int (*destroy_col_fptr) (struct sqlstore *store, sql_column *c);
+typedef int (*destroy_idx_fptr) (struct sqlstore *store, sql_idx *i);
+typedef int (*destroy_del_fptr) (struct sqlstore *store, sql_table *t);
+
+/*
+-- drop a persistent table, ie add to the list of changes
+-- returns LOG_OK, LOG_ERR
+*/
+typedef int (*drop_col_fptr) (sql_trans *tr, sql_column *c);
+typedef int (*drop_idx_fptr) (sql_trans *tr, sql_idx *i);
+typedef int (*drop_del_fptr) (sql_trans *tr, sql_table *t);
+
+typedef BUN (*clear_table_fptr) (sql_trans *tr, sql_table *t);
+
+typedef enum storage_type {
+	ST_DEFAULT = 0,
+	ST_DICT,
+	ST_FOR,
+} storage_type;
+
+typedef int (*col_compress_fptr) (sql_trans *tr, sql_column *c, storage_type st, BAT *offsets, BAT *vals);
+
+/*
+-- update_table rollforward the changes made from table ft to table tt
+-- returns LOG_OK, LOG_ERR
+*/
+typedef int (*update_table_fptr) (sql_trans *tr, sql_table *ft, sql_table *tt);
+
+/* backing struct for this interface */
+typedef struct store_functions {
+
+	bind_col_fptr bind_col;
+	bind_idx_fptr bind_idx;
+	bind_cands_fptr bind_cands;
+
+	append_col_fptr append_col;
+	append_idx_fptr append_idx;
+
+	update_col_fptr update_col;
+	update_idx_fptr update_idx;
+
+	delete_tab_fptr delete_tab;
+	claim_tab_fptr claim_tab;
+	claim_tab_fptr key_claim_tab;
+	tab_validate_fptr tab_validate;
+
+	count_del_fptr count_del;
+	count_col_fptr count_col;
+	count_idx_fptr count_idx;
+	dcount_col_fptr dcount_col;
+	min_max_col_fptr min_max_col;
+	prop_col_fptr sorted_col;
+	prop_col_fptr unique_col;
+	prop_col_fptr double_elim_col; /* varsize col with double elimination */
+
+	col_dup_fptr col_dup;
+	idx_dup_fptr idx_dup;
+	del_dup_fptr del_dup;
+
+	create_col_fptr create_col;
+	create_idx_fptr create_idx;
+	create_del_fptr create_del;
+
+	destroy_col_fptr destroy_col;
+	destroy_idx_fptr destroy_idx;
+	destroy_del_fptr destroy_del;
+
+	drop_col_fptr drop_col;
+	drop_idx_fptr drop_idx;
+	drop_del_fptr drop_del;
+
+	clear_table_fptr clear_table;
+	col_compress_fptr col_compress;
+
+	upgrade_col_fptr upgrade_col;
+	upgrade_idx_fptr upgrade_idx;
+	upgrade_del_fptr upgrade_del;
+	swap_bats_fptr swap_bats;
+} store_functions;
+
+typedef int (*logger_create_fptr) (struct sqlstore *store, int debug, const char *logdir, int catalog_version);
+
+typedef void (*logger_destroy_fptr) (struct sqlstore *store);
+typedef int (*logger_flush_fptr) (struct sqlstore *store, lng save_id);
+typedef int (*logger_activate_fptr) (struct sqlstore *store);
+typedef int (*logger_cleanup_fptr) (struct sqlstore *store);
+
+typedef int (*logger_changes_fptr)(struct sqlstore *store);
+typedef int (*logger_get_sequence_fptr) (struct sqlstore *store, int seq, lng *id);
+
+typedef int (*log_isnew_fptr)(struct sqlstore *store);
+typedef int (*log_tstart_fptr) (struct sqlstore *store, bool flush);
+typedef int (*log_tend_fptr) (struct sqlstore *store);
+typedef int (*log_tdone_fptr) (struct sqlstore *store, ulng commit_ts);
+typedef lng (*log_save_id_fptr) (struct sqlstore *store);
+typedef int (*log_sequence_fptr) (struct sqlstore *store, int seq, lng id);
+
+/*
+-- List which parts of which files must be included in a hot snapshot.
+-- This is written to the given stream in the following format:
+-- - The first line is the absolute path of the db dir. All other paths
+--   are relative to this.
+-- - Every other line is either "c %ld %s\n" or "w %ld %s\n". The long
+--   is always the number of bytes to write. The %s is the relative path of the
+--   destination. For "c" lines (copy), it is also the relative path of the
+--   source file. For "w" lines (write), the %ld bytes to write follow directly
+--   after the newline.
+-- Using a stream (buffer) instead of a list data structure simplifies debugging
+-- and avoids a lot of tiny allocations and pointer manipulations.
+*/
+typedef gdk_return (*logger_get_snapshot_files_fptr)(struct sqlstore *store, stream *plan);
+
+typedef struct logger_functions {
+	logger_create_fptr create;
+	logger_destroy_fptr destroy;
+	logger_flush_fptr flush;
+	logger_activate_fptr activate;
+
+	logger_changes_fptr changes;
+	logger_get_sequence_fptr get_sequence;
+
+	logger_get_snapshot_files_fptr get_snapshot_files;
+
+	log_isnew_fptr log_isnew;
+	log_tstart_fptr log_tstart;
+	log_tend_fptr log_tend;
+	log_tdone_fptr log_tdone;
+	log_save_id_fptr log_save_id;
+	log_sequence_fptr log_sequence;
+} logger_functions;
+
+/* we need to add an interface for result_tables later */
+
+extern res_table *res_table_create(sql_trans *tr, int res_id, oid query_id, int nr_cols, mapi_query_t querytype, res_table *next, void *order);
+extern res_col *res_col_create(sql_trans *tr, res_table *t, const char *tn, const char *name, const char *typename, int digits, int scale, char mtype, void *v, bool cache);
+
+extern void res_table_destroy(res_table *t);
+
+extern res_table *res_tables_remove(res_table *results, res_table *t);
+sql_export void res_tables_destroy(res_table *results);
+extern res_table *res_tables_find(res_table *results, int res_id);
+
+extern struct sqlstore *store_init(int debug, store_type store, int readonly, int singleuser);
+extern void store_exit(struct sqlstore *store);
+
+extern void store_suspend_log(struct sqlstore *store);
+extern void store_resume_log(struct sqlstore *store);
+extern lng store_hot_snapshot(struct sqlstore *store, str tarfile);
+extern lng store_hot_snapshot_to_stream(struct sqlstore *store, stream *s);
+
+extern ulng store_function_counter(struct sqlstore *store);
+
+extern ulng store_oldest(struct sqlstore *store);
+extern ulng store_get_timestamp(struct sqlstore *store);
+extern void store_manager(struct sqlstore *store);
+
+extern void store_lock(struct sqlstore *store);
+extern void store_unlock(struct sqlstore *store);
+extern sqlid store_next_oid(struct sqlstore *store);
+
+extern int store_readonly(struct sqlstore *store);
+
+extern sql_trans *sql_trans_create(struct sqlstore *store, sql_trans *parent, const char *name);
+extern sql_trans *sql_trans_destroy(sql_trans *tr);
+//extern bool sql_trans_validate(sql_trans *tr);
+extern int sql_trans_commit(sql_trans *tr);
+
+extern int sql_trans_create_type(sql_trans *tr, sql_schema *s, const char *sqlname, unsigned int digits, unsigned int scale, int radix, const char *impl);
+extern int sql_trans_drop_type(sql_trans *tr, sql_schema * s, sqlid id, int drop_action);
+
+extern int sql_trans_create_func(sql_func **fres, sql_trans *tr, sql_schema *s, const char *func, list *args, list *res, sql_ftype type, sql_flang lang,
+								 const char *mod, const char *impl, const char *query, bit varres, bit vararg, bit system, bit side_effect);
+
+extern int sql_trans_drop_func(sql_trans *tr, sql_schema *s, sqlid id, int drop_action);
+extern int sql_trans_drop_all_func(sql_trans *tr, sql_schema *s, list *list_func, int drop_action);
+
+extern void sql_trans_update_tables(sql_trans *tr, sql_schema *s);
+extern void sql_trans_update_schemas(sql_trans *tr);
+
+extern int sql_trans_create_schema(sql_trans *tr, const char *name, sqlid auth_id, sqlid owner);
+extern int sql_trans_rename_schema(sql_trans *tr, sqlid id, const char *new_name);
+extern int sql_trans_drop_schema(sql_trans *tr, sqlid id, int drop_action);
+
+sql_export int sql_trans_create_table(sql_table **tres, sql_trans *tr, sql_schema *s, const char *name, const char *sql, int tt, bit system, int persistence, int commit_action, int sz, bte properties);
+
+extern int sql_trans_set_partition_table(sql_trans *tr, sql_table *t);
+extern int sql_trans_add_table(sql_trans *tr, sql_table *mt, sql_table *pt);
+extern int sql_trans_add_range_partition(sql_trans *tr, sql_table *mt, sql_table *pt, sql_subtype tpe, ptr min, ptr max, bit with_nills, int update, sql_part** err);
+extern int sql_trans_add_value_partition(sql_trans *tr, sql_table *mt, sql_table *pt, sql_subtype tpe, list* vals, bit with_nills, int update, sql_part **err);
+
+extern int sql_trans_rename_table(sql_trans *tr, sql_schema *s, sqlid id, const char *new_name);
+extern int sql_trans_set_table_schema(sql_trans *tr, sqlid id, sql_schema *os, sql_schema *ns);
+extern int sql_trans_del_table(sql_trans *tr, sql_table *mt, sql_table *pt, int drop_action);
+
+extern int sql_trans_drop_table(sql_trans *tr, sql_schema *s, const char *name, int drop_action);
+extern int sql_trans_drop_table_id(sql_trans *tr, sql_schema *s, sqlid id, int drop_action);
+extern BUN sql_trans_clear_table(sql_trans *tr, sql_table *t);
+extern int sql_trans_alter_access(sql_trans *tr, sql_table *t, sht access);
+
+extern int sql_trans_create_column(sql_column **rcol, sql_trans *tr, sql_table *t, const char *name, sql_subtype *tpe);
+extern int sql_trans_rename_column(sql_trans *tr, sql_table *t, sqlid id, const char *old_name, const char *new_name);
+extern int sql_trans_drop_column(sql_trans *tr, sql_table *t, sqlid id, int drop_action);
+extern int sql_trans_alter_null(sql_trans *tr, sql_column *col, int isnull);
+extern int sql_trans_alter_default(sql_trans *tr, sql_column *col, char *val);
+extern int sql_trans_alter_storage(sql_trans *tr, sql_column *col, char *storage);
+extern int sql_trans_is_sorted(sql_trans *tr, sql_column *col);
+extern int sql_trans_is_unique(sql_trans *tr, sql_column *col);
+extern int sql_trans_is_duplicate_eliminated(sql_trans *tr, sql_column *col);
+extern size_t sql_trans_dist_count(sql_trans *tr, sql_column *col);
+extern int sql_trans_ranges(sql_trans *tr, sql_column *col, void **min, void **max);
+
+extern void column_destroy(struct sqlstore *store, sql_column *c);
+extern void idx_destroy(struct sqlstore *store, sql_idx * i);
+extern void table_destroy(struct sqlstore *store, sql_table *t);
+
+extern int sql_trans_create_ukey(sql_key **res, sql_trans *tr, sql_table *t, const char *name, key_type kt);
+extern int sql_trans_key_done(sql_trans *tr, sql_key *k);
+
+extern int sql_trans_create_fkey(sql_fkey **res, sql_trans *tr, sql_table *t, const char *name, key_type kt, sql_key *rkey, int on_delete, int on_update);
+extern int sql_trans_create_kc(sql_trans *tr, sql_key *k, sql_column *c);
+extern int sql_trans_create_fkc(sql_trans *tr, sql_fkey *fk, sql_column *c);
+extern int sql_trans_drop_key(sql_trans *tr, sql_schema *s, sqlid id, int drop_action);
+
+extern int sql_trans_create_idx(sql_idx **i, sql_trans *tr, sql_table *t, const char *name, idx_type it);
+extern int sql_trans_create_ic(sql_trans *tr, sql_idx *i, sql_column *c);
+extern int sql_trans_drop_idx(sql_trans *tr, sql_schema *s, sqlid id, int drop_action);
+
+extern int sql_trans_create_trigger(sql_trigger **tres, sql_trans *tr, sql_table *t, const char *name,sht time, sht orientation, sht event, const char *old_name, const char *new_name,const char *condition, const char *statement);
+extern int sql_trans_drop_trigger(sql_trans *tr, sql_schema *s, sqlid id, int drop_action);
+
+extern sql_sequence *create_sql_sequence(struct sqlstore *store, sql_allocator *sa, sql_schema *s, const char *name, lng start, lng min, lng max, lng inc, lng cacheinc, bit cycle);
+extern int sql_trans_create_sequence(sql_trans *tr, sql_schema *s, const char *name, lng start, lng min, lng max, lng inc, lng cacheinc, bit cycle, bit bedropped);
+extern int sql_trans_drop_sequence(sql_trans *tr, sql_schema *s, sql_sequence *seq, int drop_action);
+extern int sql_trans_alter_sequence(sql_trans *tr, sql_sequence *seq, lng min, lng max, lng inc, lng cache, bit cycle);
+extern int sql_trans_sequence_restart(sql_trans *tr, sql_sequence *seq, lng start);
+
+extern sql_session * sql_session_create(struct sqlstore *store, sql_allocator *sa, int autocommit);
+extern void sql_session_destroy(sql_session *s);
+extern int sql_session_reset(sql_session *s, int autocommit);
+extern int sql_trans_begin(sql_session *s);
+extern int sql_trans_end(sql_session *s, int commit /* rollback=0, or commit=1 temporaries */);
+
+extern list* sql_trans_schema_user_dependencies(sql_trans *tr, sqlid schema_id);
+extern int sql_trans_create_dependency(sql_trans *tr, sqlid id, sqlid depend_id, sql_dependency depend_type);
+extern int sql_trans_drop_dependencies(sql_trans *tr, sqlid depend_id);
+extern int sql_trans_drop_dependency(sql_trans *tr, sqlid id, sqlid depend_id, sql_dependency depend_type);
+extern list* sql_trans_get_dependencies(sql_trans *tr, sqlid id, sql_dependency depend_type, list *ignore_ids);
+extern int sql_trans_get_dependency_type(sql_trans *tr, sqlid depend_id, sql_dependency depend_type);
+extern int sql_trans_check_dependency(sql_trans *tr, sqlid id, sqlid depend_id, sql_dependency depend_type);
+extern list* sql_trans_owner_schema_dependencies(sql_trans *tr, sqlid id);
+
+extern sql_table *create_sql_table(struct sqlstore *store, sql_allocator *sa, const char *name, sht type, bit system, int persistence, int commit_action, bit properties);
+extern sql_column *create_sql_column(struct sqlstore *store, sql_allocator *sa, sql_table *t, const char *name, sql_subtype *tpe);
+extern sql_key *create_sql_ukey(struct sqlstore *store, sql_allocator *sa, sql_table *t, const char *nme, key_type kt);
+extern sql_fkey *create_sql_fkey(struct sqlstore *store, sql_allocator *sa, sql_table *t, const char *nme, key_type kt, sql_key *rkey, int on_delete, int on_update );
+extern sql_key *create_sql_kc(struct sqlstore *store, sql_allocator *sa, sql_key *k, sql_column *c);
+extern sql_key * key_create_done(sql_trans *tr, sql_allocator *sa, sql_key *k);
+
+extern sql_idx *create_sql_idx(struct sqlstore *store, sql_allocator *sa, sql_table *t, const char *nme, idx_type it);
+extern sql_idx *create_sql_ic(struct sqlstore *store, sql_allocator *sa, sql_idx *i, sql_column *c);
+extern sql_idx *create_sql_idx_done(sql_trans *tr, sql_idx *i);
+extern sql_func *create_sql_func(struct sqlstore *store, sql_allocator *sa, const char *func, list *args, list *res, sql_ftype type, sql_flang lang, const char *mod,
+								 const char *impl, const char *query, bit varres, bit vararg, bit system, bit side_effect);
+
+/* for alter we need to duplicate a table */
+extern sql_table *dup_sql_table(sql_allocator *sa, sql_table *t);
+extern void drop_sql_column(sql_table *t, sqlid id, int drop_action);
+extern void drop_sql_idx(sql_table *t, sqlid id);
+extern void drop_sql_key(sql_table *t, sqlid id, int drop_action);
+
+extern int sql_trans_copy_column(sql_trans *tr, sql_table *t, sql_column *c, sql_column **cres);
+extern int sql_trans_copy_key(sql_trans *tr, sql_table *t, sql_key *k, sql_key **kres);
+extern int sql_trans_copy_idx(sql_trans *tr, sql_table *t, sql_idx *i, sql_idx **ires);
+extern int sql_trans_copy_trigger(sql_trans *tr, sql_table *t, sql_trigger *tri, sql_trigger **tres);
+extern sql_table *globaltmp_instantiate(sql_trans *tr, sql_table *t);
+
+#define NR_TABLE_LOCKS 64
+#define NR_COLUMN_LOCKS 512
+#define TRANSACTION_ID_BASE	(1ULL<<63)
+
+typedef struct sqlstore {
+	int catalog_version;	/* software version of the catalog */
+	sql_catalog *cat;		/* the catalog of persistent tables (what to do with tmp tables ?) */
+	sql_schema *tmp;		/* keep pointer to default (empty) tmp schema */
+	MT_Lock lock;			/* lock protecting concurrent writes (not reads, ie use rcu) */
+	MT_Lock commit;			/* protect transactions, only single commit (one wal writer) */
+	MT_Lock flush;			/* flush lock protecting concurrent writes (not reads, ie use rcu) */
+	MT_Lock table_locks[NR_TABLE_LOCKS];		/* protecting concurrent writes to tables (storage) */
+	MT_Lock column_locks[NR_COLUMN_LOCKS];		/* protecting concurrent writes to columns (storage) */
+	list *active;			/* list of running transactions */
+
+	ATOMIC_TYPE nr_active;	/* count number of transactions */
+	ATOMIC_TYPE lastactive;	/* timestamp of last active client */
+	ATOMIC_TYPE timestamp;	/* timestamp counter */
+	ATOMIC_TYPE transaction;/* transaction id counter */
+	ATOMIC_TYPE function_counter;/* function counter used during function instantiation */
+	ulng oldest;
+	ulng oldest_pending;
+	int readonly;			/* store is readonly */
+	int singleuser;			/* store is for a single user only (==1 enable, ==2 single user session running) */
+	int first;				/* just created the db */
+	int initialized;		/* used during bootstrap only */
+	int debug;				/* debug mask */
+	store_type active_type;
+	list *changes;			/* pending changes to cleanup */
+	sql_hash *dependencies; /* pending dependencies created to cleanup */
+	sql_hash *depchanges;	/* pending dependencies changes to cleanup */
+	list *seqchanges;		/* pending sequence number changes to be add to the first commiting transaction */
+	sql_hash *sequences;	/* loaded store sequence numbers */
+
+	sql_allocator *sa;		/* for now a store allocator, needs a special version with free operations (with reuse) */
+	sqlid obj_id, prev_oid;
+
+	store_functions storage_api;
+	table_functions table_api;
+	logger_functions logger_api;
+	void *logger;			/* space to keep logging structure of storage backend */
+} sqlstore;
+
+typedef enum sql_dependency_change_type {
+	ddl = 1,
+	dml = 2
+} sql_dependency_change_type;
+
+typedef struct sql_dependency_change {
+	sqlid objid; /* id of the object where the dependency was created */
+	sql_dependency_change_type type; /* type of dependency */
+	ulng ts; /* transaction timestamp of the dependency */
+} sql_dependency_change;
+
+typedef struct sql_change {
+	sql_base *obj;
+	ulng ts;        /* commit/rollback timestamp */
+	void *data;	/* data changes */
+	bool committed;	/* commit or rollback */
+	bool handled;	/* handled in commit */
+	tc_log_fptr log;		/* callback to log changes */
+	tc_commit_fptr commit;	/* callback to commit or rollback the changes */
+	tc_cleanup_fptr cleanup;/* callback to cleanup changes */
+} sql_change;
+
+extern void trans_add(sql_trans *tr, sql_base *b, void *data, tc_cleanup_fptr cleanup, tc_commit_fptr commit, tc_log_fptr log);
+extern int tr_version_of_parent(sql_trans *tr, ulng ts);
+
+extern int sql_trans_add_predicate(sql_trans* tr, sql_column *c, unsigned int cmp, atom *r, atom *f, bool anti, bool semantics);
+extern int sql_trans_add_dependency(sql_trans* tr, sqlid id, sql_dependency_change_type tp);
+sql_export int sql_trans_add_dependency_change(sql_trans *tr, sqlid id, sql_dependency_change_type tp);
+
+/* later move intop dict.h on this level */
+extern BAT *DICTenlarge(BAT *offsets, BUN cnt, BUN sz, role_t role);
+extern BAT *DICTdecompress_(BAT *o, BAT *u, role_t role);
+extern int DICTprepare4append(BAT **noffsets, BAT *vals, BAT *dict);
+extern int DICTprepare4append_vals(void **noffsets, void *vals, BUN cnt, BAT *dict);
+
+extern BAT *FORdecompress_(BAT *o, lng minval, int tt, role_t role);
+extern int FORprepare4append(BAT **noffsets, BAT *vals, lng minval, int tt);
+extern int FORprepare4append_vals(void **noffsets, void *vals, BUN cnt, lng minval, int vtype, int ft);
+
+#endif /*SQL_STORAGE_H */

monetdb/msvc/sql_string.h

@@ -0,0 +1,31 @@
+/*
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0.  If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright 1997 - July 2008 CWI, August 2008 - 2022 MonetDB B.V.
+ */
+
+#ifndef _SQL_STRING_H_
+#define _SQL_STRING_H_
+
+#define D__SQL	16
+
+#define _(String) (String)
+#define N_(String) (String)
+
+extern char *mkLower(char *v);
+extern char *toLower(const char *v);
+extern char *toUpper(const char *v);
+extern char *strconcat(const char *s1, const char *s2);
+extern char *strip_extra_zeros(char *v);
+extern char *sql_strdup(char *s);
+extern char *sql_escape_str(sql_allocator *sa, char *s);
+extern const char *sql_escape_ident(sql_allocator *sa, const char *s);
+extern char *sql_message(_In_z_ _Printf_format_string_ const char *format, ...)
+	__attribute__((__format__(__printf__, 1, 2)));
+extern char *sa_message(sql_allocator *sa, _In_z_ _Printf_format_string_ const char *format, ...)
+	__attribute__((__format__(__printf__, 2, 3)));
+
+#endif /*_SQL_STRING_H_*/
+

monetdb/msvc/sql_symbol.h

@@ -0,0 +1,98 @@
+/*
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0.  If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright 1997 - July 2008 CWI, August 2008 - 2022 MonetDB B.V.
+ */
+
+#ifndef SQL_SYMBOL_H
+#define SQL_SYMBOL_H
+
+#include "sql_types.h"
+#include "sql_atom.h"
+#include "sql_tokens.h"
+
+typedef enum symtype {
+	type_int,
+	type_lng,
+	type_string,
+	type_list,
+	type_symbol,
+	type_type
+} symtype;
+
+typedef union symbdata {
+	int i_val;
+	lng l_val;
+	char *sval;
+	struct dlist *lval;
+	struct symbol *sym;
+	struct sql_subtype typeval;
+} symbdata;
+
+typedef struct dnode {
+	struct dnode *next;
+	symbdata data;
+	symtype type;
+} dnode;
+
+typedef struct dlist {
+	dnode *h;
+	dnode *t;
+	int cnt;
+} dlist;
+
+extern dlist *dlist_create(sql_allocator *sa);
+extern int dlist_length(dlist *l);
+
+extern dlist *dlist_append_string(sql_allocator *sa, dlist *l, const char *data);
+extern dlist *dlist_append_list(sql_allocator *sa, dlist *l, dlist *data);
+extern dlist *dlist_append_int(sql_allocator *sa, dlist *l, int data);
+extern dlist *dlist_append_lng(sql_allocator *sa, dlist *l, lng data);
+extern dlist *dlist_append_symbol(sql_allocator *sa, dlist *l, struct symbol *data);
+extern dlist *dlist_append_type(sql_allocator *sa, dlist *l, struct sql_subtype *data);
+
+typedef struct symbol {
+	tokens token;
+	symtype type;
+	symbdata data;
+} symbol;
+
+typedef struct SelectNode {
+	symbol s;
+
+	symbol *limit;
+	symbol *offset;
+	symbol *sample;
+	symbol *seed;
+	int distinct;
+	int lateral;
+	struct dlist *selection;
+	struct dlist *into;	/* ?? */
+	symbol *from;
+	symbol *where;
+	symbol *groupby;
+	symbol *having;
+	symbol *orderby;
+	symbol *name;
+	symbol *window;
+} SelectNode;
+
+typedef struct AtomNode {
+	symbol s;
+	struct atom *a;
+} AtomNode;
+
+extern symbol *symbol_create(sql_allocator *sa, tokens token, char *data);
+extern symbol *symbol_create_list(sql_allocator *sa, tokens token, dlist *data);
+extern symbol *symbol_create_int(sql_allocator *sa, tokens token, int data);
+extern symbol *symbol_create_lng(sql_allocator *sa, tokens token, lng data);
+extern symbol *symbol_create_symbol(sql_allocator *sa, tokens token, symbol *data);
+
+extern symbol *newSelectNode(sql_allocator *sa, int distinct, struct dlist *selection, struct dlist *into, symbol *from, symbol *where, symbol *groupby, symbol *having, symbol *orderby, symbol *name, symbol *limit, symbol *offset, symbol *sample, symbol *seed, symbol *window);
+
+extern symbol *newAtomNode(sql_allocator *sa, atom *a);
+
+#endif /* SQL_SYMBOL_H */
+

monetdb/msvc/sql_tokens.h

@@ -0,0 +1,193 @@
+/*
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0.  If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright 1997 - July 2008 CWI, August 2008 - 2022 MonetDB B.V.
+ */
+
+#ifndef SQL_TOKENS_H
+#define SQL_TOKENS_H
+
+typedef enum tokens {
+	// Please keep this list sorted for ease of maintenance
+	SQL_AGGR,
+	SQL_ALTER_SEQ,
+	SQL_ALTER_TABLE,
+	SQL_ALTER_USER,
+	SQL_ANALYZE,
+	SQL_AND,
+	SQL_ASSIGN,
+	SQL_ATOM,
+	SQL_BETWEEN,
+	SQL_BINCOPYFROM,
+	SQL_BINOP,
+	SQL_CACHE,
+	SQL_CALL,
+	SQL_CASE,
+	SQL_CAST,
+	SQL_CHARSET,
+	SQL_CHECK,
+	SQL_COALESCE,
+	SQL_COLUMN,
+	SQL_COLUMN_GROUP,
+	SQL_COLUMN_OPTIONS,
+	SQL_COMMENT,
+	SQL_COMPARE,
+	SQL_CONSTRAINT,
+	SQL_COPYFROM,
+	SQL_COPYLOADER,
+	SQL_COPYTO,
+	SQL_CREATE_FUNC,
+	SQL_CREATE_INDEX,
+	SQL_CREATE_ROLE,
+	SQL_CREATE_SCHEMA,
+	SQL_CREATE_SEQ,
+	SQL_CREATE_TABLE,
+	SQL_CREATE_TABLE_LOADER,
+	SQL_CREATE_TRIGGER,
+	SQL_CREATE_TYPE,
+	SQL_CREATE_USER,
+	SQL_CREATE_VIEW,
+	SQL_CROSS,
+	SQL_CUBE,
+	SQL_CURRENT_ROW,
+	SQL_CYCLE,
+	SQL_DECLARE,
+	SQL_DECLARE_TABLE,
+	SQL_DEFAULT,
+	SQL_DELETE,
+	SQL_DROP_COLUMN,
+	SQL_DROP_CONSTRAINT,
+	SQL_DROP_DEFAULT,
+	SQL_DROP_FUNC,
+	SQL_DROP_INDEX,
+	SQL_DROP_ROLE,
+	SQL_DROP_SCHEMA,
+	SQL_DROP_SEQ,
+	SQL_DROP_TABLE,
+	SQL_DROP_TRIGGER,
+	SQL_DROP_TYPE,
+	SQL_DROP_USER,
+	SQL_DROP_VIEW,
+	SQL_ELSE,
+	SQL_ESCAPE,
+	SQL_EXCEPT,
+	SQL_EXECUTE,
+	SQL_EXISTS,
+	SQL_FILTER,
+	SQL_FOLLOWING,
+	SQL_FOREIGN_KEY,
+	SQL_FRAME,
+	SQL_FROM,
+	SQL_FUNC,
+	SQL_GRANT,
+	SQL_GRANT_ROLES,
+	SQL_GROUPBY,
+	SQL_GROUPING_SETS,
+	SQL_IDENT,
+	SQL_IF,
+	SQL_IN,
+	SQL_INC,
+	SQL_INDEX,
+	SQL_INSERT,
+	SQL_INTERSECT,
+	SQL_IS_NOT_NULL,
+	SQL_IS_NULL,
+	SQL_JOIN,
+	SQL_LIKE,
+	SQL_MAXVALUE,
+	SQL_MERGE,
+	SQL_MERGE_MATCH,
+	SQL_MERGE_NO_MATCH,
+	SQL_MERGE_PARTITION,
+	SQL_MINVALUE,
+	SQL_MULSTMT,
+	SQL_NAME,
+	SQL_NEXT,
+	SQL_NOP,
+	SQL_NOT,
+	SQL_NOT_BETWEEN,
+	SQL_NOT_EXISTS,
+	SQL_NOT_IN,
+	SQL_NOT_LIKE,
+	SQL_NOT_NULL,
+	SQL_NULL,
+	SQL_NULLIF,
+	SQL_OP,
+	SQL_OR,
+	SQL_ORDERBY,
+	SQL_PARAMETER,
+	SQL_PARTITION_COLUMN,
+	SQL_PARTITION_EXPRESSION,
+	SQL_PARTITION_LIST,
+	SQL_PARTITION_RANGE,
+	SQL_PATH,
+	SQL_PRECEDING,
+	SQL_PREP,
+	SQL_PRIMARY_KEY,
+	SQL_PW_ENCRYPTED,
+	SQL_PW_UNENCRYPTED,
+	SQL_RANK,
+	SQL_RENAME_COLUMN,
+	SQL_RENAME_SCHEMA,
+	SQL_RENAME_TABLE,
+	SQL_RENAME_USER,
+	SQL_RETURN,
+	SQL_REVOKE,
+	SQL_REVOKE_ROLES,
+	SQL_ROLLUP,
+	SQL_ROUTINE,
+	SQL_SCHEMA,
+	SQL_SELECT,
+	SQL_SEQUENCE,
+	SQL_SET,
+	SQL_SET_TABLE_SCHEMA,
+	SQL_START,
+	SQL_STORAGE,
+	SQL_TABLE,
+	SQL_TRUNCATE,
+	SQL_TYPE,
+	SQL_UNION,
+	SQL_UNIQUE,
+	SQL_UNOP,
+	SQL_UPDATE,
+	SQL_USING,
+	SQL_VALUES,
+	SQL_VIEW,
+	SQL_WHEN,
+	SQL_WHILE,
+	SQL_WINDOW,
+	SQL_WITH,
+	SQL_XMLATTRIBUTE,
+	SQL_XMLCOMMENT,
+	SQL_XMLCONCAT,
+	SQL_XMLDOCUMENT,
+	SQL_XMLELEMENT,
+	SQL_XMLFOREST,
+	SQL_XMLPARSE,
+	SQL_XMLPI,
+	SQL_XMLTEXT,
+	TR_COMMIT,
+	TR_MODE,
+	TR_RELEASE,
+	TR_ROLLBACK,
+	TR_SAVEPOINT,
+	TR_START
+	// Please keep this list sorted for ease of maintenance
+} tokens;
+
+typedef enum jt {
+	jt_inner = 0,
+	jt_left = 1,
+	jt_right = 2,
+	jt_full = 3
+} jt;
+
+typedef enum {
+	endian_big = 2,
+	endian_little = 1,
+	endian_native = 3,
+} endianness;
+
+#endif

monetdb/msvc/sql_types.h

@@ -0,0 +1,61 @@
+/*
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0.  If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright 1997 - July 2008 CWI, August 2008 - 2022 MonetDB B.V.
+ */
+
+#ifndef SQL_TYPES_H
+#define SQL_TYPES_H
+
+#include "sql_mem.h"
+#include "sql_list.h"
+#include "sql_string.h"
+#include "sql_catalog.h"
+#include "sql_storage.h"
+#include "sql_backend.h"
+#include "stream.h"
+
+extern list *aliases;
+extern list *types;
+extern list *funcs;
+
+extern unsigned int bits2digits(unsigned int b);
+extern unsigned int digits2bits(unsigned int d);
+extern unsigned int type_digits_to_char_digits(sql_subtype *t);
+
+extern int sql_type_convert(int form, int to); /* return 1, convert possible but it's a down cast, 2 convert possible can be done savely */
+extern bool is_commutative(const char *sname, const char *fnm); 	/* return true if commutative */
+
+extern char *sql_bind_alias(const char *alias);
+
+extern sql_subtype *arg_type( sql_arg *a);
+
+extern int sql_find_subtype(sql_subtype *res, const char *name, unsigned int digits, unsigned int scale);
+extern sql_subtype *sql_find_numeric(sql_subtype *r, int localtype, unsigned int digits);
+
+extern sql_subtype *sql_bind_subtype(sql_allocator *sa, const char *name, unsigned int digits, unsigned int scale);
+extern sql_subtype *sql_bind_localtype(const char *name);
+extern sql_subtype *sql_create_subtype(sql_allocator *sa, sql_type *t, unsigned int s, unsigned int d);
+sql_export void sql_init_subtype(sql_subtype *res, sql_type *t, unsigned int digits, unsigned int scale);
+
+extern int type_cmp(sql_type *t1, sql_type *t2);
+extern int subtype_cmp(sql_subtype *t1, sql_subtype *t2);
+extern int arg_subtype_cmp(sql_arg *a, sql_subtype *t);
+extern int is_subtype(sql_subtype *t1, sql_subtype *t2);
+
+extern char *sql_subtype_string(sql_allocator *sa, sql_subtype *t);
+extern char *subtype2string2(sql_allocator *sa, sql_subtype *tpe);
+
+extern sql_arg *sql_create_arg(sql_allocator *sa, const char *name, sql_subtype *t, char inout);
+
+extern int subfunc_cmp(sql_subfunc *f1, sql_subfunc *f2);
+extern sql_subfunc *sql_dup_subfunc(sql_allocator *sa, sql_func *f, list *ops, sql_subtype *member);
+
+extern char *sql_func_imp(sql_func *f);
+extern char *sql_func_mod(sql_func *f);
+
+extern void types_init(sql_allocator *sa);
+
+#endif /* SQL_TYPES_H */

monetdb/msvc/store_sequence.h

@@ -0,0 +1,29 @@
+/*
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0.  If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright 1997 - July 2008 CWI, August 2008 - 2022 MonetDB B.V.
+ */
+
+#ifndef STORE_SEQ_H
+#define STORE_SEQ_H
+
+#include "sql_catalog.h"
+
+extern void* sequences_init(void);
+extern void sequences_exit(void);
+
+extern int seq_hash(void *seq);
+extern void seq_hash_destroy(sql_hash *h);
+extern int seq_get_value(sql_store store, sql_sequence *seq, lng *val);
+extern int seq_next_value(sql_store store, sql_sequence *seq, lng *val);
+extern int seq_restart(sql_store store, sql_sequence *seq, lng start);
+
+extern void log_store_sequence(sql_store store, void *seq); /* called locked */
+extern int seqbulk_next_value(sql_store store, sql_sequence *seq, lng cnt, lng* dest);
+
+extern void sequences_lock(sql_store store);
+extern void sequences_unlock(sql_store store);
+
+#endif /* STORE_SEQ_H */

monetdb/msvc/stream.h

@@ -0,0 +1,293 @@
+/*
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0.  If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright 1997 - July 2008 CWI, August 2008 - 2022 MonetDB B.V.
+ */
+
+#ifndef _STREAM_H_
+#define _STREAM_H_
+
+/*
+ * File: stream.h
+ * Auteur: Niels J. Nes
+ * Date: 09-01-2001
+ *
+ * Version 0.1: start
+ *
+ * This is the general interface to input/output. Each stream will
+ * contains some stream info (for now only byteorder). This is
+ * required for proper conversion on different byte order platforms.
+ */
+
+//#include <unistd.h>
+#include <ctype.h>
+#include <stdio.h>
+
+#include <stdlib.h>
+#include <signal.h>
+#include <limits.h>
+
+/* avoid using "#ifdef WIN32" so that this file does not need our config.h */
+#if defined(_MSC_VER) || defined(__CYGWIN__) || defined(__MINGW32__)
+# ifndef LIBSTREAM
+#  define stream_export extern __declspec(dllimport)
+# else
+#  define stream_export extern __declspec(dllexport)
+# endif
+#else
+# define stream_export extern
+#endif
+#ifndef HAVE_HGE
+# ifdef HAVE___INT128
+#  define HAVE_HGE 1
+typedef __int128 hge;
+# else
+#  ifdef HAVE___INT128_T
+#   define HAVE_HGE 1
+typedef __int128_t hge;
+#  endif
+# endif
+#endif
+
+/* Defines to help the compiler check printf-style format arguments.
+ * These defines are also in our config.h, but we repeat them here so
+ * that we don't need that for this file.*/
+#ifndef __GNUC__
+/* This feature is available in gcc versions 2.5 and later.  */
+# ifndef __attribute__
+#  define __attribute__(Spec)	/* empty */
+# endif
+#endif
+#if !defined(_MSC_VER) && !defined(_In_z_)
+# define _In_z_
+# define _Printf_format_string_
+#endif
+
+#define EOT 4
+
+/* fwf gets turned into a csv with these parameters */
+#define STREAM_FWF_FIELD_SEP '|'
+#define STREAM_FWF_ESCAPE '\\'
+#define STREAM_FWF_RECORD_SEP '\n'
+#define STREAM_FWF_FILLER ' '
+
+typedef struct stream stream;
+
+/* some os specific initialization */
+stream_export int mnstr_init(void);
+
+
+typedef enum mnstr_error_kind {
+	MNSTR_NO__ERROR = 0,
+	MNSTR_OPEN_ERROR,
+	MNSTR_READ_ERROR,
+	MNSTR_WRITE_ERROR,
+	MNSTR_TIMEOUT,
+	MNSTR_UNEXPECTED_EOF,
+} mnstr_error_kind;
+
+typedef enum mnstr_flush_level {
+	MNSTR_FLUSH_DATA = 1, // write all data
+	MNSTR_FLUSH_ALL = 2,  // write all data and also reset compression state
+} mnstr_flush_level;
+
+stream_export const char *mnstr_version(void);
+
+stream_export char *mnstr_error(const stream *s);
+stream_export const char* mnstr_peek_error(const stream *s);
+stream_export mnstr_error_kind mnstr_errnr(const stream *s);
+stream_export const char *mnstr_error_kind_name(mnstr_error_kind k);
+stream_export void mnstr_clearerr(stream *s);
+
+/* all mnstr_readX/mnstr_writeX return
+ *  0 on error
+ * !0 on success
+ */
+stream_export int mnstr_readBte(stream *restrict s, int8_t *restrict val); // unused
+stream_export int mnstr_readChr(stream *restrict s, char *restrict val); // used once in gdk_logger.c
+stream_export int mnstr_writeChr(stream *s, char val); // used once in gdk_logger and mclient.c
+stream_export int mnstr_writeBte(stream *s, int8_t val); // used in sql_result.c/mapi10
+
+stream_export int mnstr_readSht(stream *restrict s, int16_t *restrict val); // unused
+stream_export int mnstr_writeSht(stream *s, int16_t val); // used in sql_result.c/mapi10
+stream_export int mnstr_readInt(stream *restrict s, int *restrict val); // used in gdk
+stream_export int mnstr_writeInt(stream *s, int val); // used in gdk
+stream_export int mnstr_readLng(stream *restrict s, int64_t *restrict val); // used in gdk_logger.c
+stream_export int mnstr_writeLng(stream *s, int64_t val); // used in gdk_logger.c, sql_result.c/mapi10
+
+
+stream_export int mnstr_writeFlt(stream *s, float val); // sql_result.c/mapi10
+stream_export int mnstr_writeDbl(stream *s, double val); // sql_result.c/mapi10
+
+#ifdef HAVE_HGE
+stream_export int mnstr_readHge(stream *restrict s, hge *restrict val); // unused
+stream_export int mnstr_writeHge(stream *s, hge val); // sql_result.c/mapi10
+#endif
+
+stream_export int mnstr_readBteArray(stream *restrict s, int8_t *restrict val, size_t cnt); // unused
+stream_export int mnstr_writeBteArray(stream *restrict s, const int8_t *restrict val, size_t cnt);
+stream_export int mnstr_writeStr(stream *restrict s, const char *restrict val); // sql_result.c/mapi10
+stream_export int mnstr_readStr(stream *restrict s, char *restrict val); // unused
+
+stream_export int mnstr_readShtArray(stream *restrict s, int16_t *restrict val, size_t cnt); // unused
+stream_export int mnstr_writeShtArray(stream *restrict s, const int16_t *restrict val, size_t cnt); //unused
+stream_export int mnstr_readIntArray(stream *restrict s, int *restrict val, size_t cnt); // used once in geom.c
+stream_export int mnstr_writeIntArray(stream *restrict s, const int *restrict val, size_t cnt); // used once in geom.c
+stream_export int mnstr_readLngArray(stream *restrict s, int64_t *restrict val, size_t cnt); // unused
+stream_export int mnstr_writeLngArray(stream *restrict s, const int64_t *restrict val, size_t cnt); // unused
+#ifdef HAVE_HGE
+stream_export int mnstr_readHgeArray(stream *restrict s, hge *restrict val, size_t cnt); // unused
+stream_export int mnstr_writeHgeArray(stream *restrict s, const hge *restrict val, size_t cnt); // unused
+#endif
+stream_export int mnstr_printf(stream *restrict s, _In_z_ _Printf_format_string_ const char *restrict format, ...) // USED all over
+	__attribute__((__format__(__printf__, 2, 3)));
+stream_export ssize_t mnstr_read(stream *restrict s, void *restrict buf, size_t elmsize, size_t cnt); // USED all over
+stream_export ssize_t mnstr_readline(stream *restrict s, void *restrict buf, size_t maxcnt); // used in mclient, sql.c/mvc_export_table_wrap, bat_logger.c
+stream_export ssize_t mnstr_write(stream *restrict s, const void *restrict buf, size_t elmsize, size_t cnt); // USED all over
+stream_export void mnstr_close(stream *s);
+stream_export void mnstr_destroy(stream *s);
+stream_export int mnstr_flush(stream *s, mnstr_flush_level flush_level); // used all over
+stream_export int mnstr_fsync(stream *s); // used in gdk_logger.c, wlc.c and store.c
+stream_export int mnstr_fgetpos(stream *restrict s, fpos_t *restrict p); // unused
+stream_export int mnstr_fsetpos(stream *restrict s, fpos_t *restrict p); // unused
+stream_export char *mnstr_name(const stream *s); // used when wrapping in mclient.c
+stream_export bool mnstr_isbinary(const stream *s); // unused
+stream_export bool mnstr_get_swapbytes(const stream *s); // sql_result.c/mapi10
+stream_export void mnstr_set_bigendian(stream *s, bool bigendian); // used in mapi.c and mal_session.c
+stream_export void mnstr_settimeout(stream *s, unsigned int ms, bool (*func)(void *), void *data); // used in mapi.c and mal_session.c
+stream_export int mnstr_isalive(const stream *s); // used once in mal_interpreter.c
+stream_export bool mnstr_eof(const stream *s); // stream saw end-of-file
+
+stream_export stream *open_rstream(const char *filename); // used in mclient.c, gdk_logger.c, store.c, snapshot.c
+stream_export stream *open_wstream(const char *filename); // used in gdk_logger.c and store.c
+
+/* open in ascii stream in read mode */
+stream_export stream *open_rastream(const char *filename); // used 13 times
+
+/* open in ascii stream in write mode*/
+stream_export stream *open_wastream(const char *filename); // used in mclient.c, mapi.c, mal_io.c, wlc.c, sql.c, wlr.c
+
+stream_export void close_stream(stream *s);
+
+stream_export stream *open_urlstream(const char *url); // mclient.c, future copy from remote
+
+stream_export stream *file_rstream(FILE *restrict fp, bool binary, const char *restrict name); // unused
+stream_export stream *file_wstream(FILE *restrict fp, bool binary, const char *restrict name); // unused
+stream_export stream *stdin_rastream(void);
+stream_export stream *stdout_wastream(void);
+stream_export stream *stderr_wastream(void);
+
+stream_export stream *xz_stream(stream *inner, int preset);
+stream_export stream *gz_stream(stream *inner, int preset);
+stream_export stream *bz2_stream(stream *inner, int preset);
+stream_export stream *lz4_stream(stream *inner, int preset);
+stream_export stream *compressed_stream(stream *inner, int preset);
+
+stream_export FILE *getFile(stream *s); // gdk_logger.c progress messages
+stream_export int getFileNo(stream *s);	/* fileno(getFile(s)) */ // mclient.c, gdk_logger.c progress messages
+stream_export size_t getFileSize(stream *s); // mal_import.c, sql_scenario.c, wlr.c, store.c, bat_logger.c
+
+stream_export stream *iconv_rstream(stream *restrict ss, const char *restrict charset, const char *restrict name); // mclient.c stdin
+stream_export stream *iconv_wstream(stream *restrict ss, const char *restrict charset, const char *restrict name); // mclient.c stdout
+
+typedef struct buffer {
+	char *buf;
+	size_t pos;
+	size_t len;
+} buffer;
+
+stream_export void buffer_init(buffer *restrict b, char *restrict buf, size_t size); // used in many places
+stream_export buffer *buffer_create(size_t size); // used in sql_gencode.c and store.c/snapshot
+stream_export char *buffer_get_buf(buffer *b);
+stream_export void buffer_destroy(buffer *b);
+
+stream_export stream *buffer_rastream(buffer *restrict b, const char *restrict name); // used in many places
+stream_export stream *buffer_wastream(buffer *restrict b, const char *restrict name); // sql_gencode.c
+stream_export buffer *mnstr_get_buffer(stream *s);
+
+/* note, the size is fixed to 8K, you cannot simply change it to any
+ * value */
+#define BLOCK (8 * 1024 - 2)
+
+/* Block stream is a stream which sends data in blocks of a known size
+ * (BLOCK size or dynamically changed using CHANGE_BLOCK_SIZE msg).
+ *
+ * A block is written once more than BLOCK size data has been written
+ * using the write commands or when the flush command is sent.
+ *
+ * All full blocks together with a single not full block form a major
+ * block. Major blocks can be used to synchronize the communication.
+ * Example server sends some reply, ie a major block consisting of
+ * various minor blocks. The header of the major block can contain
+ * special info which the client can interpret.
+ *
+ * Each read attempt tries to return the number of bytes. Once a lower
+ * number of bytes can be read the end of the major block is
+ * found. The next read will then start with a new major block.
+ */
+stream_export stream *block_stream(stream *s); // mapi.c, mal_mapi.c, client.c, merovingian
+stream_export bool isa_block_stream(const stream *s); // mapi.c, mal_client.c, remote.c, sql_scenario.c/sqlReader, sql_scan.c
+stream_export stream *bs_stream(stream *s); // unused
+stream_export void set_prompting(stream *block_stream, const char *prompt, stream *prompt_stream);
+
+
+typedef enum {
+	PROTOCOL_AUTO = 0, // unused
+	PROTOCOL_9 = 1, // mal_mapi.c, mal_client.c;
+	PROTOCOL_COLUMNAR = 3 // sql_result.c
+} protocol_version;
+
+typedef enum {
+	COMPRESSION_NONE = 0, // mal_mapi.c
+	COMPRESSION_SNAPPY = 1, // mcrypt.c, mal_mapi.c
+	COMPRESSION_LZ4 = 2, // same
+	COMPRESSION_AUTO = 255 // never used
+} compression_method;
+
+stream_export stream *block_stream2(stream *s, size_t bufsiz, compression_method comp); // mal_mapi.c
+stream_export int bs2_resizebuf(stream *ss, size_t bufsiz); // sql_result.c
+stream_export buffer bs2_buffer(stream *s); // sql_result.c
+stream_export void bs2_setpos(stream *ss, size_t pos); // sql_result.c
+
+
+/* read block of data including the end of block marker */
+stream_export ssize_t mnstr_read_block(stream *restrict s, void *restrict buf, size_t elmsize, size_t cnt);
+// used in mapi.c ,mal_mapi,c, merovingian/{client.c,controlrunner.c,control.c}
+
+typedef struct bstream {
+	stream *s;
+	char *buf;
+	size_t size;		/* size of buf */
+	size_t pos;		/* the data cursor (ie read until pos) */
+	size_t len;		/* len of the data (<= size) */
+	size_t mode;		/* 0 line mode else size for block mode */
+	bool eof;
+} bstream;
+
+stream_export bstream *bstream_create(stream *rs, size_t chunk_size); // used all over
+stream_export void bstream_destroy(bstream *s); // all over
+stream_export ssize_t bstream_read(bstream *s, size_t size); // tablet.c, tokenizer.c
+stream_export ssize_t bstream_next(bstream *s); // all over
+
+/* Callback stream is a stream where the read and write functions are
+ * provided by the caller.  close and destroy are also provided.  The
+ * private pointer is passed on to the callback functions when they
+ * are invoked. */
+stream_export stream *callback_stream(
+	void *restrict priv,
+	ssize_t (*read)(void *restrict priv, void *restrict buf, size_t elmsize, size_t cnt),
+	ssize_t (*write)(void *restrict priv, const void *restrict buf, size_t elmsize, size_t cnt),
+	void (*close)(void *priv),
+	void (*destroy)(void *priv),
+	const char *restrict name); // used in mclient.c, for readline
+
+stream_export stream *stream_blackhole_create(void); // never used
+
+stream_export stream *stream_fwf_create(stream *restrict s, size_t num_fields, size_t *restrict widths, char filler); // sql.c
+
+
+stream_export stream *create_text_stream(stream *s);
+
+#endif /*_STREAM_H_*/

msc-plugin/launcher.vcxproj

@@ -181,7 +181,7 @@
     <ClCompile>
       <WarningLevel>Level3</WarningLevel>
       <SDLCheck>true</SDLCheck>
-      <PreprocessorDefinitions>_CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>_CRT_SECURE_NO_WARNINGS;THREADING;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <ConformanceMode>true</ConformanceMode>
       <MultiProcessorCompilation>true</MultiProcessorCompilation>
       <EnableParallelCodeGeneration>true</EnableParallelCodeGeneration>
@@ -202,7 +202,7 @@
       <FunctionLevelLinking>true</FunctionLevelLinking>
       <IntrinsicFunctions>true</IntrinsicFunctions>
       <SDLCheck>true</SDLCheck>
-      <PreprocessorDefinitions>_CRT_SECURE_NO_WARNINGS;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>_CRT_SECURE_NO_WARNINGS;THREADING;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <ConformanceMode>true</ConformanceMode>
       <MultiProcessorCompilation>true</MultiProcessorCompilation>
       <EnableParallelCodeGeneration>true</EnableParallelCodeGeneration>
@@ -227,7 +227,7 @@
       <FunctionLevelLinking>true</FunctionLevelLinking>
       <IntrinsicFunctions>true</IntrinsicFunctions>
       <SDLCheck>true</SDLCheck>
-      <PreprocessorDefinitions>_CRT_SECURE_NO_WARNINGS;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>_CRT_SECURE_NO_WARNINGS;THREADING;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <ConformanceMode>true</ConformanceMode>
       <MultiProcessorCompilation>true</MultiProcessorCompilation>
       <EnableParallelCodeGeneration>true</EnableParallelCodeGeneration>
@@ -252,7 +252,7 @@
       <FunctionLevelLinking>true</FunctionLevelLinking>
       <IntrinsicFunctions>true</IntrinsicFunctions>
       <SDLCheck>true</SDLCheck>
-      <PreprocessorDefinitions>_CRT_SECURE_NO_WARNINGS;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>_CRT_SECURE_NO_WARNINGS;THREADING;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <ConformanceMode>true</ConformanceMode>
       <MultiProcessorCompilation>true</MultiProcessorCompilation>
       <EnableParallelCodeGeneration>true</EnableParallelCodeGeneration>
@@ -276,7 +276,7 @@
     <ClCompile>
       <WarningLevel>Level3</WarningLevel>
       <SDLCheck>true</SDLCheck>
-      <PreprocessorDefinitions>_CRT_SECURE_NO_WARNINGS;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>_CRT_SECURE_NO_WARNINGS;THREADING;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <ConformanceMode>true</ConformanceMode>
       <MultiProcessorCompilation>true</MultiProcessorCompilation>
       <EnableParallelCodeGeneration>true</EnableParallelCodeGeneration>
@@ -301,7 +301,7 @@
       <FunctionLevelLinking>true</FunctionLevelLinking>
       <IntrinsicFunctions>true</IntrinsicFunctions>
       <SDLCheck>true</SDLCheck>
-      <PreprocessorDefinitions>_CRT_SECURE_NO_WARNINGS;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>_CRT_SECURE_NO_WARNINGS;THREADING;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <ConformanceMode>true</ConformanceMode>
       <MultiProcessorCompilation>true</MultiProcessorCompilation>
       <EnableParallelCodeGeneration>true</EnableParallelCodeGeneration>
@@ -326,7 +326,7 @@
       <FunctionLevelLinking>true</FunctionLevelLinking>
       <IntrinsicFunctions>true</IntrinsicFunctions>
       <SDLCheck>true</SDLCheck>
-      <PreprocessorDefinitions>_CRT_SECURE_NO_WARNINGS;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>_CRT_SECURE_NO_WARNINGS;THREADING;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <ConformanceMode>true</ConformanceMode>
       <MultiProcessorCompilation>true</MultiProcessorCompilation>
       <EnableParallelCodeGeneration>true</EnableParallelCodeGeneration>
@@ -351,7 +351,7 @@
       <FunctionLevelLinking>true</FunctionLevelLinking>
       <IntrinsicFunctions>true</IntrinsicFunctions>
       <SDLCheck>true</SDLCheck>
-      <PreprocessorDefinitions>_CRT_SECURE_NO_WARNINGS;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>_CRT_SECURE_NO_WARNINGS;THREADING;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <ConformanceMode>true</ConformanceMode>
       <MultiProcessorCompilation>true</MultiProcessorCompilation>
       <EnableParallelCodeGeneration>true</EnableParallelCodeGeneration>

msc-plugin/libaquery.vcxproj

@@ -130,7 +130,7 @@
     <ClCompile>
       <WarningLevel>Level3</WarningLevel>
       <SDLCheck>true</SDLCheck>
-      <PreprocessorDefinitions>QUERY_DECLSPEC=__declspec(dllexport);_CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>QUERY_DECLSPEC=__declspec(dllexport);_CRT_SECURE_NO_WARNINGS;THREADING;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <ConformanceMode>true</ConformanceMode>
       <LanguageStandard>stdcpplatest</LanguageStandard>
       <LanguageStandard_C>stdc17</LanguageStandard_C>
@@ -156,7 +156,7 @@
       <FunctionLevelLinking>true</FunctionLevelLinking>
       <IntrinsicFunctions>true</IntrinsicFunctions>
       <SDLCheck>false</SDLCheck>
-      <PreprocessorDefinitions>QUERY_DECLSPEC=__declspec(dllexport);_CRT_SECURE_NO_WARNINGS;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>QUERY_DECLSPEC=__declspec(dllexport);_CRT_SECURE_NO_WARNINGS;THREADING;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <ConformanceMode>true</ConformanceMode>
       <LanguageStandard>stdcpplatest</LanguageStandard>
       <LanguageStandard_C>stdc17</LanguageStandard_C>
@@ -197,7 +197,7 @@
       <FunctionLevelLinking>true</FunctionLevelLinking>
       <IntrinsicFunctions>true</IntrinsicFunctions>
       <SDLCheck>false</SDLCheck>
-      <PreprocessorDefinitions>QUERY_DECLSPEC=__declspec(dllexport);_CRT_SECURE_NO_WARNINGS;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>QUERY_DECLSPEC=__declspec(dllexport);_CRT_SECURE_NO_WARNINGS;THREADING;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <ConformanceMode>true</ConformanceMode>
       <LanguageStandard>stdcpplatest</LanguageStandard>
       <LanguageStandard_C>stdc17</LanguageStandard_C>
@@ -236,7 +236,7 @@
     <ClCompile>
       <WarningLevel>Level3</WarningLevel>
       <SDLCheck>true</SDLCheck>
-      <PreprocessorDefinitions>QUERY_DECLSPEC=__declspec(dllexport);_CRT_SECURE_NO_WARNINGS;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>QUERY_DECLSPEC=__declspec(dllexport);_CRT_SECURE_NO_WARNINGS;THREADING;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <ConformanceMode>true</ConformanceMode>
       <LanguageStandard>stdcpplatest</LanguageStandard>
       <LanguageStandard_C>stdc17</LanguageStandard_C>
@@ -266,7 +266,7 @@
       <FunctionLevelLinking>true</FunctionLevelLinking>
       <IntrinsicFunctions>true</IntrinsicFunctions>
       <SDLCheck>false</SDLCheck>
-      <PreprocessorDefinitions>QUERY_DECLSPEC=__declspec(dllexport);_CRT_SECURE_NO_WARNINGS;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>QUERY_DECLSPEC=__declspec(dllexport);_CRT_SECURE_NO_WARNINGS;THREADING;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <ConformanceMode>true</ConformanceMode>
       <LanguageStandard>stdcpplatest</LanguageStandard>
       <LanguageStandard_C>stdc17</LanguageStandard_C>
@@ -307,7 +307,7 @@
       <FunctionLevelLinking>true</FunctionLevelLinking>
       <IntrinsicFunctions>true</IntrinsicFunctions>
       <SDLCheck>false</SDLCheck>
-      <PreprocessorDefinitions>QUERY_DECLSPEC=__declspec(dllexport);_CRT_SECURE_NO_WARNINGS;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>QUERY_DECLSPEC=__declspec(dllexport);_CRT_SECURE_NO_WARNINGS;THREADING;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <ConformanceMode>true</ConformanceMode>
       <LanguageStandard>stdcpplatest</LanguageStandard>
       <LanguageStandard_C>stdc17</LanguageStandard_C>
@@ -361,7 +361,9 @@
     <None Include="..\server\cpp.hint" />
   </ItemGroup>
   <ItemGroup>
+    <ClCompile Include="..\server\monetdb_ext.c" />
     <ClCompile Include="..\server\server.cpp" />
+    <ClCompile Include="..\server\threading.cpp" />
     <ClCompile Include="..\server\winhelper.cpp" />
     <ClCompile Include="..\server\libaquery.cpp" />
     <ClCompile Include="..\server\monetdb_conn.cpp" />

msc-plugin/msc-plugin.vcxproj

@@ -134,7 +134,7 @@
     <ClCompile>
       <WarningLevel>Level3</WarningLevel>
       <SDLCheck>true</SDLCheck>
-      <PreprocessorDefinitions>QUERY_DECLSPEC=__declspec(dllimport);_CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>QUERY_DECLSPEC=__declspec(dllimport);THREADING;_CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <ConformanceMode>true</ConformanceMode>
       <LanguageStandard>stdcpplatest</LanguageStandard>
       <ControlFlowGuard>Guard</ControlFlowGuard>
@@ -166,7 +166,7 @@
       <FunctionLevelLinking>true</FunctionLevelLinking>
       <IntrinsicFunctions>true</IntrinsicFunctions>
       <SDLCheck>false</SDLCheck>
-      <PreprocessorDefinitions>QUERY_DECLSPEC=__declspec(dllimport);_CRT_SECURE_NO_WARNINGS;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>QUERY_DECLSPEC=__declspec(dllimport);THREADING;_CRT_SECURE_NO_WARNINGS;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <ConformanceMode>true</ConformanceMode>
       <LanguageStandard>stdcpplatest</LanguageStandard>
       <AdditionalIncludeDirectories>$(ProjectDir)\..\monetdb\msvc</AdditionalIncludeDirectories>
@@ -209,7 +209,7 @@
       <FunctionLevelLinking>true</FunctionLevelLinking>
       <IntrinsicFunctions>true</IntrinsicFunctions>
       <SDLCheck>false</SDLCheck>
-      <PreprocessorDefinitions>QUERY_DECLSPEC=__declspec(dllimport);_CRT_SECURE_NO_WARNINGS;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>QUERY_DECLSPEC=__declspec(dllimport);THREADING;_CRT_SECURE_NO_WARNINGS;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <ConformanceMode>true</ConformanceMode>
       <LanguageStandard>stdcpplatest</LanguageStandard>
       <AdditionalIncludeDirectories>$(ProjectDir)\..\monetdb\msvc</AdditionalIncludeDirectories>
@@ -250,7 +250,7 @@
     <ClCompile>
       <WarningLevel>Level3</WarningLevel>
       <SDLCheck>true</SDLCheck>
-      <PreprocessorDefinitions>QUERY_DECLSPEC=__declspec(dllimport);_ALLOW_RTCc_IN_STL;_CRT_SECURE_NO_WARNINGS;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>QUERY_DECLSPEC=__declspec(dllimport);THREADING;_ALLOW_RTCc_IN_STL;_CRT_SECURE_NO_WARNINGS;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <ConformanceMode>true</ConformanceMode>
       <LanguageStandard>stdcpplatest</LanguageStandard>
       <ControlFlowGuard>false</ControlFlowGuard>
@@ -300,7 +300,7 @@
       <FunctionLevelLinking>true</FunctionLevelLinking>
       <IntrinsicFunctions>true</IntrinsicFunctions>
       <SDLCheck>false</SDLCheck>
-      <PreprocessorDefinitions>QUERY_DECLSPEC=__declspec(dllimport);_CRT_SECURE_NO_WARNINGS;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>QUERY_DECLSPEC=__declspec(dllimport);THREADING;_CRT_SECURE_NO_WARNINGS;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <ConformanceMode>true</ConformanceMode>
       <LanguageStandard>stdcpplatest</LanguageStandard>
       <AdditionalIncludeDirectories>$(ProjectDir)\..\monetdb\msvc</AdditionalIncludeDirectories>
@@ -343,7 +343,7 @@
       <FunctionLevelLinking>true</FunctionLevelLinking>
       <IntrinsicFunctions>true</IntrinsicFunctions>
       <SDLCheck>false</SDLCheck>
-      <PreprocessorDefinitions>QUERY_DECLSPEC=__declspec(dllimport);_CRT_SECURE_NO_WARNINGS;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>QUERY_DECLSPEC=__declspec(dllimport);THREADING;_CRT_SECURE_NO_WARNINGS;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <ConformanceMode>true</ConformanceMode>
       <LanguageStandard>stdcpplatest</LanguageStandard>
       <AdditionalIncludeDirectories>$(ProjectDir)\..\monetdb\msvc</AdditionalIncludeDirectories>

msc-plugin/sdk_example.vcxproj

@@ -132,7 +132,7 @@
     <ClCompile>
       <WarningLevel>Level3</WarningLevel>
       <SDLCheck>true</SDLCheck>
-      <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>THREADING;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <ConformanceMode>true</ConformanceMode>
       <LanguageStandard>stdcpplatest</LanguageStandard>
       <LanguageStandard_C>stdc17</LanguageStandard_C>
@@ -142,6 +142,7 @@
     <Link>
       <SubSystem>Console</SubSystem>
       <GenerateDebugInformation>true</GenerateDebugInformation>
+      <AdditionalDependencies>../libaquery.lib;$(CoreLibraryDependencies);%(AdditionalDependencies)</AdditionalDependencies>
     </Link>
   </ItemDefinitionGroup>
   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
@@ -150,7 +151,7 @@
       <FunctionLevelLinking>true</FunctionLevelLinking>
       <IntrinsicFunctions>true</IntrinsicFunctions>
       <SDLCheck>true</SDLCheck>
-      <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>THREADING;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <ConformanceMode>true</ConformanceMode>
       <LanguageStandard>stdcpplatest</LanguageStandard>
       <LanguageStandard_C>stdc17</LanguageStandard_C>
@@ -160,6 +161,7 @@
       <EnableCOMDATFolding>true</EnableCOMDATFolding>
       <OptimizeReferences>true</OptimizeReferences>
       <GenerateDebugInformation>true</GenerateDebugInformation>
+      <AdditionalDependencies>../libaquery.lib;$(CoreLibraryDependencies);%(AdditionalDependencies)</AdditionalDependencies>
     </Link>
   </ItemDefinitionGroup>
   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='RelWDbgInf|Win32'">
@@ -168,7 +170,7 @@
       <FunctionLevelLinking>true</FunctionLevelLinking>
       <IntrinsicFunctions>true</IntrinsicFunctions>
       <SDLCheck>true</SDLCheck>
-      <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>THREADING;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <ConformanceMode>true</ConformanceMode>
       <LanguageStandard>stdcpplatest</LanguageStandard>
       <LanguageStandard_C>stdc17</LanguageStandard_C>
@@ -178,13 +180,14 @@
       <EnableCOMDATFolding>true</EnableCOMDATFolding>
       <OptimizeReferences>true</OptimizeReferences>
       <GenerateDebugInformation>true</GenerateDebugInformation>
+      <AdditionalDependencies>../libaquery.lib;$(CoreLibraryDependencies);%(AdditionalDependencies)</AdditionalDependencies>
     </Link>
   </ItemDefinitionGroup>
   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
     <ClCompile>
       <WarningLevel>Level3</WarningLevel>
       <SDLCheck>true</SDLCheck>
-      <PreprocessorDefinitions>_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>THREADING;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <ConformanceMode>true</ConformanceMode>
       <LanguageStandard>stdcpplatest</LanguageStandard>
       <LanguageStandard_C>stdc17</LanguageStandard_C>
@@ -195,6 +198,7 @@
     <Link>
       <SubSystem>Console</SubSystem>
       <GenerateDebugInformation>true</GenerateDebugInformation>
+      <AdditionalDependencies>../libaquery.lib;$(CoreLibraryDependencies);%(AdditionalDependencies)</AdditionalDependencies>
     </Link>
     <Bscmake>
       <PreserveSbr>true</PreserveSbr>
@@ -206,7 +210,7 @@
       <FunctionLevelLinking>true</FunctionLevelLinking>
       <IntrinsicFunctions>true</IntrinsicFunctions>
       <SDLCheck>true</SDLCheck>
-      <PreprocessorDefinitions>NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>THREADING;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <ConformanceMode>true</ConformanceMode>
       <LanguageStandard>stdcpplatest</LanguageStandard>
       <LanguageStandard_C>stdc17</LanguageStandard_C>
@@ -216,6 +220,7 @@
       <EnableCOMDATFolding>true</EnableCOMDATFolding>
       <OptimizeReferences>true</OptimizeReferences>
       <GenerateDebugInformation>true</GenerateDebugInformation>
+      <AdditionalDependencies>../libaquery.lib;$(CoreLibraryDependencies);%(AdditionalDependencies)</AdditionalDependencies>
     </Link>
   </ItemDefinitionGroup>
   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='RelWDbgInf|x64'">
@@ -224,7 +229,7 @@
       <FunctionLevelLinking>true</FunctionLevelLinking>
       <IntrinsicFunctions>true</IntrinsicFunctions>
       <SDLCheck>true</SDLCheck>
-      <PreprocessorDefinitions>NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>THREADING;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <ConformanceMode>true</ConformanceMode>
       <LanguageStandard>stdcpplatest</LanguageStandard>
       <LanguageStandard_C>stdc17</LanguageStandard_C>
@@ -234,6 +239,7 @@
       <EnableCOMDATFolding>true</EnableCOMDATFolding>
       <OptimizeReferences>true</OptimizeReferences>
       <GenerateDebugInformation>true</GenerateDebugInformation>
+      <AdditionalDependencies>../libaquery.lib;$(CoreLibraryDependencies);%(AdditionalDependencies)</AdditionalDependencies>
     </Link>
   </ItemDefinitionGroup>
   <ItemGroup>

msc-plugin/server.vcxproj

@@ -192,8 +192,9 @@
     <Link>
       <SubSystem>Console</SubSystem>
       <GenerateDebugInformation>true</GenerateDebugInformation>
-      <AdditionalDependencies>..\libaquery.lib;$(ProjectDir)\..\monetdb\msvc\monetdbe.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <AdditionalDependencies>bat.lib;monetdbsql.lib;..\libaquery.lib;$(ProjectDir)\..\monetdb\msvc\monetdbe.lib;%(AdditionalDependencies)</AdditionalDependencies>
       <AdditionalOptions>/WHOLEARCHIVE:libaquery.lib %(AdditionalOptions)</AdditionalOptions>
+      <AdditionalLibraryDirectories>../monetdb/msvc</AdditionalLibraryDirectories>
     </Link>
   </ItemDefinitionGroup>
   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
@@ -216,9 +217,10 @@
       <EnableCOMDATFolding>true</EnableCOMDATFolding>
       <OptimizeReferences>true</OptimizeReferences>
       <GenerateDebugInformation>false</GenerateDebugInformation>
-      <AdditionalDependencies>..\libaquery.lib;$(ProjectDir)\..\monetdb\msvc\monetdbe.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <AdditionalDependencies>bat.lib;monetdbsql.lib;..\libaquery.lib;$(ProjectDir)\..\monetdb\msvc\monetdbe.lib;%(AdditionalDependencies)</AdditionalDependencies>
       <DataExecutionPrevention>false</DataExecutionPrevention>
       <AdditionalOptions>/WHOLEARCHIVE:libaquery.lib %(AdditionalOptions)</AdditionalOptions>
+      <AdditionalLibraryDirectories>../monetdb/msvc</AdditionalLibraryDirectories>
     </Link>
   </ItemDefinitionGroup>
   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='RelWDbgInf|Win32'">
@@ -241,9 +243,10 @@
       <EnableCOMDATFolding>true</EnableCOMDATFolding>
       <OptimizeReferences>true</OptimizeReferences>
       <GenerateDebugInformation>false</GenerateDebugInformation>
-      <AdditionalDependencies>..\libaquery.lib;$(ProjectDir)\..\monetdb\msvc\monetdbe.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <AdditionalDependencies>bat.lib;monetdbsql.lib;..\libaquery.lib;$(ProjectDir)\..\monetdb\msvc\monetdbe.lib;%(AdditionalDependencies)</AdditionalDependencies>
       <DataExecutionPrevention>false</DataExecutionPrevention>
       <AdditionalOptions>/WHOLEARCHIVE:libaquery.lib %(AdditionalOptions)</AdditionalOptions>
+      <AdditionalLibraryDirectories>../monetdb/msvc</AdditionalLibraryDirectories>
     </Link>
   </ItemDefinitionGroup>
   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='sharedlib|Win32'">
@@ -265,8 +268,9 @@
       <EnableCOMDATFolding>true</EnableCOMDATFolding>
       <OptimizeReferences>true</OptimizeReferences>
       <GenerateDebugInformation>true</GenerateDebugInformation>
-      <AdditionalDependencies>$(ProjectDir)\..\monetdb\msvc\monetdbe.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <AdditionalDependencies>bat.lib;monetdbsql.lib;$(ProjectDir)\..\monetdb\msvc\monetdbe.lib;%(AdditionalDependencies)</AdditionalDependencies>
       <AdditionalOptions>/WHOLEARCHIVE:libaquery.lib %(AdditionalOptions)</AdditionalOptions>
+      <AdditionalLibraryDirectories>../monetdb/msvc</AdditionalLibraryDirectories>
     </Link>
     <PostBuildEvent>
       <Command>copy $(OutDir)$(TargetName)$(TargetExt) $(ProjectDir)\..\server.so  /y</Command>
@@ -288,8 +292,9 @@
     <Link>
       <SubSystem>Console</SubSystem>
       <GenerateDebugInformation>true</GenerateDebugInformation>
-      <AdditionalDependencies>..\libaquery.lib;$(ProjectDir)\..\monetdb\msvc\monetdbe.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <AdditionalDependencies>bat.lib;monetdbsql.lib;..\libaquery.lib;$(ProjectDir)\..\monetdb\msvc\monetdbe.lib;%(AdditionalDependencies)</AdditionalDependencies>
       <AdditionalOptions>/WHOLEARCHIVE:libaquery.lib %(AdditionalOptions)</AdditionalOptions>
+      <AdditionalLibraryDirectories>../monetdb/msvc</AdditionalLibraryDirectories>
     </Link>
     <Bscmake>
       <PreserveSbr>true</PreserveSbr>
@@ -315,9 +320,10 @@
       <EnableCOMDATFolding>true</EnableCOMDATFolding>
       <OptimizeReferences>true</OptimizeReferences>
       <GenerateDebugInformation>false</GenerateDebugInformation>
-      <AdditionalDependencies>..\libaquery.lib;$(ProjectDir)\..\monetdb\msvc\monetdbe.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <AdditionalDependencies>bat.lib;monetdbsql.lib;..\libaquery.lib;$(ProjectDir)\..\monetdb\msvc\monetdbe.lib;%(AdditionalDependencies)</AdditionalDependencies>
       <DataExecutionPrevention>false</DataExecutionPrevention>
       <AdditionalOptions>/WHOLEARCHIVE:libaquery.lib %(AdditionalOptions)</AdditionalOptions>
+      <AdditionalLibraryDirectories>../monetdb/msvc</AdditionalLibraryDirectories>
     </Link>
   </ItemDefinitionGroup>
   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='RelWDbgInf|x64'">
@@ -340,9 +346,10 @@
       <EnableCOMDATFolding>true</EnableCOMDATFolding>
       <OptimizeReferences>true</OptimizeReferences>
       <GenerateDebugInformation>false</GenerateDebugInformation>
-      <AdditionalDependencies>..\libaquery.lib;$(ProjectDir)\..\monetdb\msvc\monetdbe.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <AdditionalDependencies>bat.lib;monetdbsql.lib;..\libaquery.lib;$(ProjectDir)\..\monetdb\msvc\monetdbe.lib;%(AdditionalDependencies)</AdditionalDependencies>
       <DataExecutionPrevention>false</DataExecutionPrevention>
       <AdditionalOptions>/WHOLEARCHIVE:libaquery.lib %(AdditionalOptions)</AdditionalOptions>
+      <AdditionalLibraryDirectories>../monetdb/msvc</AdditionalLibraryDirectories>
     </Link>
   </ItemDefinitionGroup>
   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='sharedlib|x64'">
@@ -364,8 +371,9 @@
       <EnableCOMDATFolding>true</EnableCOMDATFolding>
       <OptimizeReferences>true</OptimizeReferences>
       <GenerateDebugInformation>true</GenerateDebugInformation>
-      <AdditionalDependencies>$(ProjectDir)\..\monetdb\msvc\monetdbe.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <AdditionalDependencies>bat.lib;monetdbsql.lib;$(ProjectDir)\..\monetdb\msvc\monetdbe.lib;%(AdditionalDependencies)</AdditionalDependencies>
       <AdditionalOptions>/WHOLEARCHIVE:libaquery.lib %(AdditionalOptions)</AdditionalOptions>
+      <AdditionalLibraryDirectories>../monetdb/msvc</AdditionalLibraryDirectories>
     </Link>
     <PostBuildEvent>
       <Command>copy "$(OutDir)$(TargetName)$(TargetExt)" "$(ProjectDir)\..\server.so" /y</Command>

msc-plugin/testmain.vcxproj

@@ -181,7 +181,7 @@
     <ClCompile>
       <WarningLevel>Level3</WarningLevel>
       <SDLCheck>true</SDLCheck>
-      <PreprocessorDefinitions>_CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>_CRT_SECURE_NO_WARNINGS;THREADING;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <ConformanceMode>true</ConformanceMode>
       <MultiProcessorCompilation>true</MultiProcessorCompilation>
       <EnableParallelCodeGeneration>true</EnableParallelCodeGeneration>
@@ -202,7 +202,7 @@
       <FunctionLevelLinking>true</FunctionLevelLinking>
       <IntrinsicFunctions>true</IntrinsicFunctions>
       <SDLCheck>true</SDLCheck>
-      <PreprocessorDefinitions>_CRT_SECURE_NO_WARNINGS;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>_CRT_SECURE_NO_WARNINGS;THREADING;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <ConformanceMode>true</ConformanceMode>
       <MultiProcessorCompilation>true</MultiProcessorCompilation>
       <EnableParallelCodeGeneration>true</EnableParallelCodeGeneration>
@@ -228,7 +228,7 @@
       <FunctionLevelLinking>true</FunctionLevelLinking>
       <IntrinsicFunctions>true</IntrinsicFunctions>
       <SDLCheck>true</SDLCheck>
-      <PreprocessorDefinitions>_CRT_SECURE_NO_WARNINGS;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>_CRT_SECURE_NO_WARNINGS;THREADING;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <ConformanceMode>true</ConformanceMode>
       <MultiProcessorCompilation>true</MultiProcessorCompilation>
       <EnableParallelCodeGeneration>true</EnableParallelCodeGeneration>
@@ -254,7 +254,7 @@
       <FunctionLevelLinking>true</FunctionLevelLinking>
       <IntrinsicFunctions>true</IntrinsicFunctions>
       <SDLCheck>true</SDLCheck>
-      <PreprocessorDefinitions>_CRT_SECURE_NO_WARNINGS;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>_CRT_SECURE_NO_WARNINGS;THREADING;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <ConformanceMode>true</ConformanceMode>
       <MultiProcessorCompilation>true</MultiProcessorCompilation>
       <EnableParallelCodeGeneration>true</EnableParallelCodeGeneration>
@@ -278,7 +278,7 @@
     <ClCompile>
       <WarningLevel>Level3</WarningLevel>
       <SDLCheck>true</SDLCheck>
-      <PreprocessorDefinitions>_CRT_SECURE_NO_WARNINGS;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>_CRT_SECURE_NO_WARNINGS;THREADING;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <ConformanceMode>true</ConformanceMode>
       <MultiProcessorCompilation>true</MultiProcessorCompilation>
       <EnableParallelCodeGeneration>true</EnableParallelCodeGeneration>
@@ -303,7 +303,7 @@
       <FunctionLevelLinking>true</FunctionLevelLinking>
       <IntrinsicFunctions>true</IntrinsicFunctions>
       <SDLCheck>true</SDLCheck>
-      <PreprocessorDefinitions>_CRT_SECURE_NO_WARNINGS;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>_CRT_SECURE_NO_WARNINGS;THREADING;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <ConformanceMode>true</ConformanceMode>
       <MultiProcessorCompilation>true</MultiProcessorCompilation>
       <EnableParallelCodeGeneration>true</EnableParallelCodeGeneration>
@@ -329,7 +329,7 @@
       <FunctionLevelLinking>true</FunctionLevelLinking>
       <IntrinsicFunctions>true</IntrinsicFunctions>
       <SDLCheck>true</SDLCheck>
-      <PreprocessorDefinitions>_CRT_SECURE_NO_WARNINGS;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>_CRT_SECURE_NO_WARNINGS;THREADING;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <ConformanceMode>true</ConformanceMode>
       <MultiProcessorCompilation>true</MultiProcessorCompilation>
       <EnableParallelCodeGeneration>true</EnableParallelCodeGeneration>
@@ -355,7 +355,7 @@
       <FunctionLevelLinking>true</FunctionLevelLinking>
       <IntrinsicFunctions>true</IntrinsicFunctions>
       <SDLCheck>true</SDLCheck>
-      <PreprocessorDefinitions>_CRT_SECURE_NO_WARNINGS;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>_CRT_SECURE_NO_WARNINGS;THREADING;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <ConformanceMode>true</ConformanceMode>
       <MultiProcessorCompilation>true</MultiProcessorCompilation>
       <EnableParallelCodeGeneration>true</EnableParallelCodeGeneration>

paper

@@ -1 +1 @@
-Subproject commit 0d0ec214297a3e46947cead37f887eb480043419
+Subproject commit 6f49544835a35915e1561a2e6581ee9ac0759381

prompt.py

@@ -101,12 +101,12 @@ import numpy as np
 from mo_parsing import ParseException
 
 import aquery_parser as parser
-import engine
-import engine.ddl
-import engine.projection
+import common
+import common.ddl
+import common.projection
 import reconstruct as xengine
 from build import build_manager
-from engine.utils import add_dll_dir, base62uuid, nullstream, ws
+from common.utils import add_dll_dir, base62uuid, nullstream, ws
 
 
 ## CLASSES BEGIN
@@ -336,7 +336,7 @@ def init_threaded(state : PromptState):
         state.get_storedproc.restype = StoredProcedure
         aquery_config.have_hge = server_so['have_hge']()
         if aquery_config.have_hge != 0:
-            from engine.types import get_int128_support
+            from common.types import get_int128_support
             get_int128_support()
         state.th = threading.Thread(target=server_so['main'], args=(-1, ctypes.POINTER(ctypes.c_char_p)(state.cfg.c)), daemon=True)
         state.th.start() 
@@ -345,7 +345,7 @@ def init_prompt() -> PromptState:
     aquery_config.init_config()
     
     state = PromptState()
-    engine.utils.session_context = state
+    common.utils.session_context = state
     # if aquery_config.rebuild_backend:
     #     try:
     #         os.remove(state.server_bin) 
@@ -412,7 +412,7 @@ def prompt(running = lambda:True, next = lambda:input('> '), state : Optional[Pr
     q = ''
     payload = None
     keep = True
-    cxt = engine.initialize()
+    cxt = common.initialize()
     parser.parse('SELECT "**** WELCOME TO AQUERY++! ****";')
     
     # state.currstats = QueryStats()
@@ -442,7 +442,7 @@ def prompt(running = lambda:True, next = lambda:input('> '), state : Optional[Pr
                 continue
             if False and q == 'exec': # generate build and run (AQuery Engine)
                 state.cfg.backend_type = Backend_Type.BACKEND_AQuery.value
-                cxt = engine.exec(state.stmts, cxt, keep)
+                cxt = common.exec(state.stmts, cxt, keep)
                 if state.buildmgr.build_dll() == 0:
                     state.set_ready()
                 continue
@@ -588,7 +588,7 @@ def prompt(running = lambda:True, next = lambda:input('> '), state : Optional[Pr
                     qs = qs[1]
                 with open(qs) as file:
                     qs = file.readline()
-                    from engine.utils import _Counter
+                    from common.utils import _Counter
                     lst_tell = -1
                     while(qs):
                         while(not ws.sub('', qs) or qs.strip().startswith('#')):
@@ -629,7 +629,7 @@ def prompt(running = lambda:True, next = lambda:input('> '), state : Optional[Pr
             elif q.startswith('procedure'):
                 qs = re.split(r'[ \t\r\n]', q)
                 procedure_help = '''Usage: procedure <procedure_name> [record|stop|run|remove|save|load]'''
-                from engine.utils import send_to_server
+                from common.utils import send_to_server
                 if len(qs) > 2:
                     if qs[2].lower() =='record':
                         if state.current_procedure is not None and state.current_procedure != qs[1]:             
@@ -722,7 +722,7 @@ if __name__ == '__main__':
     if nextcmd or check_param(['-s', '--script']):
         with open(nextcmd) as file:
             nextcmd = file.readline()
-            from engine.utils import _Counter
+            from common.utils import _Counter
             if file.name.endswith('aquery') or nextcmd.strip() == '#!aquery':
                 state = init_prompt()
                 while(nextcmd):

reconstruct/ast.py

@@ -3,8 +3,8 @@ from dataclasses import dataclass
 from enum import Enum, auto
 from typing import Dict, List, Optional, Set, Tuple, Union
 
-from engine.types import *
-from engine.utils import (base62alp, base62uuid, enlist, 
+from common.types import *
+from common.utils import (base62alp, base62uuid, enlist, 
                           get_innermost, get_legal_name)
 from reconstruct.storage import ColRef, Context, TableInfo
 
@@ -1096,7 +1096,7 @@ class create_trigger(ast_node):
         pass
     
     def produce(self, node):
-        from engine.utils import send_to_server, get_storedproc
+        from common.utils import send_to_server, get_storedproc
         node = node['create_trigger']
         self.trigger_name = node['name']
         self.action_name = node['action']
@@ -1134,11 +1134,11 @@ class create_trigger(ast_node):
         self.context.triggers_active.add(self)
 
     def execute(self): 
-        from engine.utils import send_to_server
+        from common.utils import send_to_server
         self.context.queries.append(f'TA{self.query_name}\0{self.action_name}')
 
     def remove(self):
-        from engine.utils import send_to_server
+        from common.utils import send_to_server
         self.context.queries.append(f'TR{self.trigger_name}')
 
 
@@ -1443,7 +1443,7 @@ class udf(ast_node):
         all = ('_builtin_len', '_builtin_ret')
         
     def decltypecall(self, c_code = False, *args):
-        from engine.types import fn_behavior
+        from common.types import fn_behavior
         class dummy:
             def __init__(self, name):
                 self.cname = name + '_gettype'
@@ -1451,7 +1451,7 @@ class udf(ast_node):
         return fn_behavior(dummy(self.cname), c_code, *args)
     
     def __call__(self, c_code = False, *args):
-        from engine.types import fn_behavior
+        from common.types import fn_behavior
         builtin_args = [f'{{{n}()}}' for n, v in self.builtin.items() if v.enabled]
         return fn_behavior(self, c_code, *args, *builtin_args)
     
@@ -1465,6 +1465,7 @@ class udf(ast_node):
                 255, name = 'generic_ref', cname = 'auto&'
             ))
         }
+        self.ccode = ''
         self.var_table = {}
         self.args = []
         udf.try_init_udf(self.context)
@@ -1486,7 +1487,7 @@ class udf(ast_node):
                 
         
     def produce(self, node):
-        from engine.utils import check_legal_name, get_legal_name
+        from common.utils import check_legal_name, get_legal_name
         node = node[self.name]
         # register udf
         self.agg = 'Agg' in node
@@ -1581,7 +1582,7 @@ class udf(ast_node):
                     
                     
     def consume(self, node):
-        from engine.utils import check_legal_name, get_legal_name
+        from common.utils import check_legal_name, get_legal_name
         node = node[self.name]
                     
         if 'params' in node:
@@ -1669,14 +1670,14 @@ class udf(ast_node):
         process_recursion(front)
         ccode += builtin_argstr + ') {\n'
         process_recursion(self.code_list)
-        self.context.udf += ccode + '\n'
+        self.ccode += ccode + '\n'
         ccode = ''
         if self.return_pattern == udf.ReturnPattern.elemental_return:
             ccode += f'auto {self.cname}_gettype = []('
             process_recursion(front[1:], True)
             ccode += ') {\n\tuint32_t _builtin_len = 0;\n' 
             process_recursion(self.code_list, True)
-            self.context.udf += ccode + '\n'
+            self.ccode += ccode + '\n'
             
     class ReturnPattern(Enum):
         bulk_return = auto()

reconstruct/expr.py

@@ -1,6 +1,6 @@
 from typing import Optional, Set
 
-from engine.types import *
+from common.types import *
 from reconstruct.ast import ast_node
 from reconstruct.storage import ColRef, Context
 
@@ -95,7 +95,7 @@ class expr(ast_node):
                              'stddev']
         
     def produce(self, node):
-        from engine.utils import enlist
+        from common.utils import enlist
         from reconstruct.ast import udf, projection
         
         if type(node) is dict:
@@ -412,7 +412,7 @@ class expr(ast_node):
                             exec(f'loc["{b}"] = lambda : "{b}"')
                 
             x = self.c_code if c_code is None else c_code
-            from engine.utils import escape_qoutes
+            from common.utils import escape_qoutes
             if decltypestr:
                 return eval('f\'' + escape_qoutes(self.udf_decltypecall) + '\'')
             self.sql.replace("'", "\\'")
@@ -447,7 +447,7 @@ class fastscan(expr):
                 self.requested_lens.add(segs[0])
         
     def produce(self, node):
-        from engine.utils import enlist
+        from common.utils import enlist
         if type(node) is dict:
             for key, val in node.items():
                 if key in self.operators:
@@ -468,7 +468,7 @@ class getrefs(expr):
         self.rec = None
         
     def produce(self, node):
-        from engine.utils import enlist
+        from common.utils import enlist
         if type(node) is dict:
             for key, val in node.items():
                 if key in self.operators:

reconstruct/new_expr.py

@@ -2,8 +2,8 @@ import abc
 from reconstruct.ast import ast_node
 from typing import Optional 
 from reconstruct.storage import Context, ColRef
-from engine.utils import enlist
-from engine.types import builtin_func, user_module_func, builtin_operators
+from common.utils import enlist
+from common.types import builtin_func, user_module_func, builtin_operators
 
 
 class expr_base(ast_node, metaclass = abc.ABCMeta):

reconstruct/storage.py

@@ -1,7 +1,7 @@
 from typing import Dict, List, Set
 
-from engine.types import *
-from engine.utils import CaseInsensitiveDict, base62uuid, enlist
+from common.types import *
+from common.utils import CaseInsensitiveDict, base62uuid, enlist
 
 
 class ColRef:
@@ -152,6 +152,7 @@ class Context:
         self.sql = ''  
         self.finalized = False
         self.udf = None
+        self.module_stubs = ''
         self.scans = []
         self.procs = []
         self.queries = []
@@ -167,7 +168,6 @@ class Context:
         self.tables : Set[TableInfo] = set()
         self.cols = []
         self.datasource = None
-        self.module_stubs = ''
         self.module_map = {}
         self.udf_map = dict()
         self.udf_agg_map = dict()
@@ -281,7 +281,13 @@ class Context:
         self.finalize_query()
     
     def finalize_udf(self):
-        if self.udf is not None:
+        if self.udf:
+            self.udf += '\n'.join([
+                u.ccode for u in self.udf_map.values()
+            ])
+            self.module_stubs = '\n'.join(
+                [m for m in self.module_map.values()
+            ])
             return (Context.udf_head
                 + self.module_stubs
                 + self.get_init_func()

server/libaquery.h

@@ -5,7 +5,9 @@
 	#define __AQUERY_ITC_USE_SEMPH__
 	#define THREADING
 #endif
-
+#ifdef THREADING
+	#include "threading.h"
+#endif
 #include <unordered_map>
 #include <chrono>
 #include <filesystem>
@@ -235,17 +237,7 @@ inline _This_Type* AQ_DupObject(_This_Type* __val) {
 		~A_Semaphore() { }
 	};
 #else
-	#ifdef _WIN32
-		class A_Semaphore {
-		private:
-			void* native_handle;
-		public:
-			A_Semaphore(bool);
-			void acquire();
-			void release();
-			~A_Semaphore();
-		};
-	#else
+	#ifndef _WIN32
 		#ifdef __APPLE__
 			#include <dispatch/dispatch.h>
 			class A_Semaphore {

server/monetdb_conn.cpp

@@ -11,6 +11,9 @@
 
 #ifdef _WIN32
     #include "winhelper.h"
+    #include "threading.h"
+    #undef min
+    #undef max
 #else 
     #include <dlfcn.h>
     #include <fcntl.h>

server/monetdb_ext.c

@@ -1,3 +1,4 @@
+#include "pch_msc.hpp"
 // Non-standard Extensions for MonetDBe, may break concurrency control!
 
 #include "monetdbe.h"

server/pch_msc.hpp

@@ -1,3 +1,3 @@
-#ifdef _MSCVER
+#if defined (_MSCVER) && defined(__cplusplus)
 #include "pch.hpp"
 #endif

server/threading.cpp

@@ -1,3 +1,4 @@
+#include "pch_msc.hpp"
 #include "threading.h"
 #include "libaquery.h"
 #include <thread>

server/threading.h

@@ -43,7 +43,7 @@ private:
 
 
 class A_Semphore;
-
+struct Context;
 class TriggerHost { 
 public:
     void* triggers;  

server/vector_type.hpp

@@ -55,6 +55,7 @@ public:
 	_Ty* container;
 	uint32_t size, capacity;
 	typedef _Ty* iterator_t;
+	typedef const _Ty* const_iterator;
 	typedef std::conditional_t<is_cstr<_Ty>(), astring_view, _Ty> value_t;
 	explicit vector_type(const uint32_t& size) : size(size), capacity(size) {
 		if (GC::scratch_space != nullptr) {
@@ -67,7 +68,7 @@ public:
 		}
 		// TODO: calloc for objects. 
 	}
-	explicit constexpr vector_type(std::initializer_list<_Ty> _l) {
+	constexpr vector_type(std::initializer_list<_Ty> _l) {
 		size = capacity = _l.size();
 		this->container = (_Ty*)malloc(sizeof(_Ty) * capacity);
 		_Ty* _container = this->container;
@@ -526,4 +527,13 @@ template <>
 vector_type<std::string_view>::vector_type(const char** container, uint32_t len, 
 		typename std::enable_if_t<true>*) noexcept;
 
+template<class T>
+struct vector_type_std : vector_type<T> {
+    vector_type_std() = default;
+    vector_type_std(vector_type<T> v) : vector_type<T>(v) {}
+    uint32_t size() const {
+        return vector_type<T>::size;
+    }
+};
+
 #endif

server/winhelper.h

@@ -15,7 +15,15 @@ struct SharedMemory
     SharedMemory(const char*);
     void FreeMemoryMap();
 };
-
+class A_Semaphore {
+private:
+	void* native_handle;
+public:
+	A_Semaphore(bool);
+	void acquire();
+	void release();
+	~A_Semaphore();
+};
 #endif // WIN32
 
 #endif // WINHELPER

tests/draw.a

@@ -0,0 +1,6 @@
+LOAD MODULE FROM "./libdraw.so" FUNCTIONS (draw(X:vecint, y:vecint) -> bool);
+ create table t(a int, b int);
+
+ insert into t(a, b) values (6,4), (8,8), (9,1), (5,6), (10,8), (1, 9), (4, 11);
+
+ select draw(a, b) from t;

validate.py

@@ -0,0 +1,11 @@
+def validate(y_hat):
+	y = [1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 1, 1, 1]
+
+	import math
+	errs = [yy - int(yyh) for yy, yyh in zip(y, y_hat.split(' '))]
+	err = 0
+	for e in errs:
+		err += abs(e)
+
+	err = err/len(y)
+	return err