Optimized: C++ Compilation, ~10x speedup
This commit is contained in:
@@ -1,3 +1,6 @@
|
|||||||
|
libaquery.a
|
||||||
|
aq
|
||||||
|
.cached
|
||||||
*.json
|
*.json
|
||||||
!sample_ast.json
|
!sample_ast.json
|
||||||
*.o
|
*.o
|
||||||
|
|||||||
@@ -1,11 +1,16 @@
|
|||||||
OS_SUPPORT =
|
OS_SUPPORT =
|
||||||
MonetDB_LIB =
|
MonetDB_LIB =
|
||||||
|
MonetDB_INC =
|
||||||
Threading =
|
Threading =
|
||||||
CXXFLAGS = --std=c++1z
|
CXXFLAGS = --std=c++1z
|
||||||
OPTFLAGS = -O3 -flto -march=native
|
OPTFLAGS = -O3
|
||||||
|
LINKFLAGS = -flto
|
||||||
SHAREDFLAGS = -shared -fPIC
|
SHAREDFLAGS = -shared -fPIC
|
||||||
|
LIBTOOL = $(shell $(CXX) --version | grep -q 'clang' && echo "libtool -static -o"|| echo "ar rcs")
|
||||||
|
USELIB_FLAG = $(shell $(CXX) --version | grep -q 'clang' && echo "-Wl,-force_load"|| echo "-Wl,--whole-archive")
|
||||||
|
LIBAQ_SRC = server/server.cpp server/monetdb_conn.cpp server/io.cpp
|
||||||
ifeq ($(PCH), 1)
|
ifeq ($(PCH), 1)
|
||||||
PCHFLAGS = -include server/aggregations.h
|
PCHFLAGS = -include server/pch.hpp
|
||||||
else
|
else
|
||||||
PCHFLAGS =
|
PCHFLAGS =
|
||||||
endif
|
endif
|
||||||
@@ -13,43 +18,61 @@ endif
|
|||||||
ifeq ($(OS),Windows_NT)
|
ifeq ($(OS),Windows_NT)
|
||||||
NULL_DEVICE = NUL
|
NULL_DEVICE = NUL
|
||||||
OS_SUPPORT += server/winhelper.cpp
|
OS_SUPPORT += server/winhelper.cpp
|
||||||
MonetDB_LIB += -Imonetdb/msvc msc-plugin/monetdbe.dll
|
MonetDB_LIB += msc-plugin/monetdbe.dll
|
||||||
|
MonetDB_INC += -Imonetdb/msvc
|
||||||
else
|
else
|
||||||
UNAME_S = $(shell uname -s)
|
UNAME_S = $(shell uname -s)
|
||||||
UNAME_M = $(shell uname -m)
|
UNAME_M = $(shell uname -m)
|
||||||
NULL_DEVICE = /dev/null
|
NULL_DEVICE = /dev/null
|
||||||
MonetDB_LIB =
|
MonetDB_LIB =
|
||||||
ifeq ($(UNAME_S),Darwin)
|
ifeq ($(UNAME_S),Darwin)
|
||||||
MonetDB_LIB += -L`brew --prefix monetdb`/lib
|
MonetDB_LIB += -L$(shell brew --prefix monetdb)/lib
|
||||||
|
MonetDB_INC += -I$(shell brew --prefix monetdb)/include/monetdb
|
||||||
ifneq ($(UNAME_M),arm64)
|
ifneq ($(UNAME_M),arm64)
|
||||||
OPTFLAGS += -march=native
|
OPTFLAGS += -march=native
|
||||||
endif
|
endif
|
||||||
else
|
else
|
||||||
OPTFLAGS += -march=native
|
OPTFLAGS += -march=native
|
||||||
|
MonetDB_INC += -I/usr/local/include/monetdb -I/usr/include/monetdb
|
||||||
endif
|
endif
|
||||||
MonetDB_LIB += -I/usr/local/include/monetdb -I/usr/include/monetdb -lmonetdbe
|
MonetDB_LIB += -lmonetdbe
|
||||||
endif
|
endif
|
||||||
|
|
||||||
ifeq ($(THREADING),1)
|
ifeq ($(THREADING),1)
|
||||||
Threading += server/threading.cpp -DTHREADING
|
LIBAQ_SRC += server/threading.cpp
|
||||||
|
Threading += -DTHREADING
|
||||||
endif
|
endif
|
||||||
pch:
|
|
||||||
$(CXX) -x c++-header server/aggregations.h -o server/aggregations.h.pch
|
|
||||||
info:
|
info:
|
||||||
$(info $(OS_SUPPORT))
|
$(info $(OS_SUPPORT))
|
||||||
$(info $(OS))
|
$(info $(OS))
|
||||||
$(info $(Threading))
|
$(info $(Threading))
|
||||||
$(info "test")
|
$(info "test")
|
||||||
|
$(info $(LIBTOOL))
|
||||||
|
$(info $(MonetDB_INC))
|
||||||
$(info $(CXX))
|
$(info $(CXX))
|
||||||
|
pch:
|
||||||
|
$(CXX) -x c++-header server/pch.hpp $(MonetDB_INC) $(OPTFLAGS) $(CXXFLAGS) $(Threading)
|
||||||
libaquery.a:
|
libaquery.a:
|
||||||
$(CXX) -c server/server.cpp server/io.cpp server/table.cpp $(OS_SUPPORT) $(Threading) $(OPTFLAGS) $(CXXFLAGS) -o server.bin
|
$(CXX) -c $(PCHFLAGS) $(LIBAQ_SRC) $(MonetDB_INC) $(MonetDB_LIB) $(OS_SUPPORT) $(Threading) $(OPTFLAGS) $(LINKFLAGS) $(CXXFLAGS) &&\
|
||||||
|
$(LIBTOOL) libaquery.a *.o &&\
|
||||||
|
ranlib libaquery.a
|
||||||
|
|
||||||
server.bin:
|
server.bin:
|
||||||
$(CXX) server/server.cpp server/io.cpp server/table.cpp $(OS_SUPPORT) $(Threading) $(OPTFLAGS) $(CXXFLAGS) -o server.bin
|
$(CXX) $(LIBAQ_SRC) $(LINKFLAGS) $(OS_SUPPORT) $(Threading) $(MonetDB_INC) $(MonetDB_LIB) $(OPTFLAGS) $(CXXFLAGS) -o server.bin
|
||||||
|
launcher:
|
||||||
|
$(CXX) -D__AQ_BUILD_LAUNCHER__ $(LIBAQ_SRC) $(LINKFLAGS) $(OS_SUPPORT) $(Threading) $(MonetDB_INC) $(MonetDB_LIB) $(OPTFLAGS) $(CXXFLAGS) -o aq
|
||||||
server.so:
|
server.so:
|
||||||
# $(CXX) server/server.cpp server/monetdb_conn.cpp -fPIC -shared $(OS_SUPPORT) monetdb/msvc/monetdbe.dll --std=c++1z -O3 -march=native -o server.so -I./monetdb/msvc
|
# $(CXX) server/server.cpp server/monetdb_conn.cpp -fPIC -shared $(OS_SUPPORT) monetdb/msvc/monetdbe.dll --std=c++1z -O3 -march=native -o server.so -I./monetdb/msvc
|
||||||
$(CXX) $(SHAREDFLAGS) server/server.cpp server/io.cpp server/table.cpp $(OS_SUPPORT) server/monetdb_conn.cpp $(Threading) $(MonetDB_LIB) $(OPTFLAGS) $(CXXFLAGS) -o server.so
|
$(CXX) $(SHAREDFLAGS) $(PCHFLAGS) $(LIBAQ_SRC) $(OS_SUPPORT) $(Threading) $(MonetDB_INC) $(MonetDB_LIB) $(OPTFLAGS) $(LINKFLAGS) $(CXXFLAGS) -o server.so
|
||||||
|
server_uselib:
|
||||||
|
$(CXX) $(SHAREDFLAGS) $(USELIB_FLAG),libaquery.a $(MonetDB_LIB) $(OPTFLAGS) $(LINKFLAGS) $(CXXFLAGS) -o server.so
|
||||||
|
|
||||||
snippet:
|
snippet:
|
||||||
$(CXX) $(SHAREDFLAGS) $(PCHFLAGS) out.cpp server/server.cpp server/monetdb_conn.cpp server/table.cpp server/io.cpp $(MonetDB_LIB) $(OPTFLAGS) $(CXXFLAGS) -o dll.so
|
$(CXX) $(SHAREDFLAGS) $(PCHFLAGS) out.cpp $(LIBAQ_SRC) $(MonetDB_INC) $(MonetDB_LIB) $(Threading) $(OPTFLAGS) $(LINKFLAGS) $(CXXFLAGS) -o dll.so
|
||||||
|
snippet_uselib:
|
||||||
|
$(CXX) $(SHAREDFLAGS) $(PCHFLAGS) out.cpp libaquery.a $(MonetDB_INC) $(Threading) $(MonetDB_LIB) $(OPTFLAGS) $(LINKFLAGS) $(CXXFLAGS) -o dll.so
|
||||||
|
|
||||||
docker:
|
docker:
|
||||||
docker build -t aquery .
|
docker build -t aquery .
|
||||||
|
|
||||||
|
|||||||
@@ -50,7 +50,8 @@ AQuery++ Database is a cross-platform, In-Memory Column-Store Database that inco
|
|||||||
- [x] Limitation: putting ColRefs back to monetdb.
|
- [x] Limitation: putting ColRefs back to monetdb.
|
||||||
- [ ] Limitation: String operations and Date/Time data type.
|
- [ ] Limitation: String operations and Date/Time data type.
|
||||||
- [ ] C++ Meta-Programming: Eliminate template recursions as much as possible.
|
- [ ] C++ Meta-Programming: Eliminate template recursions as much as possible.
|
||||||
|
- [ ] Limitation: Date and Time, String operations, Funcs in groupby agg.
|
||||||
|
|
||||||
# Installation
|
# Installation
|
||||||
## Requirements
|
## Requirements
|
||||||
1. Recent version of Linux, Windows or MacOS, with recent C++ compiler that has C++17 (1z) support.
|
1. Recent version of Linux, Windows or MacOS, with recent C++ compiler that has C++17 (1z) support.
|
||||||
|
|||||||
@@ -11,19 +11,6 @@ cygroot = 'c:/msys64/usr/bin'
|
|||||||
msbuildroot = 'd:/gg/vs22/MSBuild/Current/Bin'
|
msbuildroot = 'd:/gg/vs22/MSBuild/Current/Bin'
|
||||||
__config_initialized__ = False
|
__config_initialized__ = False
|
||||||
|
|
||||||
class Build_Config:
|
|
||||||
def __init__(self) -> None:
|
|
||||||
self.OptimizationLv = '4' # [O0, O1, O2, O3, Ofast]
|
|
||||||
self.Platform = 'x64'
|
|
||||||
self.PCH = ''
|
|
||||||
self.StaticLib = ''
|
|
||||||
self.fLTO = True
|
|
||||||
self.fmarchnative = True
|
|
||||||
|
|
||||||
def configure(self):
|
|
||||||
pass
|
|
||||||
|
|
||||||
|
|
||||||
def init_config():
|
def init_config():
|
||||||
global __config_initialized__, os_platform
|
global __config_initialized__, os_platform
|
||||||
## SETUP ENVIRONMENT VARIABLES
|
## SETUP ENVIRONMENT VARIABLES
|
||||||
|
|||||||
@@ -0,0 +1,157 @@
|
|||||||
|
from dataclasses import dataclass
|
||||||
|
from enum import Enum, auto
|
||||||
|
from tabnanny import check
|
||||||
|
from aquery_config import *
|
||||||
|
import sys
|
||||||
|
import os
|
||||||
|
import subprocess
|
||||||
|
import hashlib
|
||||||
|
import pickle
|
||||||
|
from engine.utils import nullstream
|
||||||
|
from typing import Dict, Optional, Set, Union
|
||||||
|
@dataclass
|
||||||
|
class checksums:
|
||||||
|
libaquery_a : Optional[Union[bytes, bool]] = None
|
||||||
|
pch_hpp_gch : Optional[Union[bytes, bool]] = None
|
||||||
|
server : Optional[Union[bytes, bool]] = None
|
||||||
|
sources : Union[Dict[str, bytes], bool] = None
|
||||||
|
def calc(self, libaquery_a = 'libaquery.a',
|
||||||
|
pch_hpp_gch = 'server/pch.hpp.gch',
|
||||||
|
server = 'server.so'
|
||||||
|
):
|
||||||
|
for key in self.__dict__.keys():
|
||||||
|
try:
|
||||||
|
with open(eval(key), 'rb') as file:
|
||||||
|
self.__dict__[key] = hashlib.md5(file.read()).digest()
|
||||||
|
except (FileNotFoundError, NameError):
|
||||||
|
pass
|
||||||
|
sources = [*build_manager.headerfiles, *build_manager.sourcefiles]
|
||||||
|
self.sources = dict()
|
||||||
|
for key in sources:
|
||||||
|
try:
|
||||||
|
with open(key, 'rb') as file:
|
||||||
|
self.sources[key] = hashlib.md5(file.read()).digest()
|
||||||
|
except FileNotFoundError:
|
||||||
|
print('missing component: ' + key)
|
||||||
|
self.sources[key] = None
|
||||||
|
def __ne__(self, __o: 'checksums') -> 'checksums':
|
||||||
|
ret = checksums()
|
||||||
|
for key in self.__dict__.keys():
|
||||||
|
try:
|
||||||
|
ret.__dict__[key] = self.__dict__[key] != __o.__dict__[key]
|
||||||
|
except KeyError:
|
||||||
|
ret.__dict__[key] = True
|
||||||
|
return ret
|
||||||
|
|
||||||
|
def __eq__(self, __o: 'checksums') -> 'checksums':
|
||||||
|
ret = checksums()
|
||||||
|
for key in self.__dict__.keys():
|
||||||
|
try:
|
||||||
|
ret.__dict__[key] = self.__dict__[key] == __o.__dict__[key]
|
||||||
|
except KeyError:
|
||||||
|
ret.__dict__[key] = False
|
||||||
|
return ret
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
class build_manager:
|
||||||
|
sourcefiles = ['server/server.cpp', 'server/io.cpp',
|
||||||
|
'server/monetdb_conn.cpp', 'server/threading.cpp',
|
||||||
|
'server/winhelper.cpp'
|
||||||
|
]
|
||||||
|
headerfiles = ['server/aggregations.h', 'server/hasher.h', 'server/io.h',
|
||||||
|
'server/libaquery.h', 'server/monetdb_conn.h', 'server/pch.hpp',
|
||||||
|
'server/table.h', 'server/threading.h', 'server/types.h', 'server/utils.h',
|
||||||
|
'server/winhelper.h', 'server/gc.hpp', 'server/vector_type.hpp',
|
||||||
|
'server/table_ext_monetdb.hpp'
|
||||||
|
]
|
||||||
|
|
||||||
|
class DriverBase:
|
||||||
|
def __init__(self, mgr : 'build_manager') -> None:
|
||||||
|
self.mgr = mgr
|
||||||
|
self.build_cmd = ['']
|
||||||
|
def libaquery_a(self) :
|
||||||
|
pass
|
||||||
|
def pch(self):
|
||||||
|
pass
|
||||||
|
def build(self, stdout = sys.stdout, stderr = sys.stderr):
|
||||||
|
for c in self.build_cmd:
|
||||||
|
subprocess.call(c, stdout = stdout, stderr = stderr)
|
||||||
|
|
||||||
|
class MakefileDriver(DriverBase):
|
||||||
|
def __init__(self, mgr : 'build_manager') -> None:
|
||||||
|
super().__init__(mgr)
|
||||||
|
os.environ['PCH'] = f'{mgr.PCH}'
|
||||||
|
os.environ['CXX'] = mgr.cxx if mgr.cxx else 'c++'
|
||||||
|
|
||||||
|
def libaquery_a(self):
|
||||||
|
self.build_cmd = [['rm', 'libaquery.a'],['make', 'libaquery.a']]
|
||||||
|
return self.build()
|
||||||
|
def pch(self):
|
||||||
|
self.build_cmd = [['rm', 'server/pch.hpp.gch'], ['make', 'pch']]
|
||||||
|
return self.build()
|
||||||
|
def server(self):
|
||||||
|
if self.mgr.StaticLib:
|
||||||
|
self.build_cmd = [['rm', 'server.so'], ['make', 'server_uselib']]
|
||||||
|
else:
|
||||||
|
self.build_cmd = [['rm', 'server.so'], ['make', 'server.so']]
|
||||||
|
return self.build()
|
||||||
|
|
||||||
|
def snippet(self):
|
||||||
|
if self.mgr.StaticLib:
|
||||||
|
self.build_cmd = [['make', 'snippet_uselib']]
|
||||||
|
else:
|
||||||
|
self.build_cmd = [['rm', 'dll.so'], ['make', 'snippet']]
|
||||||
|
return self.build()
|
||||||
|
|
||||||
|
class PythonDriver(DriverBase):
|
||||||
|
def __init__(self, mgr : 'build_manager') -> None:
|
||||||
|
super().__init__(mgr)
|
||||||
|
|
||||||
|
class Driver(Enum):
|
||||||
|
Makefile = auto()
|
||||||
|
Python = auto()
|
||||||
|
CMake = auto()
|
||||||
|
MSBuild = auto()
|
||||||
|
|
||||||
|
def __init__(self) -> None:
|
||||||
|
self.method = 'make'
|
||||||
|
self.cxx = ''
|
||||||
|
self.OptimizationLv = '4' # [O0, O1, O2, O3, Ofast]
|
||||||
|
self.Platform = 'x64'
|
||||||
|
self.PCH = 1
|
||||||
|
self.StaticLib = 1
|
||||||
|
self.fLTO = 1
|
||||||
|
self.fmarchnative = 1
|
||||||
|
self.driver = build_manager.MakefileDriver(self)
|
||||||
|
self.cache_status = checksums()
|
||||||
|
|
||||||
|
def build_dll(self):
|
||||||
|
return self.driver.snippet()
|
||||||
|
|
||||||
|
def build_caches(self, force = False):
|
||||||
|
cached = checksums()
|
||||||
|
current = checksums()
|
||||||
|
current.calc()
|
||||||
|
try:
|
||||||
|
with open('.cached', 'rb') as cache_sig:
|
||||||
|
cached = pickle.loads(cache_sig.read())
|
||||||
|
except FileNotFoundError:
|
||||||
|
pass
|
||||||
|
self.cache_status = current != cached
|
||||||
|
|
||||||
|
if force or self.cache_status.sources:
|
||||||
|
self.driver.pch()
|
||||||
|
self.driver.libaquery_a()
|
||||||
|
self.driver.server()
|
||||||
|
else:
|
||||||
|
if self.cache_status.libaquery_a:
|
||||||
|
self.driver.libaquery_a()
|
||||||
|
if self.cache_status.pch_hpp_gch:
|
||||||
|
self.driver.pch()
|
||||||
|
if self.cache_status.server:
|
||||||
|
self.driver.server()
|
||||||
|
current.calc()
|
||||||
|
with open('.cached', 'wb') as cache_sig:
|
||||||
|
cache_sig.write(pickle.dumps(current))
|
||||||
|
|
||||||
+4
-2
@@ -81,11 +81,13 @@ class _Counter:
|
|||||||
|
|
||||||
import re
|
import re
|
||||||
ws = re.compile(r'\s+')
|
ws = re.compile(r'\s+')
|
||||||
|
import os
|
||||||
|
|
||||||
def add_dll_dir(dll: str):
|
def add_dll_dir(dll: str):
|
||||||
import sys, os
|
import sys
|
||||||
if sys.version_info.major >= 3 and sys.version_info.minor >7 and os.name == 'nt':
|
if sys.version_info.major >= 3 and sys.version_info.minor >7 and os.name == 'nt':
|
||||||
os.add_dll_directory(dll)
|
os.add_dll_directory(dll)
|
||||||
else:
|
else:
|
||||||
os.environ['PATH'] = os.path.abspath(dll) + os.pathsep + os.environ['PATH']
|
os.environ['PATH'] = os.path.abspath(dll) + os.pathsep + os.environ['PATH']
|
||||||
|
|
||||||
|
nullstream = open(os.devnull, 'w')
|
||||||
|
|||||||
@@ -105,8 +105,8 @@ import ctypes
|
|||||||
import numpy as np
|
import numpy as np
|
||||||
from engine.utils import ws
|
from engine.utils import ws
|
||||||
from engine.utils import add_dll_dir
|
from engine.utils import add_dll_dir
|
||||||
|
from engine.utils import nullstream
|
||||||
nullstream = open(os.devnull, 'w')
|
from build import build_manager
|
||||||
|
|
||||||
## CLASSES BEGIN
|
## CLASSES BEGIN
|
||||||
class RunType(enum.Enum):
|
class RunType(enum.Enum):
|
||||||
@@ -179,6 +179,7 @@ class PromptState():
|
|||||||
init : Callable[['PromptState'], None] = lambda _:None
|
init : Callable[['PromptState'], None] = lambda _:None
|
||||||
stmts = ['']
|
stmts = ['']
|
||||||
payloads = {}
|
payloads = {}
|
||||||
|
buildmgr : Optional[build_manager]= None
|
||||||
## CLASSES END
|
## CLASSES END
|
||||||
|
|
||||||
## FUNCTIONS BEGIN
|
## FUNCTIONS BEGIN
|
||||||
@@ -263,14 +264,15 @@ def init_prompt() -> PromptState:
|
|||||||
aquery_config.init_config()
|
aquery_config.init_config()
|
||||||
|
|
||||||
state = PromptState()
|
state = PromptState()
|
||||||
if aquery_config.rebuild_backend:
|
# if aquery_config.rebuild_backend:
|
||||||
try:
|
# try:
|
||||||
os.remove(state.server_bin)
|
# os.remove(state.server_bin)
|
||||||
except Exception as e:
|
# except Exception as e:
|
||||||
print(type(e), e)
|
# print(type(e), e)
|
||||||
subprocess.call(['make', "info"])
|
# subprocess.call(['make', "info"])
|
||||||
subprocess.call(['make', state.server_bin], stdout=nullstream)
|
# subprocess.call(['make', state.server_bin], stdout=nullstream)
|
||||||
|
state.buildmgr = build_manager()
|
||||||
|
state.buildmgr.build_caches()
|
||||||
state.cfg = Config(state.server_mode)
|
state.cfg = Config(state.server_mode)
|
||||||
|
|
||||||
if state.server_mode == RunType.IPC:
|
if state.server_mode == RunType.IPC:
|
||||||
@@ -337,7 +339,7 @@ def prompt(running = lambda:True, next = input, state = None):
|
|||||||
if q == 'exec': # generate build and run (AQuery Engine)
|
if q == 'exec': # generate build and run (AQuery Engine)
|
||||||
state.cfg.backend_type = Backend_Type.BACKEND_AQuery.value
|
state.cfg.backend_type = Backend_Type.BACKEND_AQuery.value
|
||||||
cxt = engine.exec(state.stmts, cxt, keep)
|
cxt = engine.exec(state.stmts, cxt, keep)
|
||||||
if subprocess.call(['make', 'snippet'], stdout = nullstream) == 0:
|
if state.buildmgr.build_dll() == 0:
|
||||||
state.set_ready()
|
state.set_ready()
|
||||||
continue
|
continue
|
||||||
elif q.startswith('echo '):
|
elif q.startswith('echo '):
|
||||||
@@ -377,7 +379,7 @@ def prompt(running = lambda:True, next = input, state = None):
|
|||||||
with open('out.cpp', 'wb') as outfile:
|
with open('out.cpp', 'wb') as outfile:
|
||||||
outfile.write((cxt.finalize()).encode('utf-8'))
|
outfile.write((cxt.finalize()).encode('utf-8'))
|
||||||
if build_this:
|
if build_this:
|
||||||
subprocess.call(['make', 'snippet'], stdout = nullstream)
|
state.buildmgr.build_dll()
|
||||||
state.cfg.has_dll = 1
|
state.cfg.has_dll = 1
|
||||||
else:
|
else:
|
||||||
state.cfg.has_dll = 0
|
state.cfg.has_dll = 0
|
||||||
|
|||||||
+1
-1
@@ -296,7 +296,7 @@ class select_into(ast_node):
|
|||||||
raise Exception('No out_table found.')
|
raise Exception('No out_table found.')
|
||||||
else:
|
else:
|
||||||
self.context.headers.add('"./server/table_ext_monetdb.hpp"')
|
self.context.headers.add('"./server/table_ext_monetdb.hpp"')
|
||||||
self.ccode = f'{self.parent.out_table.table_name}->monetdb_append_table(cxt->alt_server, \"{node}\");'
|
self.ccode = f'{self.parent.out_table.contextname_cpp}->monetdb_append_table(cxt->alt_server, \"{node}\");'
|
||||||
|
|
||||||
def produce_sql(self, node):
|
def produce_sql(self, node):
|
||||||
self.sql = f' INTO {node}'
|
self.sql = f' INTO {node}'
|
||||||
|
|||||||
+3
-1
@@ -1,5 +1,6 @@
|
|||||||
#ifndef _AQUERY_H
|
#ifndef _AQUERY_H
|
||||||
#define _AQUERY_H
|
#define _AQUERY_H
|
||||||
|
|
||||||
enum Log_level {
|
enum Log_level {
|
||||||
LOG_INFO,
|
LOG_INFO,
|
||||||
LOG_ERROR,
|
LOG_ERROR,
|
||||||
@@ -11,6 +12,7 @@ enum Backend_Type {
|
|||||||
BACKEND_MonetDB,
|
BACKEND_MonetDB,
|
||||||
BACKEND_MariaDB
|
BACKEND_MariaDB
|
||||||
};
|
};
|
||||||
|
|
||||||
struct Config{
|
struct Config{
|
||||||
int running, new_query, server_mode,
|
int running, new_query, server_mode,
|
||||||
backend_type, has_dll, n_buffers;
|
backend_type, has_dll, n_buffers;
|
||||||
@@ -71,5 +73,5 @@ extern void Afree(void * mem);
|
|||||||
extern void register_memory(void* ptr, deallocator_t deallocator);
|
extern void register_memory(void* ptr, deallocator_t deallocator);
|
||||||
extern void init_session(Context* cxt);
|
extern void init_session(Context* cxt);
|
||||||
|
|
||||||
#define __AQ_NO_SESSION__ void init_session(Context*) {}
|
#define __AQ_NO_SESSION__ __AQEXPORT__(void) init_session(Context*) {}
|
||||||
#endif
|
#endif
|
||||||
+2
-3
@@ -25,7 +25,6 @@ void register_memory(void* ptr, deallocator_t deallocator){
|
|||||||
memmap->operator[](ptr) = deallocator;
|
memmap->operator[](ptr) = deallocator;
|
||||||
}
|
}
|
||||||
|
|
||||||
void init_session(Context* cxt){
|
__AQEXPORT__(void) init_session(Context* cxt){
|
||||||
session = &cxt->current;
|
session = &cxt->current;
|
||||||
// session->memory_map = new std::unordered_map<void*, deallocator_t>();
|
}
|
||||||
}
|
|
||||||
|
|||||||
+1
-1
@@ -1,5 +1,5 @@
|
|||||||
#include "aquery.h"
|
#include "aquery.h"
|
||||||
__AQ_NO_SESSION__
|
// __AQ_NO_SESSION__
|
||||||
#include "../server/table.h"
|
#include "../server/table.h"
|
||||||
|
|
||||||
__AQEXPORT__(ColRef<float>) mulvec(int a, ColRef<float> b){
|
__AQEXPORT__(ColRef<float>) mulvec(int a, ColRef<float> b){
|
||||||
|
|||||||
+2
-2
@@ -1,6 +1,6 @@
|
|||||||
debug:
|
debug:
|
||||||
g++ -g3 -O0 server/server.cpp server/table.cpp -o a.out -Wall -Wextra -Wpedantic -lpthread
|
g++ -g3 -O0 server/server.cpp server/io.cpp -o a.out -Wall -Wextra -Wpedantic -lpthread
|
||||||
|
|
||||||
test:
|
test:
|
||||||
g++ --std=c++1z -g3 -O0 server.cpp table.cpp -o a.out -Wall -Wextra -Wpedantic -lpthread
|
g++ --std=c++1z -g3 -O0 server.cpp io.cpp -o a.out -Wall -Wextra -Wpedantic -lpthread
|
||||||
|
|
||||||
|
|||||||
+6
-7
@@ -1,29 +1,28 @@
|
|||||||
#pragma once
|
#pragma once
|
||||||
#include <vector>
|
#include <vector_type>
|
||||||
#include <utility>
|
#include <utility>
|
||||||
#include <thread>
|
#include <thread>
|
||||||
#include <chrono>
|
#include <chrono>
|
||||||
class GC {
|
class GC {
|
||||||
template<class T>
|
template<class T>
|
||||||
using vector = std::vector<T>;
|
using vector = vector_type<T>;
|
||||||
template<class ...T>
|
template<class ...T>
|
||||||
using tuple = std::tuple<T...>;
|
using tuple = std::tuple<T...>;
|
||||||
size_t current_size, max_size, interval, forced_clean;
|
size_t current_size, max_size, interval, forced_clean;
|
||||||
bool running, alive;
|
bool running, alive;
|
||||||
// ptr, dealloc, ref, sz
|
// ptr, dealloc, ref, sz
|
||||||
vector<tuple<void*, void (*)(void*), uint32_t, uint32_t>> q;
|
vector<tuple<void*, void (*)(void*)>> q;
|
||||||
std::thread handle;
|
std::thread handle;
|
||||||
void gc()
|
void gc()
|
||||||
{
|
{
|
||||||
|
|
||||||
}
|
}
|
||||||
template <class T>
|
void reg(void* v, uint32_t ref, uint32_t sz,
|
||||||
void reg(T* v, uint32_t ref, uint32_t sz,
|
void(*f)(void*) = [](void* v) {free (v); }) {
|
||||||
void(*f)(void*) = [](void* v) {delete[] ((T*)v); }) {
|
|
||||||
current_size += sz;
|
current_size += sz;
|
||||||
if (current_size > max_size)
|
if (current_size > max_size)
|
||||||
gc();
|
gc();
|
||||||
q.push_back({ v, f, ref, sz });
|
q.push_back({ v, f });
|
||||||
}
|
}
|
||||||
void daemon() {
|
void daemon() {
|
||||||
using namespace std::chrono;
|
using namespace std::chrono;
|
||||||
|
|||||||
+86
-1
@@ -8,4 +8,89 @@ void setgbuf(char* buf){
|
|||||||
gbuf = b;
|
gbuf = b;
|
||||||
else
|
else
|
||||||
gbuf = buf;
|
gbuf = buf;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#include "table.h"
|
||||||
|
|
||||||
|
|
||||||
|
#ifdef __SIZEOF_INT128__
|
||||||
|
template <>
|
||||||
|
void print<__int128_t>(const __int128_t& v, const char* delimiter){
|
||||||
|
char s[41];
|
||||||
|
s[40] = 0;
|
||||||
|
std::cout<< get_int128str(v, s+40)<< delimiter;
|
||||||
|
}
|
||||||
|
template <>
|
||||||
|
void print<__uint128_t>(const __uint128_t&v, const char* delimiter){
|
||||||
|
char s[41];
|
||||||
|
s[40] = 0;
|
||||||
|
std::cout<< get_uint128str(v, s+40) << delimiter;
|
||||||
|
}
|
||||||
|
std::ostream& operator<<(std::ostream& os, __int128 & v)
|
||||||
|
{
|
||||||
|
print(v);
|
||||||
|
return os;
|
||||||
|
}
|
||||||
|
std::ostream& operator<<(std::ostream& os, __uint128_t & v)
|
||||||
|
{
|
||||||
|
print(v);
|
||||||
|
return os;
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
|
||||||
|
template <>
|
||||||
|
void print<bool>(const bool&v, const char* delimiter){
|
||||||
|
std::cout<< (v?"true":"false") << delimiter;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
#include <chrono>
|
||||||
|
#include <ctime>
|
||||||
|
namespace types {
|
||||||
|
using namespace std;
|
||||||
|
using namespace chrono;
|
||||||
|
string date_t::toString() const {
|
||||||
|
uint32_t curr_v = val;
|
||||||
|
tm;
|
||||||
|
time_t;
|
||||||
|
return string() + string("/") + string() + string("/") + string();
|
||||||
|
}
|
||||||
|
string time_t::toString() const {
|
||||||
|
uint32_t curr_v = val;
|
||||||
|
|
||||||
|
return string() + string("/") + string() + string("/") + string();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#include "utils.h"
|
||||||
|
#include <random>
|
||||||
|
using std::string;
|
||||||
|
string base62uuid(int l = 8) {
|
||||||
|
using namespace std;
|
||||||
|
constexpr static const char* base62alp = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ";
|
||||||
|
static mt19937_64 engine(chrono::system_clock::now().time_since_epoch().count());
|
||||||
|
static uniform_int_distribution<uint64_t> u(0x10000, 0xfffff);
|
||||||
|
uint64_t uuid = (u(engine) << 32ull) + (chrono::system_clock::now().time_since_epoch().count() & 0xffffffff);
|
||||||
|
printf("%p\n", uuid);
|
||||||
|
string ret;
|
||||||
|
while (uuid && l-- >= 0) {
|
||||||
|
ret = string("") + base62alp[uuid % 62] + ret;
|
||||||
|
uuid /= 62;
|
||||||
|
}
|
||||||
|
return ret;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
template<typename _Ty>
|
||||||
|
inline void vector_type<_Ty>::out(uint32_t n, const char* sep) const
|
||||||
|
{
|
||||||
|
n = n > size ? size : n;
|
||||||
|
std::cout << '(';
|
||||||
|
{
|
||||||
|
uint32_t i = 0;
|
||||||
|
for (; i < n - 1; ++i)
|
||||||
|
std::cout << this->operator[](i) << sep;
|
||||||
|
std::cout << this->operator[](i);
|
||||||
|
}
|
||||||
|
std::cout << ')';
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,17 @@
|
|||||||
|
#pragma once
|
||||||
|
|
||||||
|
#include "libaquery.h"
|
||||||
|
#include "aggregations.h"
|
||||||
|
#include <iostream>
|
||||||
|
#include <chrono>
|
||||||
|
#include "monetdb_conn.h"
|
||||||
|
#include "monetdbe.h"
|
||||||
|
#include "threading.h"
|
||||||
|
#include "../csv.h"
|
||||||
|
#ifdef _WIN32
|
||||||
|
#include "winhelper.h"
|
||||||
|
#else
|
||||||
|
#include <dlfcn.h>
|
||||||
|
#include <fcntl.h>
|
||||||
|
#include <sys/mman.h>
|
||||||
|
#endif
|
||||||
+7
-6
@@ -1,7 +1,6 @@
|
|||||||
#include "../csv.h"
|
#include "../csv.h"
|
||||||
#include <iostream>
|
#include <iostream>
|
||||||
#include <string>
|
#include <string>
|
||||||
//#include <thread>
|
|
||||||
#include <chrono>
|
#include <chrono>
|
||||||
|
|
||||||
#include "libaquery.h"
|
#include "libaquery.h"
|
||||||
@@ -69,14 +68,17 @@ __AQEXPORT__(bool) have_hge(){
|
|||||||
return false;
|
return false;
|
||||||
#endif
|
#endif
|
||||||
}
|
}
|
||||||
|
|
||||||
Context::Context() {
|
Context::Context() {
|
||||||
current.memory_map = new std::unordered_map<void*, deallocator_t>;
|
current.memory_map = new std::unordered_map<void*, deallocator_t>;
|
||||||
init_session();
|
init_session();
|
||||||
}
|
}
|
||||||
|
|
||||||
Context::~Context() {
|
Context::~Context() {
|
||||||
auto memmap = (std::unordered_map<void*, deallocator_t>*) this->current.memory_map;
|
auto memmap = (std::unordered_map<void*, deallocator_t>*) this->current.memory_map;
|
||||||
delete memmap;
|
delete memmap;
|
||||||
}
|
}
|
||||||
|
|
||||||
void Context::init_session(){
|
void Context::init_session(){
|
||||||
if (log_level == LOG_INFO){
|
if (log_level == LOG_INFO){
|
||||||
memset(&(this->current.stats), 0, sizeof(Session::Statistic));
|
memset(&(this->current.stats), 0, sizeof(Session::Statistic));
|
||||||
@@ -226,7 +228,7 @@ int dll_main(int argc, char** argv, Context* cxt){
|
|||||||
|
|
||||||
int launcher(int argc, char** argv){
|
int launcher(int argc, char** argv){
|
||||||
std::string str = " ";
|
std::string str = " ";
|
||||||
for (int i = 0; i < argc; i++){
|
for (int i = 1; i < argc; i++){
|
||||||
str += argv[i];
|
str += argv[i];
|
||||||
str += " ";
|
str += " ";
|
||||||
}
|
}
|
||||||
@@ -235,6 +237,9 @@ int launcher(int argc, char** argv){
|
|||||||
}
|
}
|
||||||
|
|
||||||
extern "C" int __DLLEXPORT__ main(int argc, char** argv) {
|
extern "C" int __DLLEXPORT__ main(int argc, char** argv) {
|
||||||
|
#ifdef __AQ_BUILD_LAUNCHER__
|
||||||
|
return launcher(argc, argv);
|
||||||
|
#endif
|
||||||
puts("running");
|
puts("running");
|
||||||
Context* cxt = new Context();
|
Context* cxt = new Context();
|
||||||
cxt->log("%d %s\n", argc, argv[1]);
|
cxt->log("%d %s\n", argc, argv[1]);
|
||||||
@@ -248,11 +253,7 @@ extern "C" int __DLLEXPORT__ main(int argc, char** argv) {
|
|||||||
if (argc < 0)
|
if (argc < 0)
|
||||||
return dll_main(argc, argv, cxt);
|
return dll_main(argc, argv, cxt);
|
||||||
else if (argc <= 1)
|
else if (argc <= 1)
|
||||||
#ifdef __AQ__TESTING__
|
|
||||||
return test_main();
|
return test_main();
|
||||||
#else
|
|
||||||
return launcher(argc, argv);
|
|
||||||
#endif
|
|
||||||
else
|
else
|
||||||
shmname = argv[1];
|
shmname = argv[1];
|
||||||
SharedMemory shm = SharedMemory(shmname);
|
SharedMemory shm = SharedMemory(shmname);
|
||||||
|
|||||||
@@ -255,7 +255,7 @@
|
|||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
<Media Include="..\out.cpp" />
|
<Media Include="..\out.cpp" />
|
||||||
<ClCompile Include="monetdb_conn.cpp" />
|
<ClCompile Include="monetdb_conn.cpp" />
|
||||||
<ClCompile Include="table.cpp" />
|
<ClCompile Include="io.cpp" />
|
||||||
<ClCompile Include="server.cpp" />
|
<ClCompile Include="server.cpp" />
|
||||||
<ClCompile Include="types.cpp" />
|
<ClCompile Include="types.cpp" />
|
||||||
<ClCompile Include="utils.cpp" />
|
<ClCompile Include="utils.cpp" />
|
||||||
|
|||||||
@@ -1,32 +0,0 @@
|
|||||||
#include "table.h"
|
|
||||||
|
|
||||||
|
|
||||||
#ifdef __SIZEOF_INT128__
|
|
||||||
template <>
|
|
||||||
void print<__int128_t>(const __int128_t& v, const char* delimiter){
|
|
||||||
char s[41];
|
|
||||||
s[40] = 0;
|
|
||||||
std::cout<< get_int128str(v, s+40)<< delimiter;
|
|
||||||
}
|
|
||||||
template <>
|
|
||||||
void print<__uint128_t>(const __uint128_t&v, const char* delimiter){
|
|
||||||
char s[41];
|
|
||||||
s[40] = 0;
|
|
||||||
std::cout<< get_uint128str(v, s+40) << delimiter;
|
|
||||||
}
|
|
||||||
std::ostream& operator<<(std::ostream& os, __int128 & v)
|
|
||||||
{
|
|
||||||
print(v);
|
|
||||||
return os;
|
|
||||||
}
|
|
||||||
std::ostream& operator<<(std::ostream& os, __uint128_t & v)
|
|
||||||
{
|
|
||||||
print(v);
|
|
||||||
return os;
|
|
||||||
}
|
|
||||||
#endif
|
|
||||||
|
|
||||||
template <>
|
|
||||||
void print<bool>(const bool&v, const char* delimiter){
|
|
||||||
std::cout<< (v?"true":"false") << delimiter;
|
|
||||||
}
|
|
||||||
@@ -7,6 +7,7 @@
|
|||||||
#include "vector_type.hpp"
|
#include "vector_type.hpp"
|
||||||
#include <iostream>
|
#include <iostream>
|
||||||
#include <string>
|
#include <string>
|
||||||
|
#include <algorithm>
|
||||||
#include "io.h"
|
#include "io.h"
|
||||||
#undef ERROR
|
#undef ERROR
|
||||||
template <typename T>
|
template <typename T>
|
||||||
|
|||||||
@@ -1,20 +0,0 @@
|
|||||||
#include "types.h"
|
|
||||||
#include <string>
|
|
||||||
#include <iostream>
|
|
||||||
#include <chrono>
|
|
||||||
#include <ctime>
|
|
||||||
namespace types {
|
|
||||||
using namespace std;
|
|
||||||
using namespace chrono;
|
|
||||||
string date_t::toString() const {
|
|
||||||
uint32_t curr_v = val;
|
|
||||||
tm;
|
|
||||||
time_t;
|
|
||||||
return string() + string("/") + string() + string("/") + string();
|
|
||||||
}
|
|
||||||
string time_t::toString() const {
|
|
||||||
uint32_t curr_v = val;
|
|
||||||
|
|
||||||
return string() + string("/") + string() + string("/") + string();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,6 +1,5 @@
|
|||||||
#ifndef _TYPES_H
|
#ifndef _TYPES_H
|
||||||
#define _TYPES_H
|
#define _TYPES_H
|
||||||
#include <typeinfo>
|
|
||||||
#include <cstdint>
|
#include <cstdint>
|
||||||
#include <type_traits>
|
#include <type_traits>
|
||||||
#include <string>
|
#include <string>
|
||||||
|
|||||||
@@ -1,18 +0,0 @@
|
|||||||
#include "utils.h"
|
|
||||||
#include <random>
|
|
||||||
#include <chrono>
|
|
||||||
using std::string;
|
|
||||||
string base62uuid(int l = 8) {
|
|
||||||
using namespace std;
|
|
||||||
constexpr static const char* base62alp = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ";
|
|
||||||
static mt19937_64 engine(chrono::system_clock::now().time_since_epoch().count());
|
|
||||||
static uniform_int_distribution<uint64_t> u(0x10000, 0xfffff);
|
|
||||||
uint64_t uuid = (u(engine) << 32ull) + (chrono::system_clock::now().time_since_epoch().count() & 0xffffffff);
|
|
||||||
printf("%p\n", uuid);
|
|
||||||
string ret;
|
|
||||||
while (uuid && l-- >= 0) {
|
|
||||||
ret = string("") + base62alp[uuid % 62] + ret;
|
|
||||||
uuid /= 62;
|
|
||||||
}
|
|
||||||
return ret;
|
|
||||||
}
|
|
||||||
@@ -1,15 +0,0 @@
|
|||||||
#include "vector_type.hpp"
|
|
||||||
#include <iostream>
|
|
||||||
template<typename _Ty>
|
|
||||||
inline void vector_type<_Ty>::out(uint32_t n, const char* sep) const
|
|
||||||
{
|
|
||||||
n = n > size ? size : n;
|
|
||||||
std::cout << '(';
|
|
||||||
{
|
|
||||||
uint32_t i = 0;
|
|
||||||
for (; i < n - 1; ++i)
|
|
||||||
std::cout << this->operator[](i) << sep;
|
|
||||||
std::cout << this->operator[](i);
|
|
||||||
}
|
|
||||||
std::cout << ')';
|
|
||||||
}
|
|
||||||
@@ -10,12 +10,7 @@
|
|||||||
#include <cstdlib>
|
#include <cstdlib>
|
||||||
#include <cstdint>
|
#include <cstdint>
|
||||||
|
|
||||||
#include <algorithm>
|
|
||||||
#include <initializer_list>
|
#include <initializer_list>
|
||||||
#include <vector>
|
|
||||||
#include <stdarg.h>
|
|
||||||
#include <limits>
|
|
||||||
#include <deque>
|
|
||||||
#include "types.h"
|
#include "types.h"
|
||||||
|
|
||||||
#pragma pack(push, 1)
|
#pragma pack(push, 1)
|
||||||
|
|||||||
BIN
Binary file not shown.
Reference in New Issue
Block a user