@ -1,26 +1,56 @@
import os
from engine . ast import Context
if __name__ == ' __main__ ' :
import mimetypes
mimetypes . _winreg = None
from dataclasses import dataclass
import enum
import re
from tabnanny import check
import time
# import dbconn
import re
from typing import Callable , List , Optional
from mo_parsing import ParseException
import aquery_parser as parser
import engine
import engine . projection
import engine . ddl
import reconstruct as xengine
import subprocess
import mmap
import sys
import os
from engine . utils import base62uuid
import atexit
import threading
import ctypes
import aquery_config
import numpy as np
from engine . utils import ws
from engine . utils import add_dll_dir
## GLOBALS BEGIN
nullstream = open ( os . devnull , ' w ' )
help_message = ''' \
Run prompt . py without supplying with any arguments to run in interactive mode .
- - help , - h
print out this help message
- - version , - v
returns current version of AQuery
- - mode , - m Optional [ threaded | IPC ]
execution engine run mode :
threaded or 1 ( default ) : run the execution engine and compiler / prompt in separate threads
IPC , standalong or 2 : run the execution engine in a new process which uses shared memory to communicate with the compiler process
- - script , - s [ SCRIPT_FILE ]
script mode : run the aquery script file
- - parse - only , - p
parse only : parse the file and print out the AST
'''
## GLOBALS END
## CLASSES BEGIN
class RunType ( enum . Enum ) :
Threaded = 0
IPC = 1
@ -30,75 +60,19 @@ class Backend_Type(enum.Enum):
BACKEND_MonetDB = 1
BACKEND_MariaDB = 2
server_mode = RunType . Threaded
server_bin = ' server.bin ' if server_mode == RunType . IPC else ' server.so '
nullstream = open ( os . devnull , ' w ' )
if aquery_config . rebuild_backend :
try :
os . remove ( server_bin )
except Exception as e :
print ( type ( e ) , e )
subprocess . call ( [ ' make ' , " info " ] )
subprocess . call ( [ ' make ' , server_bin ] , stdout = nullstream )
cleanup = True
def rm ( ) :
global cleanup
if cleanup :
mm . seek ( 0 , os . SEEK_SET )
mm . write ( b ' \x00 \x00 ' )
mm . flush ( )
try :
time . sleep ( .001 )
server . kill ( )
time . sleep ( .001 )
server . terminate ( )
except OSError :
pass
files = os . listdir ( ' . ' )
for f in files :
if f . endswith ( ' .shm ' ) :
os . remove ( f )
mm . close ( )
cleanup = False
nullstream . close ( )
def init_ipc ( ) :
global shm , server , basecmd , mm
shm = base62uuid ( )
if sys . platform != ' win32 ' :
shm + = ' .shm '
basecmd = [ ' bash ' , ' -c ' , ' rlwrap k ' ]
mm = None
if not os . path . isfile ( shm ) :
# create initial file
with open ( shm , " w+b " ) as handle :
handle . write ( b ' \x01 \x00 ' ) # [running, new job]
handle . flush ( )
mm = mmap . mmap ( handle . fileno ( ) , 2 , access = mmap . ACCESS_WRITE , offset = 0 )
if mm is None :
exit ( 1 )
else :
basecmd = [ ' bash.exe ' , ' -c ' , ' rlwrap ./k ' ]
mm = mmap . mmap ( 0 , 2 , shm )
mm . write ( b ' \x01 \x00 ' )
mm . flush ( )
server = subprocess . Popen ( [ " ./server.bin " , shm ] )
import numpy as np
c = lambda _ba : ctypes . cast ( ( ctypes . c_char * len ( _ba ) ) . from_buffer ( _ba ) , ctypes . c_char_p )
class Config :
def __init__ ( self , nq = 0 , mode = server_mode , n_bufs = 0 , bf_szs = [ ] ) - > None :
__all_attrs__ = [ ' running ' , ' new_query ' , ' server_mode ' , ' backend_type ' , ' has_dll ' , ' n_buffers ' ]
__init_attributes__ = False
def __init_self__ ( ) :
if not Config . __init_attributes__ :
from functools import partial
for _i , attr in enumerate ( Config . __all_attrs__ ) :
if not hasattr ( Config , attr ) :
setattr ( Config , attr , property ( partial ( Config . getter , i = _i ) , partial ( Config . setter , i = _i ) ) )
__init_attributes__ = True
def __init__ ( self , mode , nq = 0 , n_bufs = 0 , bf_szs = [ ] ) - > None :
Config . __init_self__ ( )
self . int_size = 4
self . n_attrib = 6
self . buf = bytearray ( ( self . n_attrib + n_bufs ) * self . int_size )
@ -121,29 +95,83 @@ class Config:
@property
def c ( self ) :
return c ( self . buf )
def binder ( cls , attr , _i ) :
from functools import partial
setattr ( cls , attr , property ( partial ( cls . getter , i = _i ) , partial ( cls . setter , i = _i ) ) )
return ctypes . cast ( \
( ctypes . c_char * len ( self . buf ) ) . from_buffer ( self . buf ) , ctypes . c_char_p )
@dataclass
class PromptState ( ) :
cleanup = True
th = None
send = None
test_parser = True
server_mode : RunType = RunType . Threaded
server_bin = ' server.bin ' if server_mode == RunType . IPC else ' server.so '
set_ready = lambda : None
get_ready = lambda : None
server_status = lambda : False
cfg : Config = None
shm : str = ' '
server : subprocess . Popen = None
basecmd : List [ str ] = None
mm : mmap = None
th : threading . Thread
send : Callable = lambda * _ : None
init : Callable [ [ ' PromptState ' ] , None ] = lambda _ : None
stmts = [ ' ' ]
payloads = { }
## CLASSES END
## FUNCTIONS BEGIN
def rm ( state : PromptState ) :
if state . cleanup :
state . mm . seek ( 0 , os . SEEK_SET )
state . mm . write ( b ' \x00 \x00 ' )
state . mm . flush ( )
binder ( Config , ' running ' , 0 )
binder ( Config , ' new_query ' , 1 )
binder ( Config , ' server_mode ' , 2 )
binder ( Config , ' backend_type ' , 3 )
binder ( Config , ' has_dll ' , 4 )
binder ( Config , ' n_buffers ' , 5 )
try :
time . sleep ( .001 )
state . server . kill ( )
time . sleep ( .001 )
state . server . terminate ( )
except OSError :
pass
cfg = Config ( )
th = None
send = None
files = os . listdir ( ' . ' )
for f in files :
if f . endswith ( ' .shm ' ) :
os . remove ( f )
state . mm . close ( )
state . cleanup = False
nullstream . close ( )
def init_threaded ( ) :
def init_ipc ( state : PromptState ) :
state . shm = base62uuid ( )
if sys . platform != ' win32 ' :
state . shm + = ' .shm '
state . basecmd = [ ' bash ' , ' -c ' , ' rlwrap k ' ]
state . mm = None
if not os . path . isfile ( shm ) :
# create initial file
with open ( shm , " w+b " ) as handle :
handle . write ( b ' \x01 \x00 ' ) # [running, new job]
handle . flush ( )
state . mm = mmap . mmap ( handle . fileno ( ) , 2 , access = mmap . ACCESS_WRITE , offset = 0 )
if state . mm is None :
exit ( 1 )
else :
state . basecmd = [ ' bash.exe ' , ' -c ' , ' rlwrap ./k ' ]
state . mm = mmap . mmap ( 0 , 2 , state . shm )
state . mm . write ( b ' \x01 \x00 ' )
state . mm . flush ( )
state . server = subprocess . Popen ( [ " ./server.bin " , state . shm ] )
def init_threaded ( state : PromptState ) :
state . cleanup = False
if os . name == ' nt ' and aquery_config . add_path_to_ldpath :
t = os . environ [ ' PATH ' ] . lower ( ) . split ( ' ; ' )
vars = re . compile ( ' %.*% ' )
os . add_dll_directory ( os . path . abspath ( ' . ' ) )
os . add_dll_directory ( os . path . abspath ( ' ./lib ' ) )
( os . path . abspath ( ' . ' ) )
( os . path . abspath ( ' ./lib ' ) )
for e in t :
if ( len ( e ) != 0 ) :
if ' % ' in e :
@ -154,7 +182,7 @@ def init_threaded():
except Exception :
continue
try :
os. add_dll_directory ( e )
( e )
except Exception :
continue
@ -162,78 +190,108 @@ def init_threaded():
os . environ [ ' PATH ' ] = os . environ [ ' PATH ' ] + os . pathsep + os . path . abspath ( ' . ' )
os . environ [ ' PATH ' ] = os . environ [ ' PATH ' ] + os . pathsep + os . path . abspath ( ' ./lib ' )
if aquery_config . run_backend :
server_so = ctypes . CDLL ( ' ./ ' + server_bin )
global cfg , th , send
send = server_so [ ' receive_args ' ]
server_so = ctypes . CDLL ( ' ./ ' + state . server_bin )
state . send = server_so [ ' receive_args ' ]
aquery_config . have_hge = server_so [ ' have_hge ' ] ( )
if aquery_config . have_hge :
from engine . types import get_int128_support
get_int128_support ( )
th = threading . Thread ( target = server_so [ ' main ' ] , args = ( - 1 , ctypes . POINTER ( ctypes . c_char_p ) ( cfg . c ) ) , daemon = True )
th . start ( )
state . th = threading . Thread ( target = server_so [ ' main ' ] , args = ( - 1 , ctypes . POINTER ( ctypes . c_char_p ) ( state . cfg . c ) ) , daemon = True )
state . th . start ( )
def init_prompt ( ) - > PromptState :
aquery_config . init_config ( )
if server_mode == RunType . IPC :
atexit . register ( rm )
init = init_ipc
set_ready = lambda : mm . seek ( 0 , os . SEEK_SET ) or mm . write ( b ' \x01 \x01 ' )
state = PromptState ( )
if aquery_config . rebuild_backend :
try :
os . remove ( state . server_bin )
except Exception as e :
print ( type ( e ) , e )
subprocess . call ( [ ' make ' , " info " ] )
subprocess . call ( [ ' make ' , state . server_bin ] , stdout = nullstream )
state . cfg = Config ( state . server_mode )
if state . server_mode == RunType . IPC :
atexit . register ( lambda : rm ( state ) )
state . init = init_ipc
state . set_ready = lambda : state . mm . seek ( 0 , os . SEEK_SET ) or state . mm . write ( b ' \x01 \x01 ' )
def __get_ready ( ) :
mm . seek ( 0 , os . SEEK_SET )
return mm . read ( 2 ) [ 1 ]
get_ready = __get_ready
server_status = lambda : server . poll ( ) is not None
else :
init = init_threaded
state . mm . seek ( 0 , os . SEEK_SET )
return state . mm . read ( 2 ) [ 1 ]
state . get_ready = __get_ready
state . server_status = lambda : state . server . poll ( ) is not None
else :
state . init = init_threaded
rm = lambda : None
def __set_ready ( ) :
global cfg
cfg . new_query = 1
set_ready = __set_ready
get_ready = lambda : aquery_config . run_backend and cfg . new_query
state . cfg . new_query = 1
state . set_ready = __set_ready
state . get_ready = lambda : aquery_config . run_backend and state . cfg . new_query
if aquery_config . run_backend :
server_status = lambda : not th . is_alive ( )
state . server_status = lambda : not state . th . is_alive ( )
else :
server_status = lambda : True
init ( )
test_parser = True
state . server_status = lambda : True
state . init ( state )
return state
# code to test parser
ws = re . compile ( r ' \ s+ ' )
q = ' SELECT p.Name, v.Name FROM Production.Product p JOIN Purchasing.ProductVendor pv ON p.ProductID = pv.ProductID JOIN Purchasing.Vendor v ON pv.BusinessEntityID = v.BusinessEntityID WHERE ProductSubcategoryID = 15 ORDER BY v.Name; '
res = parser . parse ( q )
def save ( q : str , cxt : xengine . Context ) :
savecmd = re . split ( r ' [ \ t] ' , q )
if len ( savecmd ) > 1 :
fname = savecmd [ 1 ]
else :
tm = time . gmtime ( )
fname = f ' { tm . tm_year } { tm . tm_mon } _ { tm . tm_mday } _ { tm . tm_hour } : { tm . tm_min } : { tm . tm_sec } '
if cxt :
from typing import Optional
def savefile ( attr : str , desc : str , ext : Optional [ str ] = None ) :
if hasattr ( cxt , attr ) :
attr : str = getattr ( cxt , attr )
if attr :
ext = ext if ext else ' . ' + desc
name = fname if fname . endswith ( ext ) else fname + ext
with open ( ' saves/ ' + name , ' wb ' ) as cfile :
cfile . write ( attr . encode ( ' utf-8 ' ) )
print ( f ' saved { desc } code as { name } ' )
savefile ( ' ccode ' , ' cpp ' )
savefile ( ' udf ' , ' udf ' , ' .hpp ' )
savefile ( ' sql ' , ' sql ' )
payload = None
keep = True
cxt = engine . initialize ( )
cxt . Info ( res )
while test_parser :
def main ( running = lambda : True , next = input , state = None ) :
if state is None :
state = init_prompt ( )
q = ' '
payload = None
keep = True
cxt = engine . initialize ( )
while running ( ) :
try :
if server_status ( ) :
init ( )
while get_ready ( ) :
if state . server_status ( ) :
state . init ( )
while state . get_ready ( ) :
time . sleep ( .00001 )
print ( " > " , end = " " )
q = input ( ) . lower ( )
q = nex t( ) . lower ( )
if q == ' exec ' : # generate build and run (AQuery Engine)
cfg . backend_type = Backend_Type . BACKEND_AQuery . value
cxt = engine . exec ( stmts , cxt , keep )
state . cfg . backend_type = Backend_Type . BACKEND_AQuery . value
cxt = engine . exec ( state . stmts , cxt , keep )
if subprocess . call ( [ ' make ' , ' snippet ' ] , stdout = nullstream ) == 0 :
set_ready ( )
state . set_ready ( )
continue
elif q == ' xexec ' : # generate build and run (MonetDB Engine)
cfg . backend_type = Backend_Type . BACKEND_MonetDB . value
cxt = xengine . exec ( stmts , cxt , keep )
if server_mode == RunType . Threaded :
state . cfg . backend_type = Backend_Type . BACKEND_MonetDB . value
cxt = xengine . exec ( state . stmts , cxt , keep )
if state . server_mode == RunType . Threaded :
# assignment to avoid auto gc
# sqls = [s.strip() for s in cxt.sql.split(';')]
qs = [ ctypes . c_char_p ( bytes ( q , ' utf-8 ' ) ) for q in cxt . queries if len ( q ) ]
sz = len ( qs )
payload = ( ctypes . c_char_p * sz ) ( * qs )
state . payload = payload
try :
send ( sz , payload )
state . send ( sz , payload )
except TypeError as e :
print ( e )
if cxt . udf is not None :
@ -244,10 +302,10 @@ while test_parser:
with open ( ' out.cpp ' , ' wb ' ) as outfile :
outfile . write ( ( cxt . finalize ( ) ) . encode ( ' utf-8 ' ) )
subprocess . call ( [ ' make ' , ' snippet ' ] , stdout = nullstream )
cfg . has_dll = 1
state . cfg . has_dll = 1
else :
cfg . has_dll = 0
set_ready ( )
state . cfg . has_dll = 0
state . set_ready ( )
continue
@ -270,33 +328,13 @@ while test_parser:
cxt . print ( cxt . log_level )
continue
elif q == ' k ' :
subprocess . call ( basecmd )
subprocess . call ( state . basecmd )
continue
elif q == ' print ' :
cxt . print ( stmts )
cxt . print ( state . stmts )
continue
elif q . startswith ( ' save ' ) :
savecmd = re . split ( r ' [ \ t] ' , q )
if len ( savecmd ) > 1 :
fname = savecmd [ 1 ]
else :
tm = time . gmtime ( )
fname = f ' { tm . tm_year } { tm . tm_mon } _ { tm . tm_mday } _ { tm . tm_hour } : { tm . tm_min } : { tm . tm_sec } '
if cxt :
from typing import Optional
def savefile ( attr : str , desc : str , ext : Optional [ str ] = None ) :
if hasattr ( cxt , attr ) :
attr : str = getattr ( cxt , attr )
if attr :
ext = ext if ext else ' . ' + desc
name = fname if fname . endswith ( ext ) else fname + ext
with open ( ' saves/ ' + name , ' wb ' ) as cfile :
cfile . write ( attr . encode ( ' utf-8 ' ) )
print ( f ' saved { desc } code as { name } ' )
savefile ( ' ccode ' , ' cpp ' )
savefile ( ' udf ' , ' udf ' , ' .hpp ' )
savefile ( ' sql ' , ' sql ' )
save ( q , cxt )
continue
elif q == ' keep ' :
keep = not keep
@ -304,14 +342,18 @@ while test_parser:
elif q == ' format ' or q == ' fmt ' :
subprocess . call ( [ ' clang-format ' , ' out.cpp ' ] )
elif q == ' exit ' :
break
rm ( state )
exit ( )
elif q == ' r ' : # build and run
if subprocess . call ( [ ' make ' , ' snippet ' ] ) == 0 :
set_ready ( )
state . set_ready ( )
continue
elif q == ' rr ' : # run
set_ready ( )
state . set_ready ( )
continue
elif q == ' script ' :
# TODO: script mode
pass
elif q . startswith ( ' save2 ' ) :
filename = re . split ( r ' [ \ t] ' , q )
if ( len ( filename ) > 1 ) :
@ -328,10 +370,10 @@ while test_parser:
with open ( fn , ' r ' ) as file :
contents = file . read ( ) #.lower()
stmts = parser . parse ( contents )
state . stmts = parser . parse ( contents )
continue
stmts = parser . parse ( q )
cxt . Info ( stmts )
state . stmts = parser . parse ( q )
cxt . Info ( state . stmts )
except ParseException as e :
print ( e )
continue
@ -340,6 +382,66 @@ while test_parser:
except ( KeyboardInterrupt ) :
break
except :
rm ( )
import code , traceback
sh = code . InteractiveConsole ( { * * globals ( ) , * * locals ( ) } )
sh . interact ( banner = traceback . format_exc ( ) , exitmsg = ' debugging session ended. ' )
save ( ' ' , cxt )
rm ( state )
raise
rm ( )
rm ( state )
## FUNCTIONS END
## MAIN
if __name__ == ' __main__ ' :
state = None
nextcmd = ' '
def check_param ( param : List [ str ] , args = False ) - > bool :
global nextcmd
for p in param :
if p in sys . argv :
if args :
return True
pos = sys . argv . index ( p )
if len ( sys . argv ) > pos + 1 :
nextcmd = sys . argv [ pos + 1 ]
return True
return False
if check_param ( [ ' -h ' , ' --help ' ] , True ) :
print ( help_message )
exit ( )
if len ( sys . argv ) == 2 :
nextcmd = sys . argv [ 1 ]
if nextcmd . startswith ( ' - ' ) :
nextcmd = ' '
nextcmd = ' test.aquery '
if nextcmd or check_param ( [ ' -s ' , ' --script ' ] ) :
with open ( nextcmd ) as file :
nextcmd = file . readline ( )
from engine . utils import _Counter
if file . name . endswith ( ' aquery ' ) or nextcmd . strip ( ) == ' #!aquery ' :
state = init_prompt ( )
while ( nextcmd ) :
while ( not ws . sub ( ' ' , nextcmd ) or nextcmd . strip ( ) . startswith ( ' # ' ) ) :
nextcmd = file . readline ( )
cnt = _Counter ( 1 )
main ( lambda : cnt . inc ( - 1 ) > 0 , lambda : nextcmd . strip ( ) , state )
nextcmd = file . readline ( )
if check_param ( [ ' -p ' , ' --parse ' ] ) :
with open ( nextcmd , ' r ' ) as file :
contents = file . read ( )
print ( parser . parse ( contents ) )
if check_param ( [ ' -m ' , ' --mode ' , ' --run-type ' ] ) :
nextcmd = nextcmd . lower ( )
ipc_string = [ ' ipc ' , ' proc ' , ' 2 ' , ' standalong ' ]
thread_string = [ ' thread ' , ' 1 ' ]
if any ( [ s in nextcmd for s in ipc_string ] ) :
server_mode = RunType . IPC
elif any ( [ s in nextcmd for s in thread_string ] ) :
server_mode = RunType . Threaded
main ( state = state )
