AQuery/build.py

from dataclasses import dataclass
from enum import Enum, auto
import aquery_config
import sys
import os
import subprocess
import hashlib
import pickle
from common.utils import nullstream
from typing import Dict, Optional, Set, Union

@dataclass
class checksums:
    libaquery_a : Optional[Union[bytes, bool]] = None
    pch_hpp_gch : Optional[Union[bytes, bool]] = None
    server : Optional[Union[bytes, bool]] = None
    sources : Optional[Union[Dict[str, bytes], bool]] = None
    env : str = ''
    
    def calc(self, compiler_name, libaquery_a = 'libaquery.a' , 
                pch_hpp_gch = 'server/pch.hpp.gch', 
                server = 'server.so'
        ):
        from platform import machine
        self.env = (aquery_config.os_platform +
                    machine() + 
                    aquery_config.build_driver + 
                    compiler_name + 
                    aquery_config.version_string
                    + str(os.environ['AQ_DEBUG'] == '1')
                )
        for key in self.__dict__.keys():
            try:
                with open(eval(key), 'rb') as file:
                    self.__dict__[key] = hashlib.md5(file.read()).digest()
            except (FileNotFoundError, NameError):
                pass
        sources = [*build_manager.headerfiles, *build_manager.sourcefiles]
        self.sources = dict()
        for key in sources:
            try:
                with open(key, 'rb') as file:
                    self.sources[key] = hashlib.md5(file.read()).digest()
            except FileNotFoundError:
                print('missing component: ' + key)
                self.sources[key] = None
                
    def __ne__(self, __o: 'checksums') -> 'checksums':
        ret = checksums()
        for key in self.__dict__.keys():
            try:
                ret.__dict__[key] = (
                    not (self.__dict__[key] and __o.__dict__[key]) or
                    self.__dict__[key] != __o.__dict__[key]
                )
            except KeyError:
                ret.__dict__[key] = True
        return ret
    
    def __eq__(self, __o: 'checksums') -> 'checksums':
        ret = checksums()
        for key in self.__dict__.keys():
            try:
                ret.__dict__[key] = (
                    self.__dict__[key] and __o.__dict__[key] and
                    self.__dict__[key] == __o.__dict__[key]
                )
            except KeyError:
                ret.__dict__[key] = False
        return ret

        
        
class build_manager:
    sourcefiles = [
                   'build.py', 'Makefile', 
                   'server/server.cpp', 'server/libaquery.cpp',  
                   'server/monetdb_conn.cpp', 'server/duckdb_conn.cpp',
                   'server/threading.cpp', 'server/winhelper.cpp', 
                   'server/monetdb_ext.c'
                   ]
    headerfiles = ['server/aggregations.h', 'server/hasher.h', 'server/io.h', 
                   'server/libaquery.h', 'server/monetdb_conn.h', 'server/duckdb_conn.h', 
                   'server/pch.hpp', 'server/table.h', 'server/threading.h', 
                   'server/types.h', 'server/utils.h', 'server/winhelper.h', 
                   'server/gc.h', 'server/vector_type.hpp', 'server/table_ext_monetdb.hpp' 
                   ]
   
    class DriverBase:
        def __init__(self, mgr : 'build_manager') -> None:
            self.mgr = mgr
            self.build_cmd = []
        def libaquery_a(self) :
            return False
        def pch(self):
            return False
        def build(self, stdout = sys.stdout, stderr = sys.stderr):
            ret = True
            if not aquery_config.compilation_output:
                stdout = nullstream
                stderr = nullstream
            for c in self.build_cmd:
                if c:
                    try: # only last success matters
                        ret = not subprocess.call(c, stdout = stdout, stderr = stderr) # and ret
                    except (FileNotFoundError):
                        ret = False
                        pass
            return ret
        def warmup(self):
            return True
                
    class MakefileDriver(DriverBase):
        def __init__(self, mgr : 'build_manager') -> None:
            super().__init__(mgr)
            os.environ['PCH'] = f'{mgr.PCH}'
            if 'CXX' not in os.environ:
                os.environ['CXX'] = mgr.cxx if mgr.cxx else 'c++'
            else:
                mgr.cxx = os.environ['CXX']
            if 'AQ_DEBUG' not in os.environ:
                os.environ['AQ_DEBUG'] = ('0' if mgr.OptimizationLv != '0' else '1')

        def libaquery_a(self):
            self.build_cmd = [['rm', 'libaquery.a'],['make', 'libaquery', '-j']]
            return self.build()
        def pch(self):
            self.build_cmd = [['rm', 'server/pch.hpp.gch'], ['make', 'pch', '-j']]
            return self.build()
        def server(self):
            if self.mgr.StaticLib:
                self.build_cmd = [['rm', '*.o'],['rm', 'server.so'], ['make', 'server_uselib', '-j']]
            else:
                self.build_cmd = [['rm', 'server.so'], ['make', 'server.so', '-j']]
            return self.build()
            
        def snippet(self):
            if self.mgr.StaticLib:
                self.build_cmd = [['make', 'snippet_uselib', '-j']]
            else:
                self.build_cmd = [['rm', 'dll.so'], ['make', 'snippet', '-j']]
            return self.build()

    class MSBuildDriver(DriverBase):
        platform_map = {'amd64':'x64', 'arm64':'arm64', 'x86':'win32'}
        opt_map = {'0':'Debug', '1':'RelWithDebugInfo', '2':'Release', '3':'Release', '4':'Release'}
        def __init__(self, mgr : 'build_manager') -> None:
            super().__init__(mgr)
            mgr.cxx = aquery_config.msbuildroot
            
        def get_flags(self):
            self.platform = self.platform_map[self.mgr.Platform]
            self.platform = f'/p:platform={self.platform}'
            self.opt = self.opt_map[self.mgr.OptimizationLv]
            self.opt = f'/p:configuration={self.opt}'

        def libaquery_a(self):
            loc = os.path.abspath('./msc-plugin/libaquery.vcxproj')
            self.get_flags()
            self.build_cmd = [['del', 'libaquery.lib'], [aquery_config.msbuildroot, loc, self.opt, self.platform]]
            return self.build()

        def pch(self):
            return True

        def server(self):
            print(self.opt)
            loc = os.path.abspath('./msc-plugin/server.vcxproj')
            self.get_flags()
            self.build_cmd = [['del', 'server.so'], [aquery_config.msbuildroot, loc, self.opt, self.platform]]
            return self.build()

        def snippet(self):
            loc = os.path.abspath('./msc-plugin/msc-plugin.vcxproj')
            self.get_flags()
            self.build_cmd = [[aquery_config.msbuildroot, loc, self.opt, self.platform]]
            return self.build()

        def warmup(self):
            self.build_cmd = [['make', 'warmup']]
            return self.build()
            
    #class PythonDriver(DriverBase):
    #    def __init__(self, mgr : 'build_manager') -> None:
    #        super().__init__(mgr)           

    def __init__(self) -> None:
        self.method = 'make'
        self.cxx = ''
        self.OptimizationLv = '4' # [O0, O1, O2, O3, Ofast]
        self.Platform = 'amd64'
        self.PCH = os.environ['PCH'] if 'PCH' in os.environ else 1
        self.StaticLib = 1
        self.fLTO = 1
        self.fmarchnative = 1
        if aquery_config.build_driver == 'MSBuild':
            self.driver = build_manager.MSBuildDriver(self) 
        elif aquery_config.build_driver == 'Makefile':
            self.driver = build_manager.MakefileDriver(self)
        self.cache_status = checksums()
        
    def build_dll(self):
        return self.driver.snippet()
        
    def build_caches(self, force = False):
        cached = checksums()
        current = checksums()
        libaquery_a = 'libaquery.a' 
        if aquery_config.os_platform == 'win':
            libaquery_a = 'libaquery.lib'
        current.calc(self.cxx, libaquery_a)
        try:
            with open('.cached', 'rb') as cache_sig:
                cached = pickle.loads(cache_sig.read())
        except FileNotFoundError:
            pass
        self.cache_status = current != cached
        
        success = True
        if  (force or 
             self.cache_status.sources or 
             self.cache_status.env
        ):
            success &= self.driver.pch()
            success &= self.driver.libaquery_a()
            success &= self.driver.server()
        else:
            if self.cache_status.libaquery_a:
                success &= self.driver.libaquery_a() 
            if self.cache_status.pch_hpp_gch:
                success &= self.driver.pch() 
            if self.cache_status.server:
                success &= self.driver.server() 
        if success:
            current.calc(self.cxx, libaquery_a)
            with open('.cached', 'wb') as cache_sig:
                cache_sig.write(pickle.dumps(current))
            self.driver.warmup()
            
            
        else:
            if aquery_config.os_platform == 'mac':
                os.system('./arch-check.sh')
            try:
                os.remove('./.cached')
            except:
                pass