diff --git a/buildscripts/builddrivers.py b/buildscripts/builddrivers.py index e9ac02dc..ff301cc5 100644 --- a/buildscripts/builddrivers.py +++ b/buildscripts/builddrivers.py @@ -24,7 +24,9 @@ import sys import shutil import os.path import argparse +import subprocess from buildtools import BuildUtil +from indexsymbols import * class BuildDriver(object): """Build sqlsrv and/or pdo_sqlsrv drivers with PHP source with the following properties: @@ -38,6 +40,8 @@ class BuildDriver(object): make_clean # a boolean flag - whether make clean is necessary source_path # path to a local source folder testing # whether the user has turned on testing mode + srctool_path # path to source indexing tools (empty string by default) + tag_version # tag version for source indexing (empty string by default) """ def __init__(self, phpver, driver, arch, thread, debug, repo, branch, source, path, testing, no_rename): @@ -49,6 +53,8 @@ class BuildDriver(object): self.testing = testing self.rebuild = False self.make_clean = False + self.srctool_path = '' + self.tag_version = '' def show_config(self): print() @@ -112,6 +118,34 @@ class BuildDriver(object): print("The path provided is invalid. Please re-enter.") return source + def index_all_symbols(self, ext_dir, srctool_path, tag_version): + """This takes care of indexing all the symbols + + :param ext_dir: the directory where we can find the built extension(s) + :param srctool_path: the path to the tools for source indexing + :param tag_version: tag version for source indexing + :outcome: all symbols will be source indexed + """ + work_dir = os.path.dirname(os.path.realpath(__file__)) + os.chdir(srctool_path) + + if self.util.driver == 'all': + driver = 'sqlsrv' + pdbfile = os.path.join(ext_dir, self.util.driver_name(driver, '.pdb')) + print('Indexing this symbol: ', pdbfile) + run_indexing_tools(pdbfile, driver, tag_version) + driver = 'pdo_sqlsrv' + pdbfile = os.path.join(ext_dir, self.util.driver_name(driver, '.pdb')) + print('Indexing this symbol: ', pdbfile) + run_indexing_tools(pdbfile, driver, tag_version) + else: + driver = self.util.driver + pdbfile = os.path.join(ext_dir, self.util.driver_name(driver, '.pdb')) + print('Indexing this symbol: ', pdbfile) + run_indexing_tools(pdbfile, driver, tag_version) + + os.chdir(work_dir) + def build_extensions(self, root_dir, logfile): """This takes care of getting the drivers' source files, building the drivers. If dest_path is defined, the binaries will be copied to the designated destinations. @@ -151,6 +185,12 @@ class BuildDriver(object): # ext_dir is the directory where we can find the built extension(s) ext_dir = self.util.build_drivers(self.make_clean, dest, logfile) + # Do source indexing only if the tag and tools path are both specified + if self.tag_version is not '' and self.srctool_path is not '': + print('Source indexing begins...') + self.index_all_symbols(ext_dir, self.srctool_path, self.tag_version) + print('Source indexing done') + # Copy the binaries if a destination path is defined if self.dest_path is not None: dest_drivers = os.path.join(self.dest_path, self.util.major_version(), self.util.arch) @@ -172,9 +212,12 @@ class BuildDriver(object): return ext_dir - def build(self): + def build(self, srctool_path, tag_version): """This is the main entry point of building drivers for PHP. For development, this will loop till the user decides to quit. + + :param srctool_path: the path to the tools for source indexing + :param tag_version: tag version for source indexing """ self.show_config() @@ -191,6 +234,10 @@ class BuildDriver(object): logfile = self.util.get_logfile_name() + # Save source indexing details + self.srctool_path = srctool_path + self.tag_version = tag_version + try: ext_dir = self.build_extensions(root_dir, logfile) print('Build Completed') @@ -244,6 +291,8 @@ if __name__ == '__main__': parser.add_argument('--TESTING', action='store_true', help="turns on testing mode (default: False)") parser.add_argument('--DESTPATH', default=None, help="an alternative destination for the drivers (default: None)") parser.add_argument('--NO_RENAME', action='store_true', help="drivers will not be renamed(default: False)") + parser.add_argument('--SRCIDX_PATH', default='', help="the path to the tools for source indexing (default: '')") + parser.add_argument('--TAG_VERSION', default='', help="the tag version for source indexing (default: '')") args = parser.parse_args() @@ -305,4 +354,4 @@ if __name__ == '__main__': path, testing, no_rename) - builder.build() + builder.build(args.SRCIDX_PATH, args.TAG_VERSION) diff --git a/buildscripts/buildtools.py b/buildscripts/buildtools.py index 79b36923..59f8b8f5 100644 --- a/buildscripts/buildtools.py +++ b/buildscripts/buildtools.py @@ -336,7 +336,6 @@ class BuildUtil(object): is complete. """ work_dir = os.path.dirname(os.path.realpath(__file__)) - # First, update the driver source file contents source_dir = os.path.join(work_dir, 'Source') if self.driver == 'all': @@ -402,6 +401,7 @@ class BuildUtil(object): # Final step, copy the binaries to the right place ext_dir = self.copy_binaries(sdk_dir, copy_to_ext) + return ext_dir def rename_binary(self, path, driver): @@ -454,7 +454,6 @@ class BuildUtil(object): shutil.copy(os.path.join(phpsrc, 'php.ini-production'), php_ini_file) # Copy run-tests.php as well - phpsrc = self.phpsrc_root(sdk_dir) shutil.copy(os.path.join(phpsrc, 'run-tests.php'), build_dir) print('Copying the binaries from', build_dir) diff --git a/buildscripts/indexsymbols.py b/buildscripts/indexsymbols.py new file mode 100644 index 00000000..bcf6ffbd --- /dev/null +++ b/buildscripts/indexsymbols.py @@ -0,0 +1,145 @@ +#!/usr/bin/python3 +######################################################################################### +# +# Description: This contains helper methods for source indexing +# +# Requirement: +# python 3.x +# srctool.exe and pdbstr.exe +# +############################################################################################# + +import os.path +import argparse +import subprocess +from subprocess import Popen, PIPE + +def write_index(index_filename, tag_version): + """This writes to a temporary index file for the pdbstr tool + + For example + + SRCSRV: ini ------------------------------------------------ + VERSION=1 + SRCSRV: variables ------------------------------------------ + PATH=%var2% + SRCSRVTRG=%TARG%\%PDBVERSION%\%fnbksl%(%var2%) + SRCURL=https://raw.githubusercontent.com/Microsoft/msphpsql/%SRCVERSION%/source/%PATH% + SRCSRVCMD=powershell -Command "$r=New-Object -ComObject Msxml2.XMLHTTP; $r.open('GET', '%SRCURL%', $false); $r.send(); [io.file]::WriteAllBytes('%SRCSRVTRG%', $r.responseBody)" + SRCVERSION=v5.6.0 + PDBVERSION=v5.6.0 + For example + """ + with open(index_filename, 'w') as f: + f.write('SRCSRV: ini ------------------------------------------------' + os.linesep) + f.write('VERSION=1' + os.linesep) + f.write('SRCSRV: variables ------------------------------------------' + os.linesep) + f.write('PATH=%var2%' + os.linesep) + f.write('SRCSRVTRG=%TARG%\%PDBVERSION%\%fnbksl%(%var2%)' + os.linesep) + f.write('SRCURL=https://raw.githubusercontent.com/Microsoft/msphpsql/%SRCVERSION%/source/%PATH%' + os.linesep) + f.write('SRCSRVCMD=powershell -Command ') + f.write('\"$r=New-Object -ComObject Msxml2.XMLHTTP; ') + f.write('$r.open(\'GET\', \'%SRCURL%\', $false); ') + f.write('$r.send(); [io.file]::WriteAllBytes(\'%SRCSRVTRG%\', $r.responseBody)\"' + os.linesep) + f.write('SRCVERSION=' + tag_version + os.linesep) + f.write('PDBVERSION=' + tag_version + os.linesep) + +def append_source_filess(index_filename, source_files, driver): + """This appends the paths to different source files to the temporary index file + + For example + + SRCSRV: source files --------------------------------------- + c:\php-sdk\phpdev\vc15\x86\php-7.2.14-src\ext\pdo_sqlsrv\pdo_dbh.cpp*pdo_sqlsrv/pdo_dbh.cpp + c:\php-sdk\phpdev\vc15\x86\php-7.2.14-src\ext\pdo_sqlsrv\pdo_init.cpp*pdo_sqlsrv/pdo_init.cpp + ... ... + c:\php-sdk\phpdev\vc15\x86\php-7.2.14-src\ext\pdo_sqlsrv\shared\core_stream.cpp*shared/core_stream.cpp + c:\php-sdk\phpdev\vc15\x86\php-7.2.14-src\ext\pdo_sqlsrv\shared\core_util.cpp*shared/core_util.cpp + SRCSRV: end ------------------------------------------------ + """ + failed = False + with open(index_filename, 'a') as idx_file: + idx_file.write('SRCSRV: source files ---------------------------------------' + os.linesep) + with open(source_files, 'r') as src_file: + for line in src_file: + pos = line.find('shared') + if (pos > 0): # it's a nested folder, so it must be positive + relative_path = line[pos:] + src_line = line[:-1] + '*' + relative_path.replace('\\', '/') + else: # not a file in the shared folder + pos = line.find(driver) + if (pos <= 0): + print('ERROR: Expected to find', driver, 'in', line) + failed = True + break + else: + relative_path = line[pos:] + src_line = line[:-1] + '*' + relative_path.replace('\\', '/') + idx_file.write(src_line) + idx_file.write('SRCSRV: end ------------------------------------------------' + os.linesep) + return failed + +def run_indexing_tools(pdbfile, driver, tag_version): + """This invokes the source indexing tools, srctool.exe and pdbstr.exe + + :param pdbfile: the absolute path to the symbol file + :param driver: either sqlsrv or pdo_sqlsrv + :param tag_version: tag version for source indexing + :outcome: the driver pdb file will be source indexed + """ + # run srctool.exe to get all driver's source files from the PDB file + # srctool.exe -r | find "\" | sort > files.txt + batch_filename = 'runsrctool.bat' + index_filename = 'idx.txt' + source_files = 'files.txt' + + with open(batch_filename, 'w') as batch_file: + batch_file.write('@ECHO OFF' + os.linesep) + batch_file.write('@CALL srctool -r %1 | find "%2\\" | sort > ' + source_files + ' 2>&1' + os.linesep) + + get_source_filess = batch_filename + ' {0} {1} ' + get_source_filess_cmd = get_source_filess.format(pdbfile, driver) + subprocess.call(get_source_filess_cmd) + + # create an index file using the above inputs for pdbstr.exe + write_index(index_filename, tag_version) + failed = append_source_filess(index_filename, source_files, driver) + + if failed: + print("ERROR: Failed to prepare the temporary index file for the pdbstr tool") + exit(1) + + # run pdbstr.exe to insert the information into the PDB file + # pdbstr.exe -w -p: -i:idx.txt -s:srcsrv + pdbstr_str = 'pdbstr.exe -w -p:{0} -i:{1} -s:srcsrv' + pdbstr_cmd = pdbstr_str.format(pdbfile, index_filename) + subprocess.call(pdbstr_cmd) + + os.remove(batch_filename) + os.remove(index_filename) + os.remove(source_files) + +################################### Main Function ################################### +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('PDBFILE', help="the path to the pdb file for source indexing") + parser.add_argument('DRIVER', choices=['sqlsrv', 'pdo_sqlsrv'], help="driver name of this pdb file") + parser.add_argument('TAG_VERSION', help="the tag version for source indexing (e.g. v5.6.0)") + parser.add_argument('TOOLS_PATH',help="the path to the source indexing tools") + + args = parser.parse_args() + + srctool_exe = os.path.join(args.TOOLS_PATH, 'srctool.exe') + pdbstr_exe = os.path.join(args.TOOLS_PATH, 'pdbstr.exe') + if not os.path.exists(srctool_exe) or not os.path.exists(pdbstr_exe): + print('ERROR: Missing the required source indexing tools') + exit(1) + + work_dir = os.path.dirname(os.path.realpath(__file__)) + os.chdir(args.TOOLS_PATH) + + print('Source indexing begins...') + run_indexing_tools(args.PDBFILE, args.DRIVER.lower(), args.TAG_VERSION) + print('Source indexing done') + + os.chdir(work_dir)