Added source indexing for symbols (#922)

buildscripts/builddrivers.py

@@ -24,7 +24,9 @@ import sys
 import shutil
 import os.path
 import argparse
+import subprocess
 from buildtools import BuildUtil
+from indexsymbols import *
 
 class BuildDriver(object):
     """Build sqlsrv and/or pdo_sqlsrv drivers with PHP source with the following properties:
@@ -38,6 +40,8 @@ class BuildDriver(object):
         make_clean      # a boolean flag - whether make clean is necessary
         source_path     # path to a local source folder
         testing         # whether the user has turned on testing mode
+        srctool_path    # path to source indexing tools (empty string by default)
+        tag_version     # tag version for source indexing (empty string by default)
     """
     
     def __init__(self, phpver, driver, arch, thread, debug, repo, branch, source, path, testing, no_rename):
@@ -49,6 +53,8 @@ class BuildDriver(object):
         self.testing = testing
         self.rebuild = False
         self.make_clean = False
+        self.srctool_path = ''
+        self.tag_version = ''
     
     def show_config(self):
         print()
@@ -112,6 +118,34 @@ class BuildDriver(object):
             print("The path provided is invalid. Please re-enter.")
         return source
     
+    def index_all_symbols(self, ext_dir, srctool_path, tag_version):
+        """This takes care of indexing all the symbols
+        
+        :param  ext_dir: the directory where we can find the built extension(s)
+        :param  srctool_path: the path to the tools for source indexing
+        :param  tag_version: tag version for source indexing
+        :outcome: all symbols will be source indexed
+        """
+        work_dir = os.path.dirname(os.path.realpath(__file__))
+        os.chdir(srctool_path)
+
+        if self.util.driver == 'all':
+            driver = 'sqlsrv'
+            pdbfile = os.path.join(ext_dir, self.util.driver_name(driver, '.pdb'))
+            print('Indexing this symbol: ', pdbfile)
+            run_indexing_tools(pdbfile, driver, tag_version)
+            driver = 'pdo_sqlsrv'
+            pdbfile = os.path.join(ext_dir, self.util.driver_name(driver, '.pdb'))
+            print('Indexing this symbol: ', pdbfile)
+            run_indexing_tools(pdbfile, driver, tag_version)
+        else:
+            driver = self.util.driver
+            pdbfile = os.path.join(ext_dir, self.util.driver_name(driver, '.pdb'))
+            print('Indexing this symbol: ', pdbfile)
+            run_indexing_tools(pdbfile, driver, tag_version)
+            
+        os.chdir(work_dir)
+        
     def build_extensions(self, root_dir, logfile):
         """This takes care of getting the drivers' source files, building the drivers. 
         If dest_path is defined, the binaries will be copied to the designated destinations.
@@ -151,6 +185,12 @@ class BuildDriver(object):
         # ext_dir is the directory where we can find the built extension(s)
         ext_dir = self.util.build_drivers(self.make_clean, dest, logfile)
 
+        # Do source indexing only if the tag and tools path are both specified
+        if self.tag_version is not '' and self.srctool_path is not '':
+            print('Source indexing begins...')
+            self.index_all_symbols(ext_dir, self.srctool_path, self.tag_version)
+            print('Source indexing done')
+
         # Copy the binaries if a destination path is defined
         if self.dest_path is not None:
             dest_drivers = os.path.join(self.dest_path, self.util.major_version(), self.util.arch)
@@ -172,9 +212,12 @@ class BuildDriver(object):
 
         return ext_dir
 
-    def build(self):
+    def build(self, srctool_path, tag_version):
         """This is the main entry point of building drivers for PHP. 
         For development, this will loop till the user decides to quit.
+        
+        :param  srctool_path: the path to the tools for source indexing
+        :param  tag_version: tag version for source indexing
         """
         self.show_config()
     
@@ -191,6 +234,10 @@ class BuildDriver(object):
                 
             logfile = self.util.get_logfile_name()
 
+            # Save source indexing details
+            self.srctool_path = srctool_path
+            self.tag_version = tag_version
+
             try:
                 ext_dir = self.build_extensions(root_dir, logfile)
                 print('Build Completed')
@@ -244,6 +291,8 @@ if __name__ == '__main__':
     parser.add_argument('--TESTING', action='store_true', help="turns on testing mode (default: False)")
     parser.add_argument('--DESTPATH', default=None, help="an alternative destination for the drivers (default: None)")
     parser.add_argument('--NO_RENAME', action='store_true', help="drivers will not be renamed(default: False)")
+    parser.add_argument('--SRCIDX_PATH', default='', help="the path to the tools for source indexing (default: '')")
+    parser.add_argument('--TAG_VERSION', default='', help="the tag version for source indexing (default: '')")
 
     args = parser.parse_args()
 
@@ -305,4 +354,4 @@ if __name__ == '__main__':
                           path,
                           testing,
                           no_rename)
-    builder.build()
+    builder.build(args.SRCIDX_PATH, args.TAG_VERSION)

buildscripts/buildtools.py

@@ -336,7 +336,6 @@ class BuildUtil(object):
         is complete.
         """
         work_dir = os.path.dirname(os.path.realpath(__file__))   
-
         # First, update the driver source file contents
         source_dir = os.path.join(work_dir, 'Source')
         if self.driver == 'all':
@@ -402,6 +401,7 @@ class BuildUtil(object):
         
         # Final step, copy the binaries to the right place
         ext_dir = self.copy_binaries(sdk_dir, copy_to_ext)
+        
         return ext_dir
 
     def rename_binary(self, path, driver):
@@ -454,7 +454,6 @@ class BuildUtil(object):
         shutil.copy(os.path.join(phpsrc, 'php.ini-production'), php_ini_file)
         
         # Copy run-tests.php as well
-        phpsrc = self.phpsrc_root(sdk_dir)
         shutil.copy(os.path.join(phpsrc, 'run-tests.php'), build_dir)
         
         print('Copying the binaries from', build_dir)

buildscripts/indexsymbols.py

@@ -0,0 +1,145 @@
+#!/usr/bin/python3
+#########################################################################################
+#
+# Description:  This contains helper methods for source indexing
+#
+# Requirement:
+#               python 3.x
+#               srctool.exe and pdbstr.exe
+#
+#############################################################################################
+
+import os.path
+import argparse
+import subprocess
+from subprocess import Popen, PIPE
+
+def write_index(index_filename, tag_version):
+    """This writes to a temporary index file for the pdbstr tool
+
+    For example
+
+    SRCSRV: ini ------------------------------------------------
+    VERSION=1
+    SRCSRV: variables ------------------------------------------
+    PATH=%var2%
+    SRCSRVTRG=%TARG%\%PDBVERSION%\%fnbksl%(%var2%)
+    SRCURL=https://raw.githubusercontent.com/Microsoft/msphpsql/%SRCVERSION%/source/%PATH%
+    SRCSRVCMD=powershell -Command "$r=New-Object -ComObject Msxml2.XMLHTTP; $r.open('GET', '%SRCURL%', $false); $r.send(); [io.file]::WriteAllBytes('%SRCSRVTRG%', $r.responseBody)"
+    SRCVERSION=v5.6.0
+    PDBVERSION=v5.6.0
+    For example
+    """
+    with open(index_filename, 'w') as f:
+        f.write('SRCSRV: ini ------------------------------------------------' + os.linesep)
+        f.write('VERSION=1' + os.linesep)
+        f.write('SRCSRV: variables ------------------------------------------' + os.linesep)
+        f.write('PATH=%var2%' + os.linesep)
+        f.write('SRCSRVTRG=%TARG%\%PDBVERSION%\%fnbksl%(%var2%)' + os.linesep)
+        f.write('SRCURL=https://raw.githubusercontent.com/Microsoft/msphpsql/%SRCVERSION%/source/%PATH%' + os.linesep)
+        f.write('SRCSRVCMD=powershell -Command ')
+        f.write('\"$r=New-Object -ComObject Msxml2.XMLHTTP; ')
+        f.write('$r.open(\'GET\', \'%SRCURL%\', $false); ')
+        f.write('$r.send(); [io.file]::WriteAllBytes(\'%SRCSRVTRG%\', $r.responseBody)\"' + os.linesep)
+        f.write('SRCVERSION=' + tag_version + os.linesep)
+        f.write('PDBVERSION=' + tag_version + os.linesep)
+
+def append_source_filess(index_filename, source_files, driver):
+    """This appends the paths to different source files to the temporary index file
+
+    For example
+
+    SRCSRV: source files ---------------------------------------
+    c:\php-sdk\phpdev\vc15\x86\php-7.2.14-src\ext\pdo_sqlsrv\pdo_dbh.cpp*pdo_sqlsrv/pdo_dbh.cpp
+    c:\php-sdk\phpdev\vc15\x86\php-7.2.14-src\ext\pdo_sqlsrv\pdo_init.cpp*pdo_sqlsrv/pdo_init.cpp
+    ... ...
+    c:\php-sdk\phpdev\vc15\x86\php-7.2.14-src\ext\pdo_sqlsrv\shared\core_stream.cpp*shared/core_stream.cpp
+    c:\php-sdk\phpdev\vc15\x86\php-7.2.14-src\ext\pdo_sqlsrv\shared\core_util.cpp*shared/core_util.cpp
+    SRCSRV: end ------------------------------------------------
+    """
+    failed = False
+    with open(index_filename, 'a') as idx_file:
+        idx_file.write('SRCSRV: source files ---------------------------------------' + os.linesep)
+        with open(source_files, 'r') as src_file:
+            for line in src_file:
+                pos = line.find('shared')
+                if (pos > 0):           # it's a nested folder, so it must be positive
+                    relative_path = line[pos:]
+                    src_line = line[:-1] + '*' + relative_path.replace('\\', '/')
+                else:                   # not a file in the shared folder
+                    pos = line.find(driver)
+                    if (pos <= 0):
+                        print('ERROR: Expected to find', driver, 'in', line)
+                        failed = True
+                        break
+                    else:
+                        relative_path = line[pos:]
+                        src_line = line[:-1] + '*' + relative_path.replace('\\', '/')
+                idx_file.write(src_line)
+        idx_file.write('SRCSRV: end ------------------------------------------------' + os.linesep)
+    return failed
+
+def run_indexing_tools(pdbfile, driver, tag_version):
+    """This invokes the source indexing tools, srctool.exe and pdbstr.exe
+
+    :param  pdbfile: the absolute path to the symbol file
+    :param  driver: either sqlsrv or pdo_sqlsrv
+    :param  tag_version: tag version for source indexing
+    :outcome: the driver pdb file will be source indexed
+    """
+    # run srctool.exe to get all driver's source files from the PDB file
+    # srctool.exe -r <PDBfile> | find "<driver>\" | sort > files.txt
+    batch_filename = 'runsrctool.bat'
+    index_filename = 'idx.txt'
+    source_files = 'files.txt'
+    
+    with open(batch_filename, 'w') as batch_file:
+        batch_file.write('@ECHO OFF' + os.linesep)
+        batch_file.write('@CALL srctool -r %1 | find "%2\\" | sort > ' + source_files + '  2>&1' + os.linesep)
+    
+    get_source_filess = batch_filename + ' {0} {1} '
+    get_source_filess_cmd = get_source_filess.format(pdbfile, driver)
+    subprocess.call(get_source_filess_cmd)
+    
+    # create an index file using the above inputs for pdbstr.exe
+    write_index(index_filename, tag_version)
+    failed = append_source_filess(index_filename, source_files, driver)
+    
+    if failed:
+        print("ERROR: Failed to prepare the temporary index file for the pdbstr tool")
+        exit(1)
+
+    # run pdbstr.exe to insert the information into the PDB file
+    # pdbstr.exe -w -p:<PDBfile> -i:idx.txt -s:srcsrv
+    pdbstr_str = 'pdbstr.exe -w -p:{0} -i:{1} -s:srcsrv'
+    pdbstr_cmd = pdbstr_str.format(pdbfile, index_filename)
+    subprocess.call(pdbstr_cmd)
+    
+    os.remove(batch_filename)
+    os.remove(index_filename)
+    os.remove(source_files)
+
+################################### Main Function ###################################
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('PDBFILE', help="the path to the pdb file for source indexing")
+    parser.add_argument('DRIVER', choices=['sqlsrv', 'pdo_sqlsrv'], help="driver name of this pdb file")
+    parser.add_argument('TAG_VERSION', help="the tag version for source indexing (e.g. v5.6.0)")
+    parser.add_argument('TOOLS_PATH',help="the path to the source indexing tools")
+    
+    args = parser.parse_args()
+
+    srctool_exe = os.path.join(args.TOOLS_PATH, 'srctool.exe')
+    pdbstr_exe = os.path.join(args.TOOLS_PATH, 'pdbstr.exe')
+    if not os.path.exists(srctool_exe) or not os.path.exists(pdbstr_exe):
+        print('ERROR: Missing the required source indexing tools')
+        exit(1)
+
+    work_dir = os.path.dirname(os.path.realpath(__file__))
+    os.chdir(args.TOOLS_PATH)
+    
+    print('Source indexing begins...')
+    run_indexing_tools(args.PDBFILE, args.DRIVER.lower(), args.TAG_VERSION)
+    print('Source indexing done')
+    
+    os.chdir(work_dir)