Deploy test packge for USD Internal Fork. (PixarAnimationStudios#648)

* Deploy test packge for USD Internal Fork.

* Update according to comments.

* Update comments.

* Fix typo.

Author: yangel2017

Jenkinsfile

@@ -436,7 +436,7 @@ finally {
       buildConfigs << "${build.key}_versioning"
       buildConfigs << "${build.key}_garasign"
     }
-    if ("${build.key}".contains('mac')) {
+    if ("${build.key}".contains('mac') || "${build.key}".contains('windows_x64')) {
       buildConfigs << "${build.key}_deploytest"
     }
     buildConfigs << "${build.key}_deployment"
@@ -633,6 +633,19 @@ def buildStage(buildInfo, runBuild) {
   }
 }
 
+def testPackageOnServer(buildInfo){
+  def commit = "${env.GIT_COMMIT}"[0..6]
+  def remotePath = "${ARTIFACTORY_URL}/${ARTIFACTORY_FOLDER}/${ARTIFACTORY_TEST_FOLDER}/usd_${buildInfo.Platform}_${commit}.zip"
+  def status_code = '404'
+  if ("${buildInfo.Platform}".contains("windows")){
+    status_code = COMMONSHELL.capture_script_output("""curl -o NUL --silent -Iw "%%{http_code}" ${remotePath}""")
+  }
+  else{
+    status_code = COMMONSHELL.capture_script_output("""curl -o /dev/null --silent -Iw '%{http_code}' ${remotePath}""")
+  }
+  return status_code == '200'
+}
+
 def windowsBuild(buildInfo) {
   // Clean up TBB src to force a clean build as TBB generates binaries on its source path
   // Clean up textsystem as it has the same folder name on all platforms but with different binaries
@@ -662,6 +675,13 @@ def windowsBuild(buildInfo) {
     """
   }
   else{
+    if (!(isDevBranch() || isReleaseBranch()) && testPackageOnServer(buildInfo)) {
+      // Check if test package is already deployed
+      postStatus('SUCCESS', "${buildInfo.Platform}_deploytest")
+      echo "Package is on server and skip build."
+      return
+    }
+
     bat """
       :: Set up nasm for OIIO plugin
       if not exist ${buildInfo.USDgen_src}\\nasm-2.15.05-win64.zip call curl -o ${buildInfo.USDgen_src}\\nasm-2.15.05-win64.zip https://www.nasm.us/pub/nasm/releasebuilds/2.15.05/win64/nasm-2.15.05-win64.zip
@@ -680,16 +700,14 @@ def windowsBuild(buildInfo) {
       call ${buildInfo.Python} -m pip install PyOpenGL
       call ${buildInfo.Python} -u build_scripts/build_usd.py --generator Ninja --build ${buildInfo.USDgen_build} --src ${buildInfo.USDgen_src} ${buildInfo.USDinst} --build-variant ${buildInfo.BuildVariant} ${USDBUILDINGFLAGS} -v
       """
+    testDeployStage(buildInfo)
   }
 }
 
 def macBuild(buildInfo) {
-  if ((!isDevBranch()) && (!"${buildInfo.Platform}".contains("ios"))) {
+  if (!(isDevBranch() || isReleaseBranch()) && (!"${buildInfo.Platform}".contains("ios"))) {
     // Check if the package is already deployed
-    def commit = "${env.GIT_COMMIT}"[0..6]
-    def remotePath = "${ARTIFACTORY_URL}/${ARTIFACTORY_FOLDER}/${ARTIFACTORY_TEST_FOLDER}/usd_${buildInfo.Platform}_${commit}.zip"
-    def status_code = COMMONSHELL.capture_script_output("""curl -o /dev/null --silent -Iw '%{http_code}' ${remotePath}""")
-    if (status_code == '200') {
+    if (testPackageOnServer(buildInfo)) {
       postStatus('SUCCESS', "${buildInfo.Platform}_deploytest")
       return
     }
@@ -791,6 +809,26 @@ def testStage(buildInfo, runTest) {
   }
 }
 
+def downloadTestPackage(buildInfo){
+  def commit = "${env.GIT_COMMIT}"[0..6]
+  def usdinst="${env.WORKSPACE}/usd_${buildInfo.Platform}_${commit}"
+  def artifactory_env_path = "${ARTIFACTORY_TEST_FOLDER}"
+  def server = "${ARTIFACTORY_URL}"
+  def remotePath = "${ARTIFACTORY_FOLDER}/${artifactory_env_path}"
+  withEnv(["OGS_ARTIFACTORY_SERVER=art-bobcat"]){
+    COMMONSHELL.shell """\
+      ${buildInfo.Python} -u adsk-build-scripts/deploy.py --download_test \
+        --git_commit ${commit} \
+        --remotepath ${remotePath} \
+        --platform ${buildInfo.Platform} \
+        --packagename ${PROJECT_NAME} \
+        --server ${server} \
+        --source ${env.WORKSPACE}
+    """
+  }
+  return usdinst
+}
+
 def windowsTest(buildInfo) {
   // Skip test on windows_android as no tests are generated currently
   // TODO: Run unittest on windows_android
@@ -803,13 +841,19 @@ def windowsTest(buildInfo) {
       rd %temp% /s /q
       """
 
+    def usdinst  = "${buildInfo.USDinst}"
+    def testpath = "${buildInfo.USDgen_build_usd}"
+    if (!(isDevBranch() || isReleaseBranch()) && testPackageOnServer(buildInfo)) {
+      usdinst = downloadTestPackage(buildInfo)
+      testpath = usdinst
+    }
+
     bat """
-      :: As part of relocatable support, user needs EnvVar("PXR_MTLX_STDLIB_SEARCH_PATHS") to use MaterialX libraies from USD installed path,
-      :: but it's necessay on the build machine as USD also adds path to the MaterialX standard library discovered at build time.
-      :: call set PXR_MTLX_STDLIB_SEARCH_PATHS=<Full_Of_USDinst_Extracted_Path>/libraries
-      call set PATH=%PATH%;${buildInfo.USDinst}\\lib;${buildInfo.USDinst}\\bin;${buildInfo.USDinst}\\share\\usd\\examples\\plugin;
-      call set PYTHONPATH=${buildInfo.USDinst}\\lib\\python
-      call cd /d ${buildInfo.USDgen_build_usd} && ctest -C ${buildInfo.CMakeConfig} ${USDTESTINGFLAGS}
+      :: As part of relocatable support, user needs EnvVar("PXR_MTLX_STDLIB_SEARCH_PATHS") to use MaterialX libraies from USD installed path.
+      call set PXR_MTLX_STDLIB_SEARCH_PATHS=${usdinst}\\libraries;
+      call set PATH=%PATH%;${usdinst}\\lib;${usdinst}\\bin;${usdinst}\\share\\usd\\examples\\plugin;
+      call set PYTHONPATH=${usdinst}\\lib\\python
+      call cd /d ${testpath} && ctest -C ${buildInfo.CMakeConfig} ${USDTESTINGFLAGS}
       """
   }
 }
@@ -894,22 +938,8 @@ def runMacTest(buildInfo, runlocally) {
   // Set up testing environment on the newly allocated node
   if (!runlocally) {
     checkoutGit()
-
-    def commit = "${env.GIT_COMMIT}"[0..6]
-    usdinst="${env.WORKSPACE}/usd_${buildInfo.Platform}_${commit}"
-    def artifactory_env_path = "${ARTIFACTORY_TEST_FOLDER}"
-    def server = "${ARTIFACTORY_URL}"
-    def remotePath = "${ARTIFACTORY_FOLDER}/${artifactory_env_path}"
+    usdinst = downloadTestPackage(buildInfo)
     sh """
-      export OGS_ARTIFACTORY_SERVER=art-bobcat
-      python3 -u adsk-build-scripts/deploy.py --download_test \
-        --git_commit ${commit} \
-        --remotepath ${remotePath} \
-        --platform ${buildInfo.Platform} \
-        --packagename ${PROJECT_NAME} \
-        --server ${server} \
-        --source ${env.WORKSPACE}
-
       install_name_tool -add_rpath ${usdinst}/lib ${usdinst}/bin/idiff
     """
     testpath = "${usdinst}"

adsk-build-scripts/deploy.py

@@ -544,6 +544,28 @@ def downloadPackageThreaded(url, archivePath):
         if not successful_download and os.path.isfile(archivePath):
             os.remove(archivePath)
 
+def winapi_path(dos_path, encoding=None):
+    if (not isinstance(dos_path, str) and encoding is not None):
+        dos_path = dos_path.decode(encoding)
+    path = os.path.abspath(dos_path)
+    if path.startswith(u"\\\\"):
+        return u"\\\\?\\UNC\\" + path[2:]
+    return u"\\\\?\\" + path
+
+# Custom zipfile.ZipFile to handle long path in Windows and permissions reservation.
+class CustomZipFile(zipfile.ZipFile):
+    def _extract_member(self, member, targetpath, pwd):
+        if not isinstance(member, zipfile.ZipInfo):
+            member = self.getinfo(member)
+        if platform.system() == 'Windows':
+             targetpath = winapi_path(targetpath)
+        targetpath = super()._extract_member(member, targetpath, pwd)
+
+        attr = member.external_attr >> 16
+        if attr != 0:
+            os.chmod(targetpath, attr)
+        return targetpath
+
 def download_test(cargs):
     # Get the closet Artifactory server to gain fastest download speed
     server = os.getenv('OGS_ARTIFACTORY_SERVER', None)
@@ -572,7 +594,8 @@ def download_test(cargs):
 
         start = time.time()
         try:
-            subprocess.run(['unzip', '-q', zipfile_name, '-d', extract_to])
+            with CustomZipFile(zipfile_name, 'r') as zip_ref:
+                zip_ref.extractall(extract_to)
             duration = time.time() - start
             print('Used {dur} secs to extract {file}'.format(dur=str(duration), file=zip_name))
         except BaseException as e:
@@ -587,14 +610,19 @@ def download_test(cargs):
 
     # Replace the relative paths for CTestTestfile.cmake
     contents = ''
+    extract_to_slash = extract_to.replace('\\','/')
+    usd_git_slash = usd_git.replace('\\','/')
     with open(os.path.join(extract_to, 'CTestTestfile.cmake.all'), 'r') as f:
         lines = f.read()
-        contents += lines.replace('<usd_build>', extract_to).replace('<usd_inst>', extract_to).replace('<usd_git>', usd_git)
+        contents += lines.replace('<usd_build>', extract_to_slash).replace('<usd_inst>', extract_to_slash).replace('<usd_git>', usd_git_slash)
     with open(os.path.join(extract_to, 'CTestTestfile.cmake'), 'w') as f:
         f.write(contents)
 
     sys.exit()
 
+def uppercase_first_letter(string):
+    return string[0].upper() + string[1:]
+
 def zip_test(source_dir, archive, usd_build):
     abs_usd_build=os.path.abspath(usd_build)
     usd_git = os.path.abspath(os.path.join(SCRIPT_PATH, '..'))
@@ -613,9 +641,13 @@ def zip_test(source_dir, archive, usd_build):
     # Consolidate all tests to single CTestTestfile.cmake.all,
     # and replace relative paths with defined tags to be replaced in future,
     # for it to be able to run from anywhere with local paths replaced
+    # Uppercase the drive and replace '\' to '/' for path in Windows to be consistent with CTestTestfile.cmake.
+    abs_usd_build_slash = uppercase_first_letter(abs_usd_build).replace('\\','/') if platform.system() == 'Windows' else abs_usd_build
+    source_dir_slash = uppercase_first_letter(source_dir).replace('\\','/') if platform.system() == 'Windows' else source_dir
+    usd_git_slash = uppercase_first_letter(usd_git).replace('\\','/') if platform.system() == 'Windows' else usd_git
     ctestfile_all = os.path.join(abs_usd_build, 'CTestTestfile.cmake.all')
     with open(ctestfile_all, 'w') as f:
-        f.write(contents.replace(abs_usd_build, '<usd_build>').replace(source_dir, '<usd_inst>').replace(usd_git, '<usd_git>'))
+        f.write(contents.replace(abs_usd_build_slash, '<usd_build>').replace(source_dir_slash, '<usd_inst>').replace(usd_git_slash, '<usd_git>'))
 
     # Zip test case file and necessary binaries into archive
     ziph = zipfile.ZipFile(archive, 'w', compression=zipfile.ZIP_DEFLATED)