[Daint] Upstream for large Python deps and rttov (#722)

repos/c2sm/packages/icon/package.py

@@ -316,10 +316,11 @@ def setup_build_environment(self, env):
             for d in link_dirs:
                 env.append_path('SPACK_COMPILER_IMPLICIT_RPATHS', d)
 
+        # help cmake to build dsl-stencils
         if 'none' not in self.spec.variants['dsl'].value:
             env.set("CUDAARCHS", self.spec.variants['gpu'].value)
             env.unset("CUDAHOSTCXX")
-            env.set("Boost_INCLUDE_DIR", self.spec['boost'].prefix.include)
+            env.set("BOOST_ROOT", self.spec['boost'].prefix)
 
     @run_before('configure')
     def downgrade_opt_level(self):

requirements.txt

@@ -1,2 +1,2 @@
 requests
-pytest-xdist
+pytest-xdist

src/report_tests.py

@@ -67,7 +67,8 @@ def __str__(self) -> str:
             any_tests_of_type = True
             with open(file_name, 'r') as file:
                 content = file.read()
-                if content.endswith('OK\n'):
+                second_last_line = content.split('\n')[-2]
+                if 'OK' in second_last_line:
                     table.append(':green_circle:', file_name)
                 else:
                     all_tests_of_type_passed = False

src/upstream.py

@@ -0,0 +1,27 @@
+import shutil
+import sys
+import os
+
+sys.path.insert(1, 'spack/lib/spack/external')
+from ruamel import yaml
+
+
+def read_upstream_from_spack_yaml(config_dir):
+    spack_yaml = os.path.join(config_dir, 'spack.yaml')
+    with open(spack_yaml, 'r') as f:
+        spack_config = yaml.load(f)
+    upstream = spack_config['spack']['config']['install_tree']['root']
+    return upstream
+
+
+def delete_upstream(upstream):
+    print(f'Delete upstream {upstream}')
+    shutil.rmtree(upstream, ignore_errors=True)
+
+
+if __name__ == '__main__':
+
+    if len(sys.argv) != 2:
+        raise ValueError('Need path to folder containing a spack.yaml')
+    else:
+        delete_upstream(read_upstream_from_spack_yaml(sys.argv[1]))

sysconfigs/daint/upstreams.yaml

@@ -1,3 +1,7 @@
 upstreams:
-  system:
+  base:
     install_tree: /project/g110/spack/upstream/daint_v0.18.1.2
+  icon-dsl:
+    install_tree: /project/g110/spack/upstream/daint_v0.18.1.4/icon-dsl
+  icon-rttov:
+    install_tree: /project/g110/spack/upstream/daint_v0.18.1.4/icon-rttov

test/unit_test.py

@@ -9,6 +9,8 @@
 sys.path.append(os.path.normpath(spack_c2sm_path))
 from src import machine_name, Markdown, HTML, time_format, sanitized_filename, all_machines, all_packages, explicit_scope, package_triggers
 
+from src.upstream import read_upstream_from_spack_yaml
+
 
 class MachineDetection(unittest.TestCase):
 
@@ -132,6 +134,20 @@ def test_package_triggers(self):
                         in triggers)  # Name of Test included
 
 
+class UpstreamTest(unittest.TestCase):
+
+    @unittest.expectedFailure
+    def test_non_existent_spack_yaml(self):
+        read_upstream_from_spack_yaml('/inexistent_path/to_yaml')
+
+    def test_upstream_from_config(self):
+        upstream_base = read_upstream_from_spack_yaml(
+            os.path.join(os.path.normpath(spack_c2sm_path),
+                         'upstreams/daint/base'))
+        self.assertEqual('/project/g110/spack/upstream/daint_v0.18.1.2',
+                         upstream_base)
+
+
 if __name__ == '__main__':
     if len(sys.argv) > 1:
         machine_name_from_arg = sys.argv[-1]

upstreams/daint/InstallUpstream

@@ -6,11 +6,29 @@ pipeline {
         }
     }
     stages {
-            stage('Install Upstream') {
+            stage('Install Upstream Base') {
                 steps {
                     sh"""
                     . ./setup-env.sh
-                    spack env activate upstreams/daint
+                    spack env activate upstreams/daint/base
+                    spack install -v
+                    """
+                }
+            }
+            stage('Install Upstream Icon-DSL') {
+                steps {
+                    sh"""
+                    . ./setup-env.sh
+                    spack env activate upstreams/daint/icon-dsl
+                    spack install -v
+                    """
+                }
+            }
+            stage('Install Upstream Icon-rttov') {
+                steps {
+                    sh"""
+                    . ./setup-env.sh
+                    spack env activate upstreams/daint/icon-rttov
                     spack install -v
                     """
                 }

upstreams/daint/UninstallUpstream

@@ -0,0 +1,39 @@
+pipeline {
+
+    agent {
+        node {
+            label 'daint'
+        }
+    }
+    stages {
+            stage('Uninstall Upstream Base') {
+                steps {
+                    sh"""
+                    python3 src/upstream.py upstreams/daint/base
+                    """
+                }
+            }
+            stage('Uninstall Upstream Icon-DSL') {
+                steps {
+                    sh"""
+                    python3 src/upstream.py upstreams/daint/icon-dsl
+                    """
+                }
+            }
+            stage('Uninstall Upstream Icon-rttov') {
+                steps {
+                    sh"""
+                    python3 src/upstream.py upstreams/daint/icon-rttov
+                    """
+                }
+            }
+    }
+    post {
+        always {
+            echo 'Cleaning up workspace'
+            deleteDir()
+        }
+    }
+
+}
+

upstreams/daint/icon-dsl/spack.yaml

@@ -0,0 +1,10 @@
+spack:
+  specs:
+  - py-icon4py@0.0.3%gcc@9.3.0
+  - py-gt4py@1.1.1%gcc@9.3.0
+  concretizer:
+    unify: true
+  view: false
+  config:
+    install_tree:
+      root: /project/g110/spack/upstream/do_not_use/icon-dsl

upstreams/daint/icon-rttov/spack.yaml

@@ -0,0 +1,15 @@
+spack:
+  specs:
+   # ICON + rttov
+    - rttov@13.1%nvhpc ~openmp +hdf5
+    - nvidia-blas%nvhpc
+    - nvidia-lapack%nvhpc
+    - pkg-config%gcc
+    - zlib%gcc
+    - hdf5@1.12.2%nvhpc
+  concretizer:
+    unify: true
+  view: false
+  config:
+    install_tree:
+      root: /project/g110/spack/upstream/do_not_use/icon-rttov