Build the MUSICA library in CAM-SIMA (#364)

Tag name (required for release branches):
Originator(s): @boulderdaze

Description (include the issue title, and the keyword ['closes',
'fixes', 'resolves'] followed by the issue number):

- Build the MUSICA library in CAM-SIMA as part of the #321 to enable the
full implementation of MICM and TUV.
- Update the Dockerfiles to include a newer version of ESMF that is the
same version used by Derecho, and update the contents to use MUSICA
schemes.
- Update the README instructions for downloading submodules.
- Tested case build using Docker and on Derecho.

Describe any changes made to build system: 
Added functions to build musica in `cime_config/buildlib`

Describe any changes made to the namelist: N/A

List any changes to the defaults for the input datasets (e.g. boundary
datasets): N/A

List all files eliminated and why: N/A

List all files added and what they do:
```
A       cime_config/atm_musica_config.py
```

List all existing files that have been modified, and describe the
changes:
(Helpful git command: `git diff --name-status
development...<your_branch_name>`)
```
M       .dockerignore
M       .gitmodules
M       README.md
M       cime_config/buildlib
M       cime_config/buildnml
M       cime_config/cam_autogen.py
M       cime_config/cam_config.py
M       docker/Dockerfile
M       docker/Dockerfile.esmf
M       docker/Dockerfile.musica
M       src/physics/ncar_ccpp
M       src/physics/utils/musica_ccpp_dependencies.meta
M       test/unit/python/test_cam_autogen.py
```
If there are new failures (compared to the
`test/existing-test-failures.txt` file),
have them OK'd by the gatekeeper, note them here, and add them to the
file.
If there are baseline differences, include the test and the reason for
the
diff. What is the nature of the change? Roundoff?  N/A

derecho/intel/aux_sima: N/A

derecho/gnu/aux_sima: N/A

If this changes climate describe any run(s) done to evaluate the new
climate in enough detail that it(they) could be reproduced: N/A

CAM-SIMA date used for the baseline comparison tests if different than
latest:

---------

Co-authored-by: Jesse Nusbaumer <nusbaume@ucar.edu>

Author: boulderdaze

.dockerignore

@@ -1,14 +1,11 @@
-# ignore all
-*
+.github/
+.gitignore
+LICENSE
+README.md
 
-# include things to copy
-!Externals*
-!src/
-!cime_config/
-!manage_externals/
-!test/
-!.config_files.xml
-!docker
-!bin/
-!.lib/
-!.gitmodules
+# Ignore editor temporaries and backups
+*.swp
+*~
+.#*
+\#*#
+**/.vscode/

.gitmodules

@@ -20,7 +20,7 @@
 [submodule "ncar-physics"]
         path          = src/physics/ncar_ccpp
 	url           = https://github.com/ESCOMP/atmospheric_physics
-	fxtag         = 252b500a93c89f36ece7d8ba08fd8eb025279eaa
+	fxtag         = 37224423f62d9a71922cb6fe9beca1516d9403a4
         fxrequired    = AlwaysRequired
         fxDONOTUSEurl = https://github.com/ESCOMP/atmospheric_physics
 [submodule "ccs_config"]

README.md

@@ -16,7 +16,7 @@ cd CAM-SIMA
 ## NOTE: This is **unsupported** development code and is subject to the [CESM developer's agreement](http://www.cgd.ucar.edu/cseg/development-code.html).
 ```
 git checkout development
-./manage_externals/checkout_externals
+./bin/git-fleximod update
 ```
 
 Good luck, and have a great day!

cime_config/atm_musica_config.py

@@ -0,0 +1,10 @@
+"""
+The URLs and tags provided in this script are read by buildlib to build
+and install the MUSICA library, as well as to download the MUSICA configuration.
+"""
+
+MUSICA_CCPP_SCHEME_NAME = "musica_ccpp"
+MUSICA_REPO_URL = "https://github.com/NCAR/musica.git"
+MUSICA_TAG = "cc39bb00d2220fc81c85b22d3ceea4a39bd2bacf"
+CHEMISTRY_DATA_REPO_URL = "https://github.com/NCAR/cam-sima-chemistry-data.git"
+CHEMISTRY_DATA_TAG = "2b58f2410ec7a565bcf80dee16ec20f6bc35d78b"

cime_config/buildlib

@@ -8,9 +8,13 @@ import sys
 import os
 import filecmp
 import shutil
+import subprocess
 import logging
 
 from cam_config import ConfigCAM # CAM's configure structure
+from atm_musica_config import MUSICA_CCPP_SCHEME_NAME
+from atm_musica_config import MUSICA_REPO_URL, MUSICA_TAG
+from atm_musica_config import CHEMISTRY_DATA_REPO_URL, CHEMISTRY_DATA_TAG
 
 # Check for the CIME library, and add it
 # to the python path:
@@ -27,6 +31,7 @@ from CIME.utils import stop_buffering_output
 from CIME.buildlib import parse_input
 from CIME.build import get_standard_makefile_args
 from CIME.Tools.standard_script_setup import check_minimum_python_version
+from CIME.locked_files import lock_file, unlock_file
 #pylint: enable=wrong-import-position
 
 check_minimum_python_version(3, 7) #CAM requires version 3.7 or greater
@@ -73,7 +78,7 @@ def _build_cam():
         # Re-run source generator in case registry, CCPP suites, or
         # generator scripts have been modified, and
         # to extract required source code paths:
-        config.generate_cam_src(gen_indent)
+        scheme_names = config.generate_cam_src(gen_indent)
 
         dycore = config.get_value('dyn')
         reg_dir = config.get_value('reg_dir')
@@ -122,12 +127,6 @@ def _build_cam():
             paths.append(os.path.join(atm_root, "src", "dynamics", "tests",
                                       "initial_conditions"))
 
-        # If using the CMEPS/NUOPC coupler, then add additional path:
-        if case.get_value("COMP_INTERFACE") == "nuopc":
-            paths.append(os.path.join(__CIMEROOT, "src", "drivers",
-                                      "nuopc", "nuopc_cap_share"))
-        # End if
-
         # Write Filepath text file
         with open(filepath_src, "w", encoding='utf-8') as filepath:
             filepath.write("\n".join(paths))
@@ -174,11 +173,32 @@ def _build_cam():
             if len(optional_mpas_features) > 0:
                 cmd += " OPTIONAL_MPAS_FEATURES=\"" + " ".join(optional_mpas_features) + "\""
 
+        # If MUSICA-CCPP scheme is used in a suite, download
+        # the MUSICA configuration and build the MUSICA library
+        if MUSICA_CCPP_SCHEME_NAME in scheme_names:
+            _download_musica_configuration(caseroot)
+            musica_install_path = _build_musica(caseroot)
+
+            cam_linked_libs = case.get_value("CAM_LINKED_LIBS")
+            musica_libs = "-lmusica-fortran -lmusica -lyaml-cpp"
+            if not musica_libs in cam_linked_libs:
+                _set_musica_lib_path(musica_install_path, caseroot)
+
+            cmd += ' USER_INCLDIR="'\
+                    f'-I{os.path.join(musica_install_path, "include", "micm")} '\
+                    f'-I{os.path.join(musica_install_path, "include", "musica")} '\
+                    f'-I{os.path.join(musica_install_path, "include", "musica", "micm")} '\
+                    f'-I{os.path.join(musica_install_path, "include", "musica", "tuvx")} '\
+                    f'-I{os.path.join(musica_install_path, "include", "musica", "fortran")} '\
+                    '"'
+
         retcode, out, err = run_cmd(cmd)
         _LOGGER.info("Command %s:\n\nstdout:\n%s\n\nstderr:\n%s\n", cmd, out, err)
         expect(retcode == 0, f"Command {cmd} failed with rc={retcode}")
 
+###############################################################################
 def _prepare_mpas(case: Case) -> None:
+###############################################################################
     """
     Prepare MPAS build infrastructure.
     """
@@ -206,7 +226,9 @@ def _prepare_mpas(case: Case) -> None:
     shutil.copytree(os.path.normpath(os.path.join(mpas_dycore_src_root, os.pardir, os.pardir, "assets")), mpas_dycore_bld_root, copy_function=_copy2_as_needed, dirs_exist_ok=True)
     shutil.copytree(os.path.normpath(os.path.join(mpas_dycore_src_root, os.pardir, os.pardir, "driver")), os.path.join(mpas_dycore_bld_root, "driver"), copy_function=_copy2_as_needed, dirs_exist_ok=True)
 
+###############################################################################
 def _copy2_as_needed(src: str, dst: str) -> None:
+###############################################################################
     """
     Wrapper around `shutil.copy2`. Copy the file `src` to the file or directory `dst` as needed.
     """
@@ -226,6 +248,184 @@ def _copy2_as_needed(src: str, dst: str) -> None:
                 # Example scenario: User added some new source code files.
                 shutil.copy2(src, dst)
 
+###############################################################################
+def _build_musica(clone_dest: str) -> str:
+###############################################################################
+    """
+    Builds and installs the MUSICA library.
+
+    Args:
+        clone_dest: destination where the repository will be cloned
+
+    Raises:
+        Exception: If configuring the CMake MUSICA project fails or
+            the MUSICA library build fails, an exception is raised.
+
+    Returns:
+        musica_install_path: path to the MUSICA installation directory
+    """
+    _clone_and_checkout(MUSICA_REPO_URL, MUSICA_TAG, clone_dest)
+
+    bld_path = os.path.join(clone_dest, "musica", "build")
+    if os.path.exists(bld_path):
+        shutil.rmtree(bld_path)
+    os.makedirs(bld_path)
+
+    install_dir = "install"
+    command = [
+        "cmake",
+        f"-D CMAKE_INSTALL_PREFIX={install_dir}",
+        "-D CMAKE_BUILD_TYPE=Release",
+        "-D MUSICA_ENABLE_TESTS=OFF",
+        "-D MUSICA_BUILD_FORTRAN_INTERFACE=ON",
+        ".."
+    ]
+    try:
+        subprocess.run(command, cwd=bld_path, stdout=subprocess.PIPE,
+                       stderr=subprocess.PIPE, text=True, check=False)
+    except subprocess.CalledProcessError as e:
+        raise subprocess.CalledProcessError(e.returncode, e.cmd, "The subprocess \
+            for cmake to configure the MUSICA CMake project failed.") from e
+    except FileNotFoundError as e:
+        raise FileNotFoundError("The 'cmake' command was not found.") from e
+    except OSError as e:
+        raise OSError("An error occurred while executing the 'cmake' command.") from e
+
+    command = ["cmake", "--build", ".", "--target", "install"]
+    try:
+        subprocess.run(command, cwd=bld_path, stdout=subprocess.PIPE,
+                       stderr=subprocess.PIPE, text=True, check=False)
+    except subprocess.CalledProcessError as e:
+        raise subprocess.CalledProcessError(e.returncode, e.cmd, "The subprocess \
+            for cmake to build the MUSICA library failed.") from e
+    except FileNotFoundError as e:
+        raise FileNotFoundError("The 'cmake' command was not found.") from e
+    except OSError as e:
+        raise OSError("An error occurred while executing the 'cmake' command.") from e
+
+    musica_install_path = os.path.join(bld_path, install_dir)
+
+    return musica_install_path
+
+###############################################################################
+def _download_musica_configuration(download_dest: str) -> None:
+###############################################################################
+    """
+    Downloads the MUSICA configuration and renames the configuration
+    directory to match the name in the MUSICA-CCPP configuration.
+
+    Args:
+        download_dest: destination where configuration will be downloaded
+
+    Raises:
+        Exception: If the directory to be renamed is not found or
+            any other exceptions occur during the renaming process,
+            an exception is raised with the error message.
+    """
+    musica_config_dir_name = "musica_configurations"
+
+    _clone_and_checkout(CHEMISTRY_DATA_REPO_URL, CHEMISTRY_DATA_TAG, download_dest)
+
+    original_dir = os.path.join(download_dest, "cam-sima-chemistry-data", "mechanisms")
+    renamed_dir = os.path.join(download_dest, "cam-sima-chemistry-data", musica_config_dir_name)
+    try:
+        os.rename(original_dir, renamed_dir)
+    except FileNotFoundError as e:
+        raise FileNotFoundError(f"The directory '{original_dir}' was not found.") from e
+    except FileExistsError as e:
+        raise FileExistsError(f"The destination directory '{renamed_dir}' already exists.") from e
+    except PermissionError as e:
+        raise PermissionError(f"Permission denied to rename '{original_dir}'.") from e
+    except OSError as e:
+        raise OSError("An error occurred while renaming.") from e
+
+    musica_config_path = os.path.join(download_dest, musica_config_dir_name)
+    if os.path.exists(musica_config_path):
+        shutil.rmtree(musica_config_path)
+
+    shutil.move(renamed_dir, download_dest)
+
+###############################################################################
+def _set_musica_lib_path(musica_install_path: str, caseroot: str) -> None:
+###############################################################################
+    """
+    Sets the MUSICA libraries path to CAM_LINKED_LIBS, allowing the libraries
+    to be linked during the CESM build process.
+
+    Args:
+        musica_install_path: path to the MUSICA installation directory
+        caseroot: CASEROOT where the xmlchange command is located
+
+    Raises:
+        Exception: If the subprocess for the xmlchange command fails,
+            an exception is raised with the error message.
+    """
+
+    unlock_file("env_build.xml", caseroot)
+
+    command = [
+        "./xmlchange",
+        "--append",
+        # The libraries must be on the same line because CIME flags an
+        # error for multi-character arguments preceded by a single dash
+        f"CAM_LINKED_LIBS=-L{os.path.join(musica_install_path, 'lib64')} -lmusica-fortran -lmusica -lyaml-cpp"
+    ]
+    try:
+        subprocess.run(command, cwd=caseroot, stdout=subprocess.PIPE,
+                       stderr=subprocess.PIPE, text=True, check=False)
+    except subprocess.CalledProcessError as e:
+        raise subprocess.CalledProcessError(e.returncode, e.cmd, "The subprocess \
+            for xmlchange to set the MUSICA library path failed.") from e
+    except FileNotFoundError as e:
+        raise FileNotFoundError("The 'xmlchange' command was not found.") from e
+    except OSError as e:
+        raise OSError("An error occurred while executing the 'xmlchange' command.") from e
+
+    lock_file("env_build.xml", caseroot)
+
+###############################################################################
+def _clone_and_checkout(repo_url: str, tag_name: str, clone_dest: str) -> None:
+###############################################################################
+    """
+    Clones a Git repository from the URL and checks out a specific branch.
+
+    Args:
+        repo_url: URL of the Git repository to clone
+        tag_name: tag name to check out
+        clone_dest: destination where the repository will be cloned
+
+    Raises:
+        Exception: If the `git clone` or `git checkout` commands fail,
+            an exception is raised with the error message.
+    """
+    repo_name = repo_url.split("/")[-1].replace(".git", "")
+    repo_path = os.path.join(clone_dest, repo_name)
+
+    if os.path.exists(repo_path):
+        shutil.rmtree(repo_path)
+
+    try:
+        subprocess.run(["git", "clone", repo_url, repo_path],
+           stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, check=False)
+    except subprocess.CalledProcessError as e:
+        raise subprocess.CalledProcessError(e.returncode, e.cmd, f"The subprocess \
+            for git to clone the repository {repo_url} failed.") from e
+    except FileNotFoundError as e:
+        raise FileNotFoundError("The 'git' command was not found.") from e
+    except OSError as e:
+        raise OSError("An error occurred while executing the 'git' command.") from e
+
+    try:
+        subprocess.run(["git", "-C", repo_path, "checkout", tag_name],
+           stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, check=False)
+    except subprocess.CalledProcessError as e:
+        raise subprocess.CalledProcessError(e.returncode, e.cmd, f"The subprocess \
+            for git to checkout the branch {tag_name} failed.") from e
+    except FileNotFoundError as e:
+        raise FileNotFoundError("The 'git' command was not found.") from e
+    except OSError as e:
+        raise OSError("An error occurred while executing the 'git' command.") from e
+
 ###############################################################################
 
 if __name__ == "__main__":

cime_config/buildnml

@@ -166,7 +166,7 @@ def buildnml(case, caseroot, compname):
     gen_indent = 3
 
     #Generate model code and meta-data:
-    config.generate_cam_src(gen_indent)
+    _ = config.generate_cam_src(gen_indent)
 
     #----------------------------------------------------------------
     # Create namelist attribute dictionary (to set namelist defaults):

cime_config/cam_autogen.py

@@ -452,6 +452,7 @@ def generate_physics_suites(build_cache, preproc_defs, host_name,
     # Find the SDFs specified for this model build
     sdfs = []
     scheme_files = []
+    scheme_names = set()
     xml_files = {} # key is scheme, value is xml file path
     for sdf in phys_suites_str.split(';'):
         sdf_path = _find_file(f"suite_{sdf}.xml", suite_search)
@@ -467,6 +468,8 @@ def generate_physics_suites(build_cache, preproc_defs, host_name,
         # Given an SDF, find all the schemes it calls
         _, suite = read_xml_file(sdf_path)
         sdf_schemes = _find_schemes_in_sdf(suite)
+        #Add schemes to set of all scheme names:
+        scheme_names.update(sdf_schemes)
         # For each scheme, find its metadata file
         for scheme in sdf_schemes:
             if scheme in all_scheme_files:
@@ -655,7 +658,7 @@ def generate_physics_suites(build_cache, preproc_defs, host_name,
     # End if
 
     return [physics_blddir, genccpp_dir], do_gen_ccpp, cap_output_file,       \
-        xml_files.values(), capgen_db
+        xml_files.values(), capgen_db, scheme_names
 
 ###############################################################################
 def generate_init_routines(build_cache, bldroot, force_ccpp, force_init,

cime_config/cam_config.py

@@ -868,7 +868,7 @@ def generate_cam_src(self, gen_fort_indent):
                                           self.__atm_root, self.__bldroot,
                                           reg_dir, reg_files, source_mods_dir,
                                           force_ccpp)
-        phys_dirs, force_init, _, nml_fils, capgen_db = retvals
+        phys_dirs, force_init, _, nml_fils, capgen_db, scheme_names = retvals
 
         # Add namelist definition files to dictionary:
         for nml_fil in nml_fils:
@@ -899,6 +899,9 @@ def generate_cam_src(self, gen_fort_indent):
         #--------------------------------------------------------------
         build_cache.write()
 
+        #Return the set of all scheme names present in the SDFs:
+        return scheme_names
+
     #++++++++++++++++++++++++
 
     def ccpp_phys_set(self, cam_nml_attr_dict, phys_nl_pg_dict):

docker/Dockerfile

@@ -19,7 +19,8 @@ RUN dnf -y update \
       vim \
     && dnf clean all
 
-RUN ln -s $(which python3) /usr/bin/python && \
+RUN rm -f /usr/bin/python && \
+    ln -s $(which python3) /usr/bin/python && \
     pip install --upgrade pip && \
     pip install --upgrade setuptools