diff --git a/.dockerignore b/.dockerignore index 12b85fe8..45066338 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,14 +1,11 @@ -# ignore all -* +.github/ +.gitignore +LICENSE +README.md -# include things to copy -!Externals* -!src/ -!cime_config/ -!manage_externals/ -!test/ -!.config_files.xml -!docker -!bin/ -!.lib/ -!.gitmodules +# Ignore editor temporaries and backups +*.swp +*~ +.#* +\#*# +**/.vscode/ \ No newline at end of file diff --git a/.gitmodules b/.gitmodules index 257ce8a2..8345ca7c 100644 --- a/.gitmodules +++ b/.gitmodules @@ -20,7 +20,7 @@ [submodule "ncar-physics"] path = src/physics/ncar_ccpp url = https://github.com/ESCOMP/atmospheric_physics - fxtag = 252b500a93c89f36ece7d8ba08fd8eb025279eaa + fxtag = 37224423f62d9a71922cb6fe9beca1516d9403a4 fxrequired = AlwaysRequired fxDONOTUSEurl = https://github.com/ESCOMP/atmospheric_physics [submodule "ccs_config"] diff --git a/README.md b/README.md index 35b3bcbb..6358f935 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ cd CAM-SIMA ## NOTE: This is **unsupported** development code and is subject to the [CESM developer's agreement](http://www.cgd.ucar.edu/cseg/development-code.html). ``` git checkout development -./manage_externals/checkout_externals +./bin/git-fleximod update ``` Good luck, and have a great day! diff --git a/cime_config/atm_musica_config.py b/cime_config/atm_musica_config.py new file mode 100644 index 00000000..79cfa0bc --- /dev/null +++ b/cime_config/atm_musica_config.py @@ -0,0 +1,10 @@ +""" +The URLs and tags provided in this script are read by buildlib to build +and install the MUSICA library, as well as to download the MUSICA configuration. +""" + +MUSICA_CCPP_SCHEME_NAME = "musica_ccpp" +MUSICA_REPO_URL = "https://github.com/NCAR/musica.git" +MUSICA_TAG = "cc39bb00d2220fc81c85b22d3ceea4a39bd2bacf" +CHEMISTRY_DATA_REPO_URL = "https://github.com/NCAR/cam-sima-chemistry-data.git" +CHEMISTRY_DATA_TAG = "2b58f2410ec7a565bcf80dee16ec20f6bc35d78b" diff --git a/cime_config/buildlib b/cime_config/buildlib index 7c8b629e..1aa4e2af 100755 --- a/cime_config/buildlib +++ b/cime_config/buildlib @@ -8,9 +8,13 @@ import sys import os import filecmp import shutil +import subprocess import logging from cam_config import ConfigCAM # CAM's configure structure +from atm_musica_config import MUSICA_CCPP_SCHEME_NAME +from atm_musica_config import MUSICA_REPO_URL, MUSICA_TAG +from atm_musica_config import CHEMISTRY_DATA_REPO_URL, CHEMISTRY_DATA_TAG # Check for the CIME library, and add it # to the python path: @@ -27,6 +31,7 @@ from CIME.utils import stop_buffering_output from CIME.buildlib import parse_input from CIME.build import get_standard_makefile_args from CIME.Tools.standard_script_setup import check_minimum_python_version +from CIME.locked_files import lock_file, unlock_file #pylint: enable=wrong-import-position check_minimum_python_version(3, 7) #CAM requires version 3.7 or greater @@ -73,7 +78,7 @@ def _build_cam(): # Re-run source generator in case registry, CCPP suites, or # generator scripts have been modified, and # to extract required source code paths: - config.generate_cam_src(gen_indent) + scheme_names = config.generate_cam_src(gen_indent) dycore = config.get_value('dyn') reg_dir = config.get_value('reg_dir') @@ -122,12 +127,6 @@ def _build_cam(): paths.append(os.path.join(atm_root, "src", "dynamics", "tests", "initial_conditions")) - # If using the CMEPS/NUOPC coupler, then add additional path: - if case.get_value("COMP_INTERFACE") == "nuopc": - paths.append(os.path.join(__CIMEROOT, "src", "drivers", - "nuopc", "nuopc_cap_share")) - # End if - # Write Filepath text file with open(filepath_src, "w", encoding='utf-8') as filepath: filepath.write("\n".join(paths)) @@ -174,11 +173,32 @@ def _build_cam(): if len(optional_mpas_features) > 0: cmd += " OPTIONAL_MPAS_FEATURES=\"" + " ".join(optional_mpas_features) + "\"" + # If MUSICA-CCPP scheme is used in a suite, download + # the MUSICA configuration and build the MUSICA library + if MUSICA_CCPP_SCHEME_NAME in scheme_names: + _download_musica_configuration(caseroot) + musica_install_path = _build_musica(caseroot) + + cam_linked_libs = case.get_value("CAM_LINKED_LIBS") + musica_libs = "-lmusica-fortran -lmusica -lyaml-cpp" + if not musica_libs in cam_linked_libs: + _set_musica_lib_path(musica_install_path, caseroot) + + cmd += ' USER_INCLDIR="'\ + f'-I{os.path.join(musica_install_path, "include", "micm")} '\ + f'-I{os.path.join(musica_install_path, "include", "musica")} '\ + f'-I{os.path.join(musica_install_path, "include", "musica", "micm")} '\ + f'-I{os.path.join(musica_install_path, "include", "musica", "tuvx")} '\ + f'-I{os.path.join(musica_install_path, "include", "musica", "fortran")} '\ + '"' + retcode, out, err = run_cmd(cmd) _LOGGER.info("Command %s:\n\nstdout:\n%s\n\nstderr:\n%s\n", cmd, out, err) expect(retcode == 0, f"Command {cmd} failed with rc={retcode}") +############################################################################### def _prepare_mpas(case: Case) -> None: +############################################################################### """ Prepare MPAS build infrastructure. """ @@ -206,7 +226,9 @@ def _prepare_mpas(case: Case) -> None: shutil.copytree(os.path.normpath(os.path.join(mpas_dycore_src_root, os.pardir, os.pardir, "assets")), mpas_dycore_bld_root, copy_function=_copy2_as_needed, dirs_exist_ok=True) shutil.copytree(os.path.normpath(os.path.join(mpas_dycore_src_root, os.pardir, os.pardir, "driver")), os.path.join(mpas_dycore_bld_root, "driver"), copy_function=_copy2_as_needed, dirs_exist_ok=True) +############################################################################### def _copy2_as_needed(src: str, dst: str) -> None: +############################################################################### """ Wrapper around `shutil.copy2`. Copy the file `src` to the file or directory `dst` as needed. """ @@ -226,6 +248,184 @@ def _copy2_as_needed(src: str, dst: str) -> None: # Example scenario: User added some new source code files. shutil.copy2(src, dst) +############################################################################### +def _build_musica(clone_dest: str) -> str: +############################################################################### + """ + Builds and installs the MUSICA library. + + Args: + clone_dest: destination where the repository will be cloned + + Raises: + Exception: If configuring the CMake MUSICA project fails or + the MUSICA library build fails, an exception is raised. + + Returns: + musica_install_path: path to the MUSICA installation directory + """ + _clone_and_checkout(MUSICA_REPO_URL, MUSICA_TAG, clone_dest) + + bld_path = os.path.join(clone_dest, "musica", "build") + if os.path.exists(bld_path): + shutil.rmtree(bld_path) + os.makedirs(bld_path) + + install_dir = "install" + command = [ + "cmake", + f"-D CMAKE_INSTALL_PREFIX={install_dir}", + "-D CMAKE_BUILD_TYPE=Release", + "-D MUSICA_ENABLE_TESTS=OFF", + "-D MUSICA_BUILD_FORTRAN_INTERFACE=ON", + ".." + ] + try: + subprocess.run(command, cwd=bld_path, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, text=True, check=False) + except subprocess.CalledProcessError as e: + raise subprocess.CalledProcessError(e.returncode, e.cmd, "The subprocess \ + for cmake to configure the MUSICA CMake project failed.") from e + except FileNotFoundError as e: + raise FileNotFoundError("The 'cmake' command was not found.") from e + except OSError as e: + raise OSError("An error occurred while executing the 'cmake' command.") from e + + command = ["cmake", "--build", ".", "--target", "install"] + try: + subprocess.run(command, cwd=bld_path, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, text=True, check=False) + except subprocess.CalledProcessError as e: + raise subprocess.CalledProcessError(e.returncode, e.cmd, "The subprocess \ + for cmake to build the MUSICA library failed.") from e + except FileNotFoundError as e: + raise FileNotFoundError("The 'cmake' command was not found.") from e + except OSError as e: + raise OSError("An error occurred while executing the 'cmake' command.") from e + + musica_install_path = os.path.join(bld_path, install_dir) + + return musica_install_path + +############################################################################### +def _download_musica_configuration(download_dest: str) -> None: +############################################################################### + """ + Downloads the MUSICA configuration and renames the configuration + directory to match the name in the MUSICA-CCPP configuration. + + Args: + download_dest: destination where configuration will be downloaded + + Raises: + Exception: If the directory to be renamed is not found or + any other exceptions occur during the renaming process, + an exception is raised with the error message. + """ + musica_config_dir_name = "musica_configurations" + + _clone_and_checkout(CHEMISTRY_DATA_REPO_URL, CHEMISTRY_DATA_TAG, download_dest) + + original_dir = os.path.join(download_dest, "cam-sima-chemistry-data", "mechanisms") + renamed_dir = os.path.join(download_dest, "cam-sima-chemistry-data", musica_config_dir_name) + try: + os.rename(original_dir, renamed_dir) + except FileNotFoundError as e: + raise FileNotFoundError(f"The directory '{original_dir}' was not found.") from e + except FileExistsError as e: + raise FileExistsError(f"The destination directory '{renamed_dir}' already exists.") from e + except PermissionError as e: + raise PermissionError(f"Permission denied to rename '{original_dir}'.") from e + except OSError as e: + raise OSError("An error occurred while renaming.") from e + + musica_config_path = os.path.join(download_dest, musica_config_dir_name) + if os.path.exists(musica_config_path): + shutil.rmtree(musica_config_path) + + shutil.move(renamed_dir, download_dest) + +############################################################################### +def _set_musica_lib_path(musica_install_path: str, caseroot: str) -> None: +############################################################################### + """ + Sets the MUSICA libraries path to CAM_LINKED_LIBS, allowing the libraries + to be linked during the CESM build process. + + Args: + musica_install_path: path to the MUSICA installation directory + caseroot: CASEROOT where the xmlchange command is located + + Raises: + Exception: If the subprocess for the xmlchange command fails, + an exception is raised with the error message. + """ + + unlock_file("env_build.xml", caseroot) + + command = [ + "./xmlchange", + "--append", + # The libraries must be on the same line because CIME flags an + # error for multi-character arguments preceded by a single dash + f"CAM_LINKED_LIBS=-L{os.path.join(musica_install_path, 'lib64')} -lmusica-fortran -lmusica -lyaml-cpp" + ] + try: + subprocess.run(command, cwd=caseroot, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, text=True, check=False) + except subprocess.CalledProcessError as e: + raise subprocess.CalledProcessError(e.returncode, e.cmd, "The subprocess \ + for xmlchange to set the MUSICA library path failed.") from e + except FileNotFoundError as e: + raise FileNotFoundError("The 'xmlchange' command was not found.") from e + except OSError as e: + raise OSError("An error occurred while executing the 'xmlchange' command.") from e + + lock_file("env_build.xml", caseroot) + +############################################################################### +def _clone_and_checkout(repo_url: str, tag_name: str, clone_dest: str) -> None: +############################################################################### + """ + Clones a Git repository from the URL and checks out a specific branch. + + Args: + repo_url: URL of the Git repository to clone + tag_name: tag name to check out + clone_dest: destination where the repository will be cloned + + Raises: + Exception: If the `git clone` or `git checkout` commands fail, + an exception is raised with the error message. + """ + repo_name = repo_url.split("/")[-1].replace(".git", "") + repo_path = os.path.join(clone_dest, repo_name) + + if os.path.exists(repo_path): + shutil.rmtree(repo_path) + + try: + subprocess.run(["git", "clone", repo_url, repo_path], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, check=False) + except subprocess.CalledProcessError as e: + raise subprocess.CalledProcessError(e.returncode, e.cmd, f"The subprocess \ + for git to clone the repository {repo_url} failed.") from e + except FileNotFoundError as e: + raise FileNotFoundError("The 'git' command was not found.") from e + except OSError as e: + raise OSError("An error occurred while executing the 'git' command.") from e + + try: + subprocess.run(["git", "-C", repo_path, "checkout", tag_name], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, check=False) + except subprocess.CalledProcessError as e: + raise subprocess.CalledProcessError(e.returncode, e.cmd, f"The subprocess \ + for git to checkout the branch {tag_name} failed.") from e + except FileNotFoundError as e: + raise FileNotFoundError("The 'git' command was not found.") from e + except OSError as e: + raise OSError("An error occurred while executing the 'git' command.") from e + ############################################################################### if __name__ == "__main__": diff --git a/cime_config/buildnml b/cime_config/buildnml index c00131e3..32aa01a2 100755 --- a/cime_config/buildnml +++ b/cime_config/buildnml @@ -166,7 +166,7 @@ def buildnml(case, caseroot, compname): gen_indent = 3 #Generate model code and meta-data: - config.generate_cam_src(gen_indent) + _ = config.generate_cam_src(gen_indent) #---------------------------------------------------------------- # Create namelist attribute dictionary (to set namelist defaults): diff --git a/cime_config/cam_autogen.py b/cime_config/cam_autogen.py index 50b77ca4..0ebf9e01 100644 --- a/cime_config/cam_autogen.py +++ b/cime_config/cam_autogen.py @@ -452,6 +452,7 @@ def generate_physics_suites(build_cache, preproc_defs, host_name, # Find the SDFs specified for this model build sdfs = [] scheme_files = [] + scheme_names = set() xml_files = {} # key is scheme, value is xml file path for sdf in phys_suites_str.split(';'): sdf_path = _find_file(f"suite_{sdf}.xml", suite_search) @@ -467,6 +468,8 @@ def generate_physics_suites(build_cache, preproc_defs, host_name, # Given an SDF, find all the schemes it calls _, suite = read_xml_file(sdf_path) sdf_schemes = _find_schemes_in_sdf(suite) + #Add schemes to set of all scheme names: + scheme_names.update(sdf_schemes) # For each scheme, find its metadata file for scheme in sdf_schemes: if scheme in all_scheme_files: @@ -655,7 +658,7 @@ def generate_physics_suites(build_cache, preproc_defs, host_name, # End if return [physics_blddir, genccpp_dir], do_gen_ccpp, cap_output_file, \ - xml_files.values(), capgen_db + xml_files.values(), capgen_db, scheme_names ############################################################################### def generate_init_routines(build_cache, bldroot, force_ccpp, force_init, diff --git a/cime_config/cam_config.py b/cime_config/cam_config.py index f47fdbee..20e3b3f5 100644 --- a/cime_config/cam_config.py +++ b/cime_config/cam_config.py @@ -868,7 +868,7 @@ def generate_cam_src(self, gen_fort_indent): self.__atm_root, self.__bldroot, reg_dir, reg_files, source_mods_dir, force_ccpp) - phys_dirs, force_init, _, nml_fils, capgen_db = retvals + phys_dirs, force_init, _, nml_fils, capgen_db, scheme_names = retvals # Add namelist definition files to dictionary: for nml_fil in nml_fils: @@ -899,6 +899,9 @@ def generate_cam_src(self, gen_fort_indent): #-------------------------------------------------------------- build_cache.write() + #Return the set of all scheme names present in the SDFs: + return scheme_names + #++++++++++++++++++++++++ def ccpp_phys_set(self, cam_nml_attr_dict, phys_nl_pg_dict): diff --git a/docker/Dockerfile b/docker/Dockerfile index 098db37d..5e3e33e5 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -19,7 +19,8 @@ RUN dnf -y update \ vim \ && dnf clean all -RUN ln -s $(which python3) /usr/bin/python && \ +RUN rm -f /usr/bin/python && \ + ln -s $(which python3) /usr/bin/python && \ pip install --upgrade pip && \ pip install --upgrade setuptools diff --git a/docker/Dockerfile.esmf b/docker/Dockerfile.esmf index 4a678b32..ff6976b2 100644 --- a/docker/Dockerfile.esmf +++ b/docker/Dockerfile.esmf @@ -29,7 +29,7 @@ ENV OMP_NUM_THREADS=5 ## Build and install ESMF ################################################### -ENV ESMF_TAG="8.4.2" +ENV ESMF_TAG="8.6.0" # set necessary environment variables ENV ESMF_DIR=/esmf-${ESMF_TAG} @@ -51,4 +51,4 @@ RUN wget -q https://github.com/esmf-org/esmf/archive/refs/tags/v${ESMF_TAG}.tar. # This command lets you see what esmf thinks its build options are but may not necessary to build, not sure make info && \ make -j 8 && \ - make install + make install \ No newline at end of file diff --git a/docker/Dockerfile.musica b/docker/Dockerfile.musica index 0f59f21d..f5deedbc 100644 --- a/docker/Dockerfile.musica +++ b/docker/Dockerfile.musica @@ -2,6 +2,8 @@ # esmf is am image you are expected to have built. Read the README file for instructions FROM --platform=linux/amd64 esmf:latest +ARG BUILD_TYPE=Debug + ################################################### ## Install necessary packages ################################################### @@ -19,7 +21,8 @@ RUN dnf -y update \ vim \ && dnf clean all -RUN ln -s $(which python3) /usr/bin/python && \ +RUN rm -f /usr/bin/python && \ + ln -s $(which python3) /usr/bin/python && \ pip install --upgrade pip && \ pip install --upgrade setuptools @@ -39,40 +42,22 @@ RUN cd pnetcdf-1.12.3 && \ make -j 8 install && \ ldconfig -ENV FC=gfortran - -################################################### -## Build and install MUSICA -################################################### - -RUN git clone https://github.com/NCAR/musica.git \ - && cd musica \ - && git checkout 2a5eeaac982a3eb80b96d1e2087b91b301d1e748 - -RUN mkdir /musica/build \ - && cd /musica/build \ - && cmake \ - -D ENABLE_TESTS=OFF \ - -D MUSICA_BUILD_FORTRAN_INTERFACE=ON \ - .. \ - && make install -j 8 - ################################################### ## Build CAM-SIMA ################################################### -# create a user to run the case +# Create a user to run the case RUN adduser cam_sima_user \ && echo "cam_sima_user ALL=(root) NOPASSWD:ALL" > /etc/sudoers.d/cam_sima_user \ && chmod 0440 /etc/sudoers.d/cam_sima_user -# copy in the CAM-SIMA code and give the proper user permissions +# Copy in the CAM-SIMA code and give the proper user permissions COPY --chown=cam_sima_user . /home/cam_sima_user/CAM-SIMA USER cam_sima_user WORKDIR /home/cam_sima_user/CAM-SIMA -# pull the dependencies +# Pull the dependencies RUN ./bin/git-fleximod update # Copy in the machine information for the container @@ -80,30 +65,23 @@ RUN cp /home/cam_sima_user/CAM-SIMA/docker/config_machines.xml /home/cam_sima_us # Set environment variables needed to create and build the case ENV USER=$(whoami) -ENV CASE_NAME=/home/cam_sima_user/case_name +ENV CASE_NAME=/home/cam_sima_user/case_name/test-case ENV CESMDATAROOT=/home/cam_sima_user/cesm_data ENV CIME_MACHINE=container ENV CIME_MODEL=cesm ENV ESMFMKFILE=/usr/local/lib/esmf.mk # Create a case -RUN ./cime/scripts/create_newcase --case $CASE_NAME --compset FPHYStest --res ne5_ne5_mg37 --run-unsupported +RUN /home/cam_sima_user/CAM-SIMA/cime/scripts/create_newcase --case $CASE_NAME \ + --compset FPHYStest --res ne5_ne5_mg37 --run-unsupported -WORKDIR $CASE_NAME +WORKDIR $CASE_NAME RUN ./case.setup RUN ./xmlchange CAM_CONFIG_OPTS="--dyn none --physics-suites musica" -RUN ./xmlchange CAM_LINKED_LIBS="-lmusica-fortran -lmusica -lyaml-cpp" RUN ./xmlchange ROF_NCPL=48 RUN ./xmlchange STOP_OPTION=nsteps RUN ./xmlchange STOP_N=5 -# Copy in the grid files and a snapshot file -RUN chmod +x /home/cam_sima_user/CAM-SIMA/docker/ftp_download.sh -RUN /home/cam_sima_user/CAM-SIMA/docker/ftp_download.sh - -# # add the snapshot file -RUN echo "ncdata='/home/cam_sima_user/run_heldsuarez_cam6_nt2_bigg_try005.cam.h5.0001-01-01-00000.nc'" >> user_nl_cam - -RUN ./case.build +RUN ./case.build \ No newline at end of file diff --git a/src/physics/ncar_ccpp b/src/physics/ncar_ccpp index 252b500a..37224423 160000 --- a/src/physics/ncar_ccpp +++ b/src/physics/ncar_ccpp @@ -1 +1 @@ -Subproject commit 252b500a93c89f36ece7d8ba08fd8eb025279eaa +Subproject commit 37224423f62d9a71922cb6fe9beca1516d9403a4 diff --git a/src/physics/utils/musica_ccpp_dependencies.meta b/src/physics/utils/musica_ccpp_dependencies.meta index c514b661..c7ceb258 100644 --- a/src/physics/utils/musica_ccpp_dependencies.meta +++ b/src/physics/utils/musica_ccpp_dependencies.meta @@ -30,7 +30,7 @@ protected = True [ surface_albedo ] standard_name = surface_albedo_due_to_UV_and_VIS_direct - units = none + units = fraction type = real | kind = kind_phys dimensions = (horizontal_dimension) protected = True diff --git a/test/unit/python/test_cam_autogen.py b/test/unit/python/test_cam_autogen.py index e199bd02..48fd3223 100644 --- a/test/unit/python/test_cam_autogen.py +++ b/test/unit/python/test_cam_autogen.py @@ -580,7 +580,7 @@ def test_generate_physics_suites(self): expected_results = ([f'{self.test_bldroot}'+os.sep+'ccpp_physics', f'{self.test_bldroot}'+os.sep+'ccpp'], False, f'{self.test_bldroot}'+os.sep+'ccpp'+os.sep+'ccpp_datatable.xml', - [], None) + [], None, {"temp_adjust"}) #Run physics suite generation function: gen_results = generate_physics_suites(self.test_cache, "UNSET", "cam", "simple",