buildlib

#!/usr/bin/env python3

"""
create the cam library
"""
# Python system imports
import sys
import os
import filecmp
import shutil
import subprocess
import logging

from cam_config import ConfigCAM # CAM's configure structure
from atm_musica_config import MUSICA_CCPP_SCHEME_NAME
from atm_musica_config import MUSICA_REPO_URL, MUSICA_TAG
from atm_musica_config import CHEMISTRY_DATA_REPO_URL, CHEMISTRY_DATA_TAG

# Check for the CIME library, and add it
# to the python path:
__CIMEROOT = os.environ.get("CIMEROOT")
if __CIMEROOT is None:
    raise SystemExit("ERROR: must set CIMEROOT environment variable")
sys.path.append(os.path.join(__CIMEROOT, "CIME", "Tools"))

#pylint: disable=wrong-import-position
# CIME imports
from CIME.case import Case
from CIME.utils import run_cmd, expect
from CIME.utils import stop_buffering_output
from CIME.buildlib import parse_input
from CIME.build import get_standard_makefile_args
from CIME.Tools.standard_script_setup import check_minimum_python_version
from CIME.locked_files import lock_file, unlock_file
#pylint: enable=wrong-import-position

check_minimum_python_version(3, 7) #CAM requires version 3.7 or greater
stop_buffering_output()

_LOGGER = logging.getLogger(__name__)

###############################################################################
def _build_cam():
###############################################################################
    """Configure and build CAM"""
    caseroot, libroot, bldroot = parse_input(sys.argv)

    with Case(caseroot) as case:

        # Case variables
        casetools = case.get_value("CASETOOLS")
        atm_root = case.get_value("COMP_ROOT_DIR_ATM")
        gmake_j = case.get_value("GMAKE_J")
        gmake = case.get_value("GMAKE")
        exeroot = case.get_value("EXEROOT")

        # Case-derived variables
        source_mods_dir = os.path.join(caseroot, "SourceMods", "src.cam")
        bldroot = os.path.join(exeroot, "atm", "obj")

        # Set logging level
        if case.get_value("DEBUG"):
            # CIME blocks this change so create a handler with DEBUG level
            handler = logging.StreamHandler(stream=sys.stdout)
            handler.setLevel(logging.DEBUG)
            _LOGGER.handlers = [handler]
            _LOGGER.setLevel(logging.DEBUG)
        else:
            _LOGGER.setLevel(logging.INFO)
        # End if

        # Create CAM config object:
        config = ConfigCAM(case, _LOGGER)

        #Set number of spaces used to indicate scope in generated code:
        gen_indent = 3

        # Re-run source generator in case registry, CCPP suites, or
        # generator scripts have been modified, and
        # to extract required source code paths:
        scheme_names = config.generate_cam_src(gen_indent)

        dycore = config.get_value('dyn')
        reg_dir = config.get_value('reg_dir')
        init_dir = config.get_value('init_dir')
        phys_dirs_str = config.get_value('phys_dirs')

        #Convert the phys_dirs_str into a proper list:
        phys_dirs = phys_dirs_str.split(';')

        #-------------------------------------------------------
        # Create the Filepath file with all the paths needed
        # to build this configuration of CAM
        #-------------------------------------------------------
        filepath_src = os.path.join(caseroot, "Buildconf",
                                    "camconf", "Filepath")
        filepath_dst = os.path.join(bldroot, "Filepath")
        paths = [source_mods_dir, reg_dir, init_dir,
                 os.path.join(atm_root, "src", "data"),
                 os.path.join(atm_root, "src", "control"),
                 os.path.join(atm_root, "src", "cpl",
                              case.get_value("COMP_INTERFACE")),
                 os.path.join(atm_root, "src", "dynamics", "utils"),
                 os.path.join(atm_root, "src", "physics", "utils"),
                 os.path.join(atm_root, "src", "history"),
                 os.path.join(atm_root, "src", "history", "buffers", "src"),
                 os.path.join(atm_root, "src", "history", "buffers", "src", "hash"),
                 os.path.join(atm_root, "src", "history", "buffers", "src", "util"),
                 os.path.join(atm_root, "src", "utils"),
                 os.path.join(atm_root, "src", "core_utils")]
        for path in phys_dirs:
            if path not in paths:
                paths.append(path)
            # End if
        # End for

        # Add dynamics source code directories:
        for direc in config.get_value("dyn_src_dirs"):
            dyn_dir = os.path.join(atm_root, "src", "dynamics", direc)
            if dyn_dir not in paths:
                #Add to list of filepaths if not already present:
                paths.append(dyn_dir)

        # Add analytical IC source code directories:
        paths.append(os.path.join(atm_root, "src", "dynamics", "tests")) #Required due to namelist call.
        if dycore != "none":
            paths.append(os.path.join(atm_root, "src", "dynamics", "tests",
                                      "initial_conditions"))

        # Write Filepath text file
        with open(filepath_src, "w", encoding='utf-8') as filepath:
            filepath.write("\n".join(paths))
            filepath.write("\n")
        # End with

        # Move Filepath to the bld directory unless it has not changed
        if os.path.isfile(filepath_dst):
            if not filecmp.cmp(filepath_src, filepath_dst):
                shutil.move(filepath_src, filepath_dst)
            # End if
        else:
            shutil.move(filepath_src, filepath_dst)
        # End if

        #-------------------------------------------------------
        # build the library
        #-------------------------------------------------------

        complib = os.path.join(libroot, "libatm.a")
        makefile = os.path.join(casetools, "Makefile")

        cmd = f"{gmake} complib -j {gmake_j} MODEL=cam COMPLIB={complib}"
        cmd += f" -f {makefile} {get_standard_makefile_args(case)}"

        # Add C Pre-Processor (CPP) definitions, if present:
        if config.cpp_defs:
            ccpp_defs_str = ' '.join(config.cpp_defs)
            cmd += f" USER_CPPDEFS='{ccpp_defs_str}'"

        # If dynamical core is MPAS, prepare its build infrastructure so it can be built along with CAM-SIMA below.
        if dycore == "mpas":
            _prepare_mpas(case)

            optional_mpas_features = []

            if case.get_value("USE_ESMF_LIB"):
                optional_mpas_features.append("+ESMF_LIBRARY")
                optional_mpas_features.append("+PERF_MOD_LIBRARY")

            if config.get_value("dyn_kind") == "REAL32":
                optional_mpas_features.append("+SINGLE_PRECISION")

            if len(optional_mpas_features) > 0:
                cmd += " OPTIONAL_MPAS_FEATURES=\"" + " ".join(optional_mpas_features) + "\""

        # If MUSICA-CCPP scheme is used in a suite, download
        # the MUSICA configuration and build the MUSICA library
        if MUSICA_CCPP_SCHEME_NAME in scheme_names:
            _download_musica_configuration(caseroot)
            musica_install_path = _build_musica(caseroot)

            cam_linked_libs = case.get_value("CAM_LINKED_LIBS")
            musica_libs = "-lmusica-fortran -lmusica -lyaml-cpp"
            if not musica_libs in cam_linked_libs:
                _set_musica_lib_path(musica_install_path, caseroot)

            cmd += ' USER_INCLDIR="'\
                    f'-I{os.path.join(musica_install_path, "include", "micm")} '\
                    f'-I{os.path.join(musica_install_path, "include", "musica")} '\
                    f'-I{os.path.join(musica_install_path, "include", "musica", "micm")} '\
                    f'-I{os.path.join(musica_install_path, "include", "musica", "tuvx")} '\
                    f'-I{os.path.join(musica_install_path, "include", "musica", "fortran")} '\
                    '"'

        retcode, out, err = run_cmd(cmd)
        _LOGGER.info("Command %s:\n\nstdout:\n%s\n\nstderr:\n%s\n", cmd, out, err)
        expect(retcode == 0, f"Command {cmd} failed with rc={retcode}")

###############################################################################
def _prepare_mpas(case: Case) -> None:
###############################################################################
    """
    Prepare MPAS build infrastructure.
    """

    atm_src_root = os.path.normpath(case.get_value("COMP_ROOT_DIR_ATM"))
    atm_bld_root = os.path.normpath(os.path.join(case.get_value("EXEROOT"), "atm", "obj"))

    mpas_dycore_src_root = os.path.join(atm_src_root, "src", "dynamics", "mpas", "dycore", "src")
    mpas_dycore_bld_root = os.path.join(atm_bld_root, "mpas")

    # Make sure `mpas_dycore_src_root` exists. If not, it is likely that `./bin/git-fleximod update` did not succeed.
    if os.path.isfile(mpas_dycore_src_root):
        raise FileExistsError(1, "Unexpected file", mpas_dycore_src_root)

    if not os.path.isdir(mpas_dycore_src_root):
        raise FileNotFoundError(1, "No such directory", mpas_dycore_src_root)

    # MPAS supports "in-source" build only. Copy source code to `mpas_dycore_bld_root` to avoid polluting `mpas_dycore_src_root`.
    if os.path.isfile(mpas_dycore_bld_root):
        raise FileExistsError(1, "Unexpected file", mpas_dycore_bld_root)

    shutil.copytree(mpas_dycore_src_root, mpas_dycore_bld_root, copy_function=_copy2_as_needed, dirs_exist_ok=True)
    shutil.move(os.path.join(mpas_dycore_bld_root, "Makefile"), os.path.join(mpas_dycore_bld_root, "Makefile.CESM"))

    shutil.copytree(os.path.normpath(os.path.join(mpas_dycore_src_root, os.pardir, os.pardir, "assets")), mpas_dycore_bld_root, copy_function=_copy2_as_needed, dirs_exist_ok=True)
    shutil.copytree(os.path.normpath(os.path.join(mpas_dycore_src_root, os.pardir, os.pardir, "driver")), os.path.join(mpas_dycore_bld_root, "driver"), copy_function=_copy2_as_needed, dirs_exist_ok=True)

###############################################################################
def _copy2_as_needed(src: str, dst: str) -> None:
###############################################################################
    """
    Wrapper around `shutil.copy2`. Copy the file `src` to the file or directory `dst` as needed.
    """

    if os.path.isfile(src):
        if os.path.isfile(dst):
            if int(os.path.getmtime(src)) != int(os.path.getmtime(dst)) or os.path.getsize(src) != os.path.getsize(dst):
                # `src` and `dst` are both files but their modification time or size differ.
                # Example scenario: User modified some existing source code files.
                shutil.copy2(src, dst)
        elif os.path.isdir(dst) and os.path.isfile(os.path.join(dst, os.path.basename(src))):
            dst = os.path.join(dst, os.path.basename(src))
            _copy2_as_needed(src, dst)
        else:
            if os.path.isdir(dst) or os.path.isdir(os.path.dirname(dst)):
                # `src` is a new file that does not exist at `dst`.
                # Example scenario: User added some new source code files.
                shutil.copy2(src, dst)

###############################################################################
def _build_musica(clone_dest: str) -> str:
###############################################################################
    """
    Builds and installs the MUSICA library.

    Args:
        clone_dest: destination where the repository will be cloned

    Raises:
        Exception: If configuring the CMake MUSICA project fails or
            the MUSICA library build fails, an exception is raised.

    Returns:
        musica_install_path: path to the MUSICA installation directory
    """
    _clone_and_checkout(MUSICA_REPO_URL, MUSICA_TAG, clone_dest)

    bld_path = os.path.join(clone_dest, "musica", "build")
    if os.path.exists(bld_path):
        shutil.rmtree(bld_path)
    os.makedirs(bld_path)

    install_dir = "install"
    command = [
        "cmake",
        f"-D CMAKE_INSTALL_PREFIX={install_dir}",
        "-D CMAKE_BUILD_TYPE=Release",
        "-D MUSICA_ENABLE_TESTS=OFF",
        "-D MUSICA_BUILD_FORTRAN_INTERFACE=ON",
        ".."
    ]
    try:
        subprocess.run(command, cwd=bld_path, stdout=subprocess.PIPE,
                       stderr=subprocess.PIPE, text=True, check=False)
    except subprocess.CalledProcessError as e:
        raise subprocess.CalledProcessError(e.returncode, e.cmd, "The subprocess \
            for cmake to configure the MUSICA CMake project failed.") from e
    except FileNotFoundError as e:
        raise FileNotFoundError("The 'cmake' command was not found.") from e
    except OSError as e:
        raise OSError("An error occurred while executing the 'cmake' command.") from e

    command = ["cmake", "--build", ".", "--target", "install"]
    try:
        subprocess.run(command, cwd=bld_path, stdout=subprocess.PIPE,
                       stderr=subprocess.PIPE, text=True, check=False)
    except subprocess.CalledProcessError as e:
        raise subprocess.CalledProcessError(e.returncode, e.cmd, "The subprocess \
            for cmake to build the MUSICA library failed.") from e
    except FileNotFoundError as e:
        raise FileNotFoundError("The 'cmake' command was not found.") from e
    except OSError as e:
        raise OSError("An error occurred while executing the 'cmake' command.") from e

    musica_install_path = os.path.join(bld_path, install_dir)

    return musica_install_path

###############################################################################
def _download_musica_configuration(download_dest: str) -> None:
###############################################################################
    """
    Downloads the MUSICA configuration and renames the configuration
    directory to match the name in the MUSICA-CCPP configuration.

    Args:
        download_dest: destination where configuration will be downloaded

    Raises:
        Exception: If the directory to be renamed is not found or
            any other exceptions occur during the renaming process,
            an exception is raised with the error message.
    """
    musica_config_dir_name = "musica_configurations"

    _clone_and_checkout(CHEMISTRY_DATA_REPO_URL, CHEMISTRY_DATA_TAG, download_dest)

    original_dir = os.path.join(download_dest, "cam-sima-chemistry-data", "mechanisms")
    renamed_dir = os.path.join(download_dest, "cam-sima-chemistry-data", musica_config_dir_name)
    try:
        os.rename(original_dir, renamed_dir)
    except FileNotFoundError as e:
        raise FileNotFoundError(f"The directory '{original_dir}' was not found.") from e
    except FileExistsError as e:
        raise FileExistsError(f"The destination directory '{renamed_dir}' already exists.") from e
    except PermissionError as e:
        raise PermissionError(f"Permission denied to rename '{original_dir}'.") from e
    except OSError as e:
        raise OSError("An error occurred while renaming.") from e

    musica_config_path = os.path.join(download_dest, musica_config_dir_name)
    if os.path.exists(musica_config_path):
        shutil.rmtree(musica_config_path)

    shutil.move(renamed_dir, download_dest)

###############################################################################
def _set_musica_lib_path(musica_install_path: str, caseroot: str) -> None:
###############################################################################
    """
    Sets the MUSICA libraries path to CAM_LINKED_LIBS, allowing the libraries
    to be linked during the CESM build process.

    Args:
        musica_install_path: path to the MUSICA installation directory
        caseroot: CASEROOT where the xmlchange command is located

    Raises:
        Exception: If the subprocess for the xmlchange command fails,
            an exception is raised with the error message.
    """

    unlock_file("env_build.xml", caseroot)

    command = [
        "./xmlchange",
        "--append",
        # The libraries must be on the same line because CIME flags an
        # error for multi-character arguments preceded by a single dash
        f"CAM_LINKED_LIBS=-L{os.path.join(musica_install_path, 'lib64')} -lmusica-fortran -lmusica -lyaml-cpp"
    ]
    try:
        subprocess.run(command, cwd=caseroot, stdout=subprocess.PIPE,
                       stderr=subprocess.PIPE, text=True, check=False)
    except subprocess.CalledProcessError as e:
        raise subprocess.CalledProcessError(e.returncode, e.cmd, "The subprocess \
            for xmlchange to set the MUSICA library path failed.") from e
    except FileNotFoundError as e:
        raise FileNotFoundError("The 'xmlchange' command was not found.") from e
    except OSError as e:
        raise OSError("An error occurred while executing the 'xmlchange' command.") from e

    lock_file("env_build.xml", caseroot)

###############################################################################
def _clone_and_checkout(repo_url: str, tag_name: str, clone_dest: str) -> None:
###############################################################################
    """
    Clones a Git repository from the URL and checks out a specific branch.

    Args:
        repo_url: URL of the Git repository to clone
        tag_name: tag name to check out
        clone_dest: destination where the repository will be cloned

    Raises:
        Exception: If the `git clone` or `git checkout` commands fail,
            an exception is raised with the error message.
    """
    repo_name = repo_url.split("/")[-1].replace(".git", "")
    repo_path = os.path.join(clone_dest, repo_name)

    if os.path.exists(repo_path):
        shutil.rmtree(repo_path)

    try:
        subprocess.run(["git", "clone", repo_url, repo_path],
           stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, check=False)
    except subprocess.CalledProcessError as e:
        raise subprocess.CalledProcessError(e.returncode, e.cmd, f"The subprocess \
            for git to clone the repository {repo_url} failed.") from e
    except FileNotFoundError as e:
        raise FileNotFoundError("The 'git' command was not found.") from e
    except OSError as e:
        raise OSError("An error occurred while executing the 'git' command.") from e

    try:
        subprocess.run(["git", "-C", repo_path, "checkout", tag_name],
           stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, check=False)
    except subprocess.CalledProcessError as e:
        raise subprocess.CalledProcessError(e.returncode, e.cmd, f"The subprocess \
            for git to checkout the branch {tag_name} failed.") from e
    except FileNotFoundError as e:
        raise FileNotFoundError("The 'git' command was not found.") from e
    except OSError as e:
        raise OSError("An error occurred while executing the 'git' command.") from e

###############################################################################

if __name__ == "__main__":
    _build_cam()