Merge pull request NCAR#161 from mnlevy1981/workflow_updates

Workflow updates

Author: TeaganKing

cupid/run_timeseries.py

@@ -112,7 +112,10 @@ def run_timeseries(
             if isinstance(timeseries_params["case_name"], list):
                 ts_input_dirs = []
                 for cname in timeseries_params["case_name"]:
-                    ts_input_dirs.append(global_params["CESM_output_dir"]+"/"+cname+f"/{component}/hist/")
+                    if cname == global_params["base_case_name"]:
+                        ts_input_dirs.append(global_params["base_case_output_dir"]+"/"+cname+f"/{component}/hist/")
+                    else:
+                        ts_input_dirs.append(global_params["CESM_output_dir"]+"/"+cname+f"/{component}/hist/")
             else:
                 ts_input_dirs = [
                     global_params["CESM_output_dir"] + "/" +

cupid/timeseries.py

@@ -111,7 +111,7 @@ def create_time_series(
 
         # Check that path actually exists:
         if not starting_location.is_dir():
-            emsg = "Provided 'cam_hist_loc' directory '{starting_location}' not found."
+            emsg = f"Provided 'cam_hist_loc' directory '{starting_location}' not found."
             emsg += " Script is ending here."
 
             raise FileNotFoundError(emsg)

examples/key_metrics/config.yml

@@ -142,7 +142,7 @@ compute_notebooks:
       Greenland_SMB_visual_compare_obs:
         parameter_groups:
           none:
-            obs_path: '/glade/u/home/gunterl/obs_diagnostic_cesm/'
+            obs_path: '/glade/campaign/cesm/development/cross-wg/diagnostic_framework/SMB_data'
             obs_name: 'GrIS_MARv3.12_climo_1960_1999.nc'
             climo_nyears: 40
 

helper_scripts/generate_adf_config_file.py

@@ -25,10 +25,10 @@ def _parse_args():
     )
     # Command line argument for CUPiD example from which to get config.yml
     parser.add_argument(
-        "--cupid-example",
+        "--cupid-config-loc",
         action="store",
-        dest="cupid_example",
-        default="external_diag_packages",
+        dest="cupid_config_loc",
+        default=None,
         help="CUPiD example to use as template for config.yml",
     )
     parser.add_argument(
@@ -46,27 +46,20 @@ def _parse_args():
     return parser.parse_args()
 
 
-def generate_adf_config(cesm_root, cupid_example, adf_file, out_file):
-    """Use cupid config file (YAML) from cupid_example and adf_file (YAML)
+def generate_adf_config(cesm_root, cupid_config_loc, adf_file, out_file):
+    """Use cupid config file (YAML) from cupid_config_loc and adf_file (YAML)
     to produce out_file by modifying adf_file with data from cupid config file.
     """
     sys.path.append(os.path.join(cesm_root, "cime"))
 
-    # Is cupid_example a valid value?
     cupid_root = os.path.join(cesm_root, "tools", "CUPiD")
-    cupid_examples = os.path.join(cupid_root, "examples")
-    valid_examples = [
-        example
-        for example in next(os.walk(cupid_examples))[1]
-        if example not in ["ilamb"]
-    ]
-    if cupid_example not in valid_examples:
-        error_msg = f"argument --cupid-example: invalid choice '{cupid_example}'"
-        raise KeyError(
-            f"{error_msg} (choose from subdirectories of {cupid_examples}: {valid_examples})",
-        )
+    # Is cupid_config_loc a valid value?
+    if cupid_config_loc is None:
+        cupid_config_loc = os.path.join(cupid_root, "examples", "key_metrics")
+    if not os.path.exists(os.path.join(cupid_config_loc, "config.yml")):
+        raise KeyError(f"Can not find config.yml in {cupid_config_loc}")
 
-    with open(os.path.join(cupid_root, "examples", cupid_example, "config.yml")) as c:
+    with open(os.path.join(cupid_config_loc, "config.yml")) as c:
         c_dict = yaml.safe_load(c)
     with open(adf_file, encoding="UTF-8") as a:
         a_dict = yaml.safe_load(a)
@@ -141,9 +134,9 @@ def generate_adf_config(cesm_root, cupid_example, adf_file, out_file):
         ts_case_names.index(base_case_name) if base_case_name in ts_case_names else None
     )
 
-    base_case_output_dir = c_dict["global_params"].get(
-        "base_case_output_dir",
-        DOUT + "/" + base_case_name,
+    base_case_output_dir = os.path.join(
+        c_dict["global_params"].get("base_case_output_dir", DOUT),
+        base_case_name,
     )
     base_start_date = get_date_from_ts(
         c_ts["atm"],
@@ -208,9 +201,7 @@ def generate_adf_config(cesm_root, cupid_example, adf_file, out_file):
         "regrid",
     )  # This is where ADF will make "regrid" files
     a_dict["diag_basic_info"]["cam_diag_plot_loc"] = os.path.join(
-        cupid_root,
-        "examples",
-        cupid_example,
+        cupid_config_loc,
         "ADF_output",
     )  # this is where ADF will put plots, and "website" directory
     a_dict["user"] = os.environ["USER"]
@@ -255,10 +246,10 @@ def generate_adf_config(cesm_root, cupid_example, adf_file, out_file):
         f.write(
             "# This file has been auto-generated using generate_adf_config_file.py\n",
         )
-        f.write(f"# It is based off of examples/{cupid_example}/config.yml\n")
+        f.write(f"# It is based off of {cupid_config_loc}/config.yml\n")
         f.write("# Arguments:\n")
         f.write(f"# {cesm_root=}\n")
-        f.write(f"# {cupid_example=}\n")
+        f.write(f"# {cupid_config_loc=}\n")
         f.write(f"# {adf_file=}\n")
         f.write(f"# Output: {out_file=}\n")
         # enter in each element of the dictionary into the new file
@@ -284,7 +275,7 @@ def get_date_from_ts(data: dict, keyname: str, listindex: int, default=None):
     print(args)
     generate_adf_config(
         args["cesm_root"],
-        args["cupid_example"],
+        args["cupid_config_loc"],
         args["adf_template"],
         args["out_file"],
     )

helper_scripts/generate_cupid_config_for_cesm_case.py

@@ -108,10 +108,11 @@ def generate_cupid_config(case_root, cesm_root, cupid_example):
         dout_s_root = cesm_case.get_value("DOUT_S_ROOT")
 
     # Additional options we need to get from env_cupid.xml
+    base_case = "b.e23_alpha17f.BLT1850.ne30_t232.092"
     nyears = 1
     start_date = "0001-01-01"
     end_date = f"{nyears+1:04d}-01-01"
-    climo_nyears = 1
+    climo_nyears = nyears
     base_case_output_dir = "/glade/campaign/cesm/development/cross-wg/diagnostic_framework/CESM_output_for_testing"
     base_nyears = 100
     base_end_date = f"{base_nyears+1:04d}-01-01"
@@ -120,18 +121,40 @@ def generate_cupid_config(case_root, cesm_root, cupid_example):
     with open(os.path.join(cupid_root, "examples", cupid_example, "config.yml")) as f:
         my_dict = yaml.safe_load(f)
 
+    my_dict["data_sources"]["nb_path_root"] = os.path.join(
+        cesm_root,
+        "tools",
+        "CUPiD",
+        "nblibrary",
+    )
     my_dict["global_params"]["case_name"] = case
     my_dict["global_params"]["start_date"] = start_date
     my_dict["global_params"]["end_date"] = end_date
-    my_dict["global_params"]["climo_nyears"] = climo_nyears
+    my_dict["global_params"]["base_case_name"] = base_case
     my_dict["global_params"]["base_case_output_dir"] = base_case_output_dir
     my_dict["global_params"]["base_end_date"] = base_end_date
-    my_dict["global_params"]["base_climo_nyears"] = base_climo_nyears
-    my_dict["timeseries"]["case_name"] = case
-    my_dict["timeseries"]["atm"]["end_years"] = [nyears, base_nyears]
+    my_dict["timeseries"]["case_name"] = [case, base_case]
+
+    for component in my_dict["timeseries"]:
+        if (
+            isinstance(my_dict["timeseries"][component], dict)
+            and "end_years" in my_dict["timeseries"][component]
+        ):
+            my_dict["timeseries"][component]["end_years"] = [nyears, base_nyears]
+    if "link_to_ADF" in my_dict["compute_notebooks"]["atm"]:
+        my_dict["compute_notebooks"]["atm"]["link_to_ADF"]["parameter_groups"]["none"][
+            "adf_root"
+        ] = os.path.join(case_root, "ADF_output")
+    if "Greenland_SMB_visual_compare_obs" in my_dict["compute_notebooks"]["glc"]:
+        my_dict["compute_notebooks"]["glc"]["Greenland_SMB_visual_compare_obs"][
+            "parameter_groups"
+        ]["none"]["climo_nyears"] = climo_nyears
+        my_dict["compute_notebooks"]["glc"]["Greenland_SMB_visual_compare_obs"][
+            "parameter_groups"
+        ]["none"]["base_climo_nyears"] = base_climo_nyears
 
     # replace with environment variable
-    my_dict["global_params"]["CESM_output_dir"] = dout_s_root
+    my_dict["global_params"]["CESM_output_dir"] = os.path.dirname(dout_s_root)
 
     # create new file, make it writeable
     with open("config.yml", "w") as f:

nblibrary/atm/Global_PSL_NMSE_compare_obs_lens.ipynb

@@ -204,7 +204,7 @@
     "        fix_time_dim(\n",
     "            xr.open_mfdataset(f\"{base_file_path}/*PSL*.nc\", decode_times=False)\n",
     "        )\n",
-    "        .sel(time=slice(start_date, end_date))\n",
+    "        .sel(time=slice(base_start_date, base_end_date))\n",
     "        .assign_coords({\"lon\": lon, \"lat\": lat})\n",
     "        .PSL\n",
     "        / 100.0\n",

nblibrary/atm/averaging_utils.py

@@ -36,7 +36,10 @@ def seasonal_climatology_weighted(dat):
     datw_am = dat * wgts_am
 
     ds_season = (
-        datw.rolling(min_periods=3, center=True, time=3).sum().dropna("time", how="all")
+        datw.load()
+        .rolling(min_periods=3, center=True, time=3)
+        .sum()
+        .dropna("time", how="all")
     )
     dat_djf = ds_season.where(ds_season.time.dt.month == 1, drop=True).mean("time")
     dat_mam = ds_season.where(ds_season.time.dt.month == 4, drop=True).mean("time")

nblibrary/glc/Greenland_SMB_visual_compare_obs.ipynb

@@ -36,6 +36,8 @@
    "outputs": [],
    "source": [
     "# Import packages\n",
+    "import os\n",
+    "\n",
     "import numpy as np\n",
     "import matplotlib.pyplot as plt\n",
     "import matplotlib.cm as mcm\n",
@@ -121,7 +123,7 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "cef60ddb-9ff4-4a14-a8ea-0d5740b6c18a",
+   "id": "08146fd8-d631-46c5-abc3-04fc1a004b77",
    "metadata": {
     "editable": true,
     "slideshow": {
@@ -135,18 +137,27 @@
    "source": [
     "last_year = int(end_date.split(\"-\")[0])\n",
     "\n",
-    "case_init_file = f\"{CESM_output_dir}/{case_name}/glc/hist/{case_name}.cism.gris.initial_hist.0001-01-01-00000.nc\"  # name of glc file output\n",
+    "case_init_file = os.path.join(\n",
+    "    obs_path, \"cism.gris.initial_hist.0001-01-01-00000.nc\"\n",
+    ")  # name of glc file output\n",
     "\n",
-    "case_path = f\"{CESM_output_dir}/{case_name}/cpl/hist\"  # path to glc output\n",
-    "case_file = f\"{case_path}/{case_name}.cpl.hx.1yr2glc.{last_year:04d}-01-01-00000.nc\"  # name of glc file output\n",
-    "obs_file = f\"{obs_path}/{obs_name}\"  # name of observed dataset file\n",
+    "case_path = os.path.join(\n",
+    "    CESM_output_dir, case_name, \"cpl\", \"hist\"\n",
+    ")  # path to glc output\n",
+    "case_file = os.path.join(\n",
+    "    case_path, f\"{case_name}.cpl.hx.1yr2glc.{last_year:04d}-01-01-00000.nc\"\n",
+    ")  # name of glc file output\n",
+    "obs_file = os.path.join(obs_path, obs_name)  # name of observed dataset file\n",
     "\n",
     "if base_case_name is not None:\n",
     "    base_last_year = int(base_end_date.split(\"-\")[0])\n",
-    "    base_case_path = (\n",
-    "        f\"{base_case_output_dir}/{base_case_name}/cpl/hist\"  # path to cpl output\n",
-    "    )\n",
-    "    base_file = f\"{base_case_path}/{base_case_name}.cpl.hx.1yr2glc.{base_last_year:04d}-01-01-00000.nc\"  # name of last cpl simulation output"
+    "    base_case_path = os.path.join(\n",
+    "        base_case_output_dir, base_case_name, \"cpl\", \"hist\"\n",
+    "    )  # path to cpl output\n",
+    "    base_file = os.path.join(\n",
+    "        base_case_path,\n",
+    "        f\"{base_case_name}.cpl.hx.1yr2glc.{base_last_year:04d}-01-01-00000.nc\",\n",
+    "    )  # name of last cpl simulation output"
    ]
   },
   {

nblibrary/lnd/Global_TerrestrialCouplingIndex_VisualCompareObs.ipynb

@@ -84,7 +84,16 @@
     "# obsDir   = '/glade/campaign/cgd/tss/people/mdfowler/FLUXNET2015/' ## Need to copy into CUPiD Data\n",
     "\n",
     "## Where CESM timeseries data is stored\n",
-    "CESM_output_dir = \"/glade/campaign/cesm/development/cross-wg/diagnostic_framework/CESM_output_for_testing/\"\n",
+    "CESM_output_dir = os.path.join(\n",
+    "    os.path.sep,\n",
+    "    \"glade\",\n",
+    "    \"campaign\",\n",
+    "    \"cesm\",\n",
+    "    \"development\",\n",
+    "    \"cross-wg\",\n",
+    "    \"diagnostic_framework\",\n",
+    "    \"CESM_output_for_testing\",\n",
+    ")\n",
     "\n",
     "\n",
     "## Full casenames that are present in CESM_output_dir and in individual filenames\n",
@@ -123,6 +132,14 @@
     "## - - - - - - - - - - - - - - - - - - - - - -\n",
     "##  Settings for computing coupling index\n",
     "## - - - - - - - - - - - - - - - - - - - - - -\n",
+    "\n",
+    "# Set up directory for aux output like coupling index file\n",
+    "if \"SCRATCH\" in os.environ:\n",
+    "    cupid_temp = os.path.join(os.path.sep, os.environ[\"SCRATCH\"], \"CUPiD_scratch\")\n",
+    "    os.makedirs(cupid_temp, exist_ok=True)\n",
+    "else:\n",
+    "    cupid_temp = \".\"\n",
+    "\n",
     "startYrs = [start_date.split(\"-\")[0]]\n",
     "endYrs = [f\"{int(end_date.split('-')[0])-1:04d}\"]\n",
     "\n",
@@ -200,10 +217,8 @@
    "source": [
     "for iCase in range(len(caseNames)):\n",
     "    ## Check first if coupling index has already been created:\n",
-    "    TCI_filePath = (\n",
-    "        \"/glade/derecho/scratch/mdfowler/\"\n",
-    "        + caseNames[0]\n",
-    "        + \"_TerrestrialCouplingIndex_SHvsSM.nc\"\n",
+    "    TCI_filePath = os.path.join(\n",
+    "        cupid_temp, f\"{caseNames[iCase]}_TerrestrialCouplingIndex_SHvsSM.nc\"\n",
     "    )\n",
     "\n",
     "    if os.path.exists(TCI_filePath):  # Use previously computed TCI\n",
@@ -280,22 +295,16 @@
     "        yDS = shflx_DS\n",
     "\n",
     "        couplingInd = compute_couplingIndex_cesm(xname, yname, xDS, yDS)\n",
-    "\n",
-    "        filePath = (\n",
-    "            \"/glade/derecho/scratch/mdfowler/\"\n",
-    "            + caseNames[0]\n",
-    "            + \"_TerrestrialCouplingIndex_SHvsSM.nc\"\n",
-    "        )\n",
-    "        couplingInd.to_netcdf(filePath)\n",
-    "        print(\"File created: \", filePath)"
+    "        couplingInd.to_netcdf(TCI_filePath)\n",
+    "        print(\"File created: \", TCI_filePath)"
    ]
   },
   {
    "cell_type": "markdown",
    "id": "5f8fba2a-98d2-4e94-9d71-3b2625e16032",
    "metadata": {},
    "source": [
-    "### 1.1 Read in FLUXNET data if requested"
+    "### 2.1 Read in FLUXNET data if requested"
    ]
   },
   {
@@ -659,7 +668,7 @@
    "id": "bc387253-cdd7-4a36-956b-8ce548e963bd",
    "metadata": {},
    "source": [
-    "## 2. Make plots"
+    "## 3. Make plots"
    ]
   },
   {
@@ -701,12 +710,11 @@
    "outputs": [],
    "source": [
     "for iCase in range(len(caseNames)):\n",
-    "    filePath = (\n",
-    "        \"/glade/derecho/scratch/mdfowler/\"\n",
-    "        + caseNames[iCase]\n",
-    "        + \"_TerrestrialCouplingIndex_SHvsSM.nc\"\n",
+    "    TCI_filePath = os.path.join(\n",
+    "        cupid_temp, f\"{caseNames[iCase]}_TerrestrialCouplingIndex_SHvsSM.nc\"\n",
     "    )\n",
-    "    couplingIndex_case = uxr.open_dataset(gridFile, filePath)\n",
+    "\n",
+    "    couplingIndex_case = uxr.open_dataset(gridFile, TCI_filePath)\n",
     "    # Rename the variable:\n",
     "    couplingIndex_case = couplingIndex_case.rename(\n",
     "        {\"__xarray_dataarray_variable__\": \"CouplingIndex\"}\n",