Skip to content

Commit 6577584

Browse files
AndrewEichmann-NOAAguillaumevernieres
AndrewEichmann-NOAA
and
guillaumevernieres
authored
Remove tests with R2D2 and anything referencing it (#836)
Removes soca ctests related to R2D2, associated data, and any lines in scripts, yamls, or other config files referencing it. SOCA ctests now run on obs prepped by the prep obs task. Almost takes care of #815 as references are still present in global-workflow task config files. `logging.info` has been replaced by `print` because logging hasn't been working --------- Co-authored-by: Guillaume Vernieres <guillaume.vernieres@noaa.gov>
1 parent 69b1eb4 commit 6577584

31 files changed

+9
-2108
lines changed

‎.github/workflows/unittests.yaml

-9
Original file line numberDiff line numberDiff line change
@@ -15,15 +15,6 @@ jobs:
1515
pip install netCDF4
1616
pip install xarray
1717
18-
- name: Install other dependencies
19-
run: |
20-
wget https://ftp.emc.ncep.noaa.gov/static_files/public/GDASApp/r2d2-bb361c2.tar.gz
21-
wget https://ftp.emc.ncep.noaa.gov/static_files/public/GDASApp/solo-877b414.tar.gz
22-
tar -xvf r2d2-bb361c2.tar.gz
23-
tar -xvf solo-877b414.tar.gz
24-
cd solo && pip install . && cd ..
25-
cd r2d2 && pip install . && cd ..
26-
2718
- name: Checkout wxflow
2819
uses: actions/checkout@v3
2920
with:

‎modulefiles/GDAS/hera.lua

-1
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,6 @@ local mpinproc = '-n'
7979
setenv('MPIEXEC_EXEC', mpiexec)
8080
setenv('MPIEXEC_NPROC', mpinproc)
8181

82-
setenv('R2D2_CONFIG', '/scratch1/NCEPDEV/stmp4/Cory.R.Martin/R2D2_SHARED/config_hera.yaml')
8382
setenv("CRTM_FIX","/scratch1/NCEPDEV/da/Cory.R.Martin/GDASApp/fix/crtm/2.4.0")
8483
setenv("GDASAPP_TESTDATA","/scratch1/NCEPDEV/da/Cory.R.Martin/CI/GDASApp/data")
8584
prepend_path("PATH","/scratch2/NCEPDEV/nwprod/hpc-stack/libs/hpc-stack/intel-18.0.5.274/prod_util/1.2.2/bin")

‎modulefiles/GDAS/orion.lua

-1
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,6 @@ local mpinproc = '-n'
7878
setenv('MPIEXEC_EXEC', mpiexec)
7979
setenv('MPIEXEC_NPROC', mpinproc)
8080

81-
setenv('R2D2_CONFIG', '/work2/noaa/da/cmartin/GDASApp/R2D2_SHARED/config_orion.yaml')
8281
setenv("CRTM_FIX","/work2/noaa/da/cmartin/GDASApp/fix/crtm/2.4.0")
8382
setenv("GDASAPP_TESTDATA","/work2/noaa/da/cmartin/CI/GDASApp/data")
8483
prepend_path("PATH","/apps/contrib/NCEP/libs/hpc-stack/intel-2018.4/prod_util/1.2.2/bin")

‎parm/soca/obs/obs_list_small.yaml

+1
Original file line numberDiff line numberDiff line change
@@ -4,5 +4,6 @@ observers:
44
- !INC ${OBS_YAML_DIR}/adt_j2.yaml
55
#- !INC ${OBS_YAML_DIR}/salt_profile_fnmoc.yaml
66
- !INC ${OBS_YAML_DIR}/sss_smap.yaml
7+
- !INC ${OBS_YAML_DIR}/sst_metopb_l3u_so025.yaml
78
- !INC ${OBS_YAML_DIR}/sst_noaa19_l3u.yaml
89
- !INC ${OBS_YAML_DIR}/icec_emc.yaml

‎parm/soca/obsproc/obs_list_obsproc.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ observers:
55
#- !INC ${OBS_YAML_DIR}/salt_profile_fnmoc.yaml
66
- !INC ${OBS_YAML_DIR}/sss_smap.yaml
77
- !INC ${OBS_YAML_DIR}/sss_smos.yaml
8-
- !INC ${OBS_YAML_DIR}/sst_noaa19_l3u.yaml
8+
#- !INC ${OBS_YAML_DIR}/sst_noaa19_l3u.yaml
99
- !INC ${OBS_YAML_DIR}/sst_metopb_l3u_so025.yaml
1010
- !INC ${OBS_YAML_DIR}/icec_emc.yaml
1111
- !INC ${OBS_YAML_DIR}/adt_3b_egm2008.yaml

‎parm/templates/stage.yaml

-7
Original file line numberDiff line numberDiff line change
@@ -7,13 +7,6 @@ COMOUT: !ENV ${COMOUT}
77
DATA: !ENV ${DATA}
88
background_time: '{{background_time}}'
99
dump: !ENV ${CDUMP}
10-
r2d2_obs_db: !ENV ${R2D2_OBS_DB}
11-
r2d2_obs_dump: !ENV ${R2D2_OBS_DUMP}
12-
r2d2_obs_src: !ENV ${R2D2_OBS_SRC}
13-
r2d2_bc_src: !ENV ${R2D2_BC_SRC}
14-
r2d2_bc_dump: !ENV ${R2D2_BC_DUMP}
15-
r2d2 window length: !ENV ${R2D2_OBS_WINDOW}
16-
r2d2 window begin: '{{r2d2_window_begin}}'
1710
fv3jedi_stage_files: !ENV ${FV3JEDI_STAGE_YAML}
1811
fv3jedi_stage: !ENV ${FV3JEDI_STAGE_YAML}
1912
stage_dir: !ENV ${DATA}

‎scripts/exgdas_global_marine_analysis_prep.py

+2-11
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@
4545

4646
# import UFSDA utilities
4747
import ufsda
48-
from ufsda.stage import obs, soca_fix
48+
from ufsda.stage import soca_fix
4949

5050

5151
def agg_seaice(fname_in, fname_out):
@@ -259,24 +259,14 @@ def find_clim_ens(input_date):
259259

260260
logging.info(f"---------------- Stage observations")
261261

262-
# setup the archive, local and shared R2D2 databases
263-
ufsda.r2d2.setup(r2d2_config_yaml=os.path.join(anl_dir, 'r2d2_config.yaml'), shared_root=comin_obs)
264-
265262
# create config dict from runtime env
266263
envconfig = {'window_begin': f"{window_begin.strftime('%Y-%m-%dT%H:%M:%SZ')}",
267-
'r2d2_obs_src': os.getenv('R2D2_OBS_SRC'),
268-
'r2d2_obs_dump': os.getenv('R2D2_OBS_DUMP'),
269-
'r2d2_obs_db': os.getenv('R2D2_OBS_DB'),
270264
'ATM_WINDOW_BEGIN': window_begin_iso,
271265
'ATM_WINDOW_MIDDLE': window_middle_iso,
272266
'ATM_WINDOW_LENGTH': f"PT{os.getenv('assim_freq')}H"}
273267
stage_cfg = YAMLFile(path=os.path.join(gdas_home, 'parm', 'templates', 'stage.yaml'))
274268
stage_cfg = Template.substitute_structure(stage_cfg, TemplateConstants.DOUBLE_CURLY_BRACES, envconfig.get)
275269
stage_cfg = Template.substitute_structure(stage_cfg, TemplateConstants.DOLLAR_PARENTHESES, envconfig.get)
276-
stage_cfg['r2d2_obs_out'] = os.getenv('COM_OBS')
277-
278-
# stage observations from R2D2 COMIN_OBS to COM_OBS
279-
ufsda.stage.obs(stage_cfg)
280270

281271
# concatenate altimeters into one obs space
282272
# TODO (SAMG)temporary, move this into the obs procecing eventually
@@ -294,6 +284,7 @@ def find_clim_ens(input_date):
294284
logging.info(f"******* {obs_file}")
295285
obs_src = os.path.join(os.getenv('COM_OBS'), obs_file)
296286
obs_dst = os.path.join(os.path.abspath(obs_in), obs_file)
287+
logging.info(f"******* {obs_src}")
297288
if os.path.exists(obs_src):
298289
logging.info(f"******* fetching {obs_file}")
299290
obs_list.append([obs_src, obs_dst])

‎scripts/exglobal_prep_ocean_obs.py

+3
Original file line numberDiff line numberDiff line change
@@ -85,4 +85,7 @@
8585
print("CRITICAL: Ill-formed OBS_YAML file, exiting")
8686
raise
8787

88+
if not os.path.exists(COMOUT_OBS):
89+
os.makedirs(COMOUT_OBS)
90+
8891
FileHandler({'copy': filesToSave}).sync()

‎test/atm/CMakeLists.txt

-10
Original file line numberDiff line numberDiff line change
@@ -15,16 +15,6 @@ if (BUILD_GDASBUNDLE)
1515
COMMAND ${PROJECT_SOURCE_DIR}/test/atm/test_convert_gsi_satbias.sh ${PROJECT_SOURCE_DIR} ${PROJECT_BINARY_DIR}
1616
WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/)
1717

18-
# test saving the output of the previous test to an R2D2 database
19-
add_test(NAME test_gdasapp_store_gsi_satbias
20-
COMMAND ${PROJECT_SOURCE_DIR}/test/atm/test_store_gsi_satbias.py
21-
WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/)
22-
set_tests_properties(test_gdasapp_store_gsi_satbias
23-
PROPERTIES
24-
DEPENDS "test_gdasapp_convert_gsi_satbias"
25-
ENVIRONMENT "PYTHONPATH=${PROJECT_BINARY_DIR}/ush:$ENV{PYTHONPATH}"
26-
)
27-
2818
# Test exgdas scripts from the global-worflow
2919
if (WORKFLOW_TESTS)
3020
add_subdirectory(global-workflow)

‎test/atm/test_store_gsi_satbias.py

-74
This file was deleted.

‎test/soca/CMakeLists.txt

-11
Original file line numberDiff line numberDiff line change
@@ -60,17 +60,6 @@ endforeach(FILENAME)
6060
install(FILES ${test_input}
6161
DESTINATION "test/testinput/")
6262

63-
# Create a test R2D2 database for obs
64-
# -----------------------------------
65-
file(MAKE_DIRECTORY ${PROJECT_BINARY_DIR}/test/soca/obs)
66-
set( OBS_DIR ${PROJECT_BINARY_DIR}/test/testdata )
67-
add_test(NAME test_gdasapp_soca_obsdb
68-
COMMAND ${PROJECT_SOURCE_DIR}/test/soca/create_obsdb.py
69-
WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/soca/obs/)
70-
set_tests_properties(test_gdasapp_soca_obsdb
71-
PROPERTIES
72-
ENVIRONMENT "PYTHONPATH=${PROJECT_BINARY_DIR}/ush:${PROJECT_SOURCE_DIR}/../../ush/python/wxflow/src:$ENV{PYTHONPATH};SOCA_TEST_OBS=${PROJECT_SOURCE_DIR}/test/soca/testdata")
73-
7463
# Test exgdas scripts from the global-worflow
7564
if (WORKFLOW_TESTS)
7665
add_subdirectory(gw)

‎test/soca/create_obsdb.py

-82
This file was deleted.

‎test/soca/gw/CMakeLists.txt

+2-11
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
# test for creating an experiment directory within the global-workflow
22
file(MAKE_DIRECTORY ${PROJECT_BINARY_DIR}/test/soca/gw/testrun)
3-
file(MAKE_DIRECTORY ${PROJECT_BINARY_DIR}/test/soca/gw/concatioda)
43

54
# Prepare COMROT and static dir
65
add_test(NAME test_gdasapp_soca_prep
@@ -21,14 +20,6 @@ IF (IS_DIRECTORY /scratch2/NCEPDEV/)
2120
set(PARTITION "hera")
2221
ENDIF()
2322

24-
# Test ioda-concatenation
25-
add_test(NAME test_gdasapp_soca_concatioda
26-
COMMAND ${Python3_EXECUTABLE} ${PROJECT_SOURCE_DIR}/test/soca/gw/test_concatioda.py ${PROJECT_BINARY_DIR}/test/soca/obs
27-
WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/soca/gw/concatioda)
28-
set_tests_properties(test_gdasapp_soca_concatioda
29-
PROPERTIES
30-
ENVIRONMENT "PYTHONPATH=${PROJECT_BINARY_DIR}/ush:${PROJECT_BINARY_DIR}/lib/python${Python3_VERSION_MAJOR}.${Python3_VERSION_MINOR}:$ENV{PYTHONPATH}")
31-
3223
# Clean-up
3324
add_test(NAME test_gdasapp_soca_run_clean
3425
COMMAND ${CMAKE_COMMAND} -E remove_directory ${PROJECT_BINARY_DIR}/test/soca/gw/testrun/testjjobs)
@@ -45,8 +36,8 @@ add_test(NAME test_gdasapp_soca_setup_obsproc
4536
WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/soca/gw/obsproc)
4637

4738
# Test JGDAS_GLOBAL_OCEAN_ANALYSIS_*
48-
set(jjob_list "JGDAS_GLOBAL_OCEAN_ANALYSIS_PREP"
49-
"JGLOBAL_PREP_OCEAN_OBS"
39+
set(jjob_list "JGLOBAL_PREP_OCEAN_OBS"
40+
"JGDAS_GLOBAL_OCEAN_ANALYSIS_PREP"
5041
"JGDAS_GLOBAL_OCEAN_ANALYSIS_BMAT"
5142
"JGDAS_GLOBAL_OCEAN_ANALYSIS_RUN"
5243
"JGDAS_GLOBAL_OCEAN_ANALYSIS_CHKPT"

‎test/soca/gw/run_jjobs.yaml.test

-3
Original file line numberDiff line numberDiff line change
@@ -49,10 +49,7 @@ setup_expt config:
4949
SOCA_INPUT_FIX_DIR: @HOMEgfs@/sorc/gdas.cd/build/soca_static
5050
CASE_ANL: C48
5151
SOCA_OBS_LIST: @HOMEgfs@/sorc/gdas.cd/parm/soca/obs/obs_list_small.yaml
52-
COMIN_OBS: @HOMEgfs@/sorc/gdas.cd/build/test/soca/obs/r2d2-shared
5352
SOCA_NINNER: 1
54-
R2D2_OBS_SRC: gdasapp
55-
R2D2_OBS_DUMP: soca
5653
SABER_BLOCKS_YAML: @HOMEgfs@/sorc/gdas.cd/parm/soca/berror/saber_blocks.yaml
5754
NICAS_RESOL: 1
5855
NICAS_GRID_SIZE: 150

‎test/soca/gw/test_concatioda.py

-29
This file was deleted.

0 commit comments

Comments
 (0)