Add TEST_FILE_DIR option to CMake build, where test data files can be found instead of using FTP. (#732)

Add handling for the TEST_FILE_DIR option in CMake. This allows the programmer to 
specify a local directory to check first, before downloading test data files from the FTP site.

Use this capability, and a cache, to cache the data for all CI runs as well. This prevents
the CI system from downloading these test files hundreds of times per day.

Part of #719.
Fixes #728

Author: edwardhartnett

.github/workflows/debug-docs-test_coverage.yml

@@ -100,6 +100,13 @@ jobs:
         path: ufs_utils
         submodules: recursive
 
+    - name: cache-data
+      id: cache-data
+      uses: actions/cache@v2
+      with:
+        path: ~/data
+        key: data-1
+
     - name: build
       run: |
         export ESMFMKFILE=~/esmf/lib/esmf.mk
@@ -109,7 +116,7 @@ jobs:
         export CXX=mpicxx
         export FC=mpifort
         export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:~/jasper/lib;~/jasper/lib64"
-        cmake .. -DCMAKE_PREFIX_PATH='~/jasper;~/nceplibs' -DCMAKE_BUILD_TYPE=Debug -DENABLE_DOCS=On -DCMAKE_Fortran_FLAGS="-g -fprofile-arcs -ftest-coverage -O0"
+        cmake -DTEST_FILE_DIR=/home/runner/data -DCMAKE_PREFIX_PATH='~/jasper;~/nceplibs' -DCMAKE_BUILD_TYPE=Debug -DENABLE_DOCS=On  ..
         make -j2
 
     - name: test

.github/workflows/developer.yml

@@ -1,7 +1,7 @@
 # UFS_UTILS test workflow.
 #
-# This workflow tests all developer options - address santizer,
-# warning check, documentation check, and test code coverage.
+# This workflow tests all developer options including
+# documentation check, and test code coverage.
 #
 # Ed Hartnett 12/11/22
 name: developer 
@@ -326,15 +326,21 @@ jobs:
         path: ufs_utils
         submodules: true
 
+    - name: cache-data
+      id: cache-data
+      uses: actions/cache@v2
+      with:
+        path: ~/data
+        key: data-1
+
     - name:  build
       run: |
-        set -x
         cd ufs_utils
         mkdir build
         doxygen --version
         export ESMFMKFILE=~/esmf/lib/esmf.mk
         cd build
-        cmake .. -DFTP_TEST_FILES=ON -DTEST_FILE_DIR=/home/runner/data -DENABLE_DOCS=On -DJasper_ROOT=~/jasper -DCMAKE_PREFIX_PATH="~/g2c;~/bacio;~/g2;~/w3nco;~/sfcio;~/sigio;~/nemsio;~/sp;~/ip" -DCMAKE_Fortran_FLAGS="-g -fprofile-arcs -ftest-coverage -O0" -DCMAKE_C_FLAGS="-g -fprofile-arcs -ftest-coverage -O0"  -DCMAKE_BUILD_TYPE=Debug 
+        cmake -DTEST_FILE_DIR=/home/runner/data -DENABLE_DOCS=On -DCMAKE_PREFIX_PATH="~/jasper;~/g2c;~/bacio;~/g2;~/w3nco;~/sfcio;~/sigio;~/nemsio;~/sp;~/ip" -DCMAKE_Fortran_FLAGS="-g -fprofile-arcs -ftest-coverage -O0" -DCMAKE_C_FLAGS="-g -fprofile-arcs -ftest-coverage -O0"  -DCMAKE_BUILD_TYPE=Debug ..
         make -j2 VERBOSE=1
 
     - name: test
@@ -343,6 +349,18 @@ jobs:
         ctest --verbose --rerun-failed --output-on-failure
         gcovr --root .. -v  --html-details --exclude ../tests --exclude CMakeFiles --print-summary -o test-coverage.html &> /dev/null
 
+    - name: cache-data
+      if: steps.cache-data.outputs.cache-hit != 'true'
+      run: |
+        mkdir ~/data
+        cp ufs_utils/build/tests/chgres_cube/data/* ~/data
+        cp ufs_utils/build/tests/sfc_climo_gen/data/* ~/data
+        cp ufs_utils/build/tests/cpld_gridgen/data/* ~/data
+        cp ufs_utils/tests/filter_topo/data/* ~/data
+        cp ufs_utils/tests/emcsfc_snow2mdl/data/* ~/data
+        cp ufs_utils/tests/chgres_cube/data/* ~/data
+        ls -l ~/data
+        
     - name: upload-test-coverage
       uses: actions/upload-artifact@v2
       with:

.github/workflows/intel.yml

@@ -12,7 +12,7 @@ on:
     - README.md
 
 # Use custom shell with -l so .bash_profile is sourced which loads intel/oneapi/setvars.sh
-# without having to do it in manually every step
+# without having to do it in manually every step. 
 defaults:
   run:
     shell: bash -leo pipefail {0}
@@ -166,16 +166,36 @@ jobs:
         path: ufs_utils
         submodules: recursive
 
+    - name: cache-data
+      id: cache-data
+      uses: actions/cache@v2
+      with:
+        path: ~/data
+        key: data-1
+
     - name: build
       run: |
         export ESMFMKFILE=~/esmf/lib/esmf.mk
         cd ufs_utils
         mkdir build && cd build
         export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:~/jasper/lib;~/jasper/lib64"
-        cmake .. -DCMAKE_BUILD_TYPE=Debug -DCMAKE_PREFIX_PATH='~;~/jasper;~/nceplibs;~/netcdf'
+        cmake -DTEST_FILE_DIR=/home/runner/data -DCMAKE_BUILD_TYPE=Debug -DCMAKE_PREFIX_PATH='~;~/jasper;~/nceplibs;~/netcdf' ..
         make -j2
     
     - name: test
       run: |
         cd ufs_utils/build
         ctest --rerun-failed --output-on-failure
+
+    - name: cache-data
+      if: steps.cache-data.outputs.cache-hit != 'true'
+      run: |
+        mkdir ~/data
+        cp ufs_utils/build/tests/chgres_cube/data/* ~/data
+        cp ufs_utils/build/tests/sfc_climo_gen/data/* ~/data
+        cp ufs_utils/build/tests/cpld_gridgen/data/* ~/data
+        cp ufs_utils/tests/filter_topo/data/* ~/data
+        cp ufs_utils/tests/emcsfc_snow2mdl/data/* ~/data
+        cp ufs_utils/tests/chgres_cube/data/* ~/data
+        ls -l ~/data
+        

.github/workflows/linux-mac-nceplibs-mpi.yml

@@ -217,6 +217,13 @@ jobs:
         path: ufs_utils
         submodules: recursive
 
+    - name: cache-data
+      id: cache-data
+      uses: actions/cache@v2
+      with:
+        path: ~/data
+        key: data-2
+
     - name: build
       run: |
         export ESMFMKFILE=~/esmf/lib/esmf.mk
@@ -227,7 +234,7 @@ jobs:
         export FC=mpifort
         export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:~/jasper/lib;~/jasper/lib64"
         export DYLD_LIBRARY_PATH="${LD_LIBRARY_PATH}:~/jasper/lib;~/jasper/lib64"
-        cmake .. -DCMAKE_PREFIX_PATH='~/jasper;~/nceplibs;~/netcdf'
+        cmake -DTEST_FILE_DIR=/home/runner/data -DCMAKE_PREFIX_PATH='~/jasper;~/nceplibs;~/netcdf' ..
         make -j2
 
     - name: test
@@ -236,3 +243,16 @@ jobs:
         # Oversubscribe for OpenMPI to run more processes than CPUs
         export OMPI_MCA_rmaps_base_oversubscribe=1
         ctest --rerun-failed --output-on-failure
+
+    - name: cache-data
+      if: steps.cache-data.outputs.cache-hit != 'true'
+      run: |
+        mkdir ~/data
+        cp ufs_utils/build/tests/chgres_cube/data/* ~/data
+        cp ufs_utils/build/tests/sfc_climo_gen/data/* ~/data
+        cp ufs_utils/build/tests/cpld_gridgen/data/* ~/data
+        cp ufs_utils/tests/filter_topo/data/* ~/data
+        cp ufs_utils/tests/emcsfc_snow2mdl/data/* ~/data
+        cp ufs_utils/tests/chgres_cube/data/* ~/data
+        ls -l ~/data
+        

.github/workflows/netcdf-versions.yml

@@ -151,6 +151,13 @@ jobs:
         path: ufs_utils
         submodules: recursive
 
+    - name: cache-data
+      id: cache-data
+      uses: actions/cache@v2
+      with:
+        path: ~/data
+        key: data-1
+
     - name: build
       run: |
         export ESMFMKFILE=~/esmf/lib/esmf.mk
@@ -160,11 +167,25 @@ jobs:
         export CXX=mpicxx
         export FC=mpifort
         export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:~/jasper/lib;~/jasper/lib64"
-        cmake .. -DCMAKE_PREFIX_PATH='~/jasper;~/nceplibs;~/netcdf' -DCMAKE_BUILD_TYPE=Debug 
+        cmake -DTEST_FILE_DIR=/home/runner/work/UFS_UTILS/UFS_UTILS/data -DCMAKE_PREFIX_PATH='~/jasper;~/nceplibs;~/netcdf' -DCMAKE_BUILD_TYPE=Debug ..
         make -j2
 
     - name: test
       run: |
         cd ufs_utils/build
 #        export LSAN_OPTIONS=suppressions=LSanSuppress.supp
         ctest --rerun-failed --output-on-failure
+
+    - name: cache-data
+      if: steps.cache-data.outputs.cache-hit != 'true'
+      run: |
+        mkdir ~/data
+        cp ufs_utils/build/tests/chgres_cube/data/* ~/data
+        cp ufs_utils/build/tests/sfc_climo_gen/data/* ~/data
+        cp ufs_utils/build/tests/cpld_gridgen/data/* ~/data
+        cp ufs_utils/tests/filter_topo/data/* ~/data
+        cp ufs_utils/tests/emcsfc_snow2mdl/data/* ~/data
+        cp ufs_utils/tests/chgres_cube/data/* ~/data
+        ls -l ~/data
+        
+

CMakeLists.txt

@@ -15,7 +15,9 @@ list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
 # User options.
 option(OPENMP "use OpenMP threading" ON)
 option(ENABLE_DOCS "Enable generation of doxygen-based documentation." OFF)
+SET(TEST_FILE_DIR "." CACHE STRING "Check this directory for test files before using FTP.")
 
+# Set the build type.
 if(NOT CMAKE_BUILD_TYPE MATCHES "^(Debug|Release|RelWithDebInfo|MinSizeRel)$")
   message(STATUS "Setting build type to 'Release' as none was specified.")
   set(CMAKE_BUILD_TYPE

tests/CMakeLists.txt

@@ -3,19 +3,29 @@
 #
 # Ed Hartnett 2/11/21
 
-# This function is used to download unit test data.
-# It takes two arguments, the URL and the file to
-# be downloaded.
-
+# Some test files are large and are kept on the NOAA EMC FTP
+# site. This function is used to download such test data. It takes two
+# arguments, the URL and the file to be downloaded.
 function(PULL_DATA THE_URL THE_FILE)
+  # If the TEST_FILE_DIR was specified, look for our test data files
+  # there before FTPing them. Developers can keep all test files on
+  # their machines, and save the time of downloading them every time.
+  message(STATUS "TEST_FILE_DIR ${TEST_FILE_DIR} THE_FILE ${THE_FILE}")  
+  if(NOT ${TEST_FILE_DIR} STREQUAL ".")
+    if (EXISTS ${TEST_FILE_DIR}/${THE_FILE})
+      message(STATUS "Copying file ${TEST_FILE_DIR}/${THE_FILE} to test data directory.")
+      FILE(COPY ${TEST_FILE_DIR}/${THE_FILE}
+        DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/data)
+    endif()
+  endif()
   if(NOT EXISTS "${CMAKE_CURRENT_BINARY_DIR}/data/${THE_FILE}")
+    message(STATUS "Downloading file ${CMAKE_CURRENT_BINARY_DIR}/data/${THE_FILE}.")
     file(DOWNLOAD
       ${THE_URL}/${THE_FILE}
       ${CMAKE_CURRENT_BINARY_DIR}/data/${THE_FILE}
-      SHOW_PROGRESS
       STATUS status
-      INACTIVITY_TIMEOUT 120
-      )
+      INACTIVITY_TIMEOUT 30
+    )
     list(GET status 0 status_num)
     if(NOT status_num EQUAL 0 OR NOT EXISTS "${CMAKE_CURRENT_BINARY_DIR}/data/${THE_FILE}")
       message(FATAL_ERROR "Could not download ${THE_FILE}")