ufs-community · DusanJovic-NOAA · Jul 28, 2020 · May 13, 2020 · May 13, 2020 · May 20, 2020
diff --git a/CCPP.appBuilder b/CCPP.appBuilder
diff --git a/CCPP_repro.appBuilder b/CCPP_repro.appBuilder
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -1,9 +1,7 @@
 cmake_minimum_required(VERSION 3.15)
 
-foreach(env_var IN ITEMS
-         CMAKE_C_COMPILER CMAKE_CXX_COMPILER CMAKE_Fortran_COMPILER
-         CMAKE_Platform)
-  if ( NOT DEFINED ENV{${env_var}})
+foreach(env_var IN ITEMS CMAKE_C_COMPILER CMAKE_CXX_COMPILER CMAKE_Fortran_COMPILER)
+  if(NOT DEFINED ENV{${env_var}})
     message(FATAL_ERROR "${env_var} is not defined")
   endif()
 endforeach()
@@ -18,10 +16,68 @@ project(ufs-weather-model
         LANGUAGES C CXX Fortran)
 
 list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/cmake)
-find_package(MPI REQUIRED)
-find_package(ESMF REQUIRED)
 
-include(${PROJECT_SOURCE_DIR}/cmake/configure_${CMAKE_Platform}.cmake)
+set(32BIT           OFF CACHE BOOL "Enable 32BIT (single precision arithmetic in dycore)")
+set(AVX2            ON  CACHE BOOL "Enable AVX2 instruction set")
+set(CCPP            ON  CACHE BOOL "Enable CCPP")
+set(DEBUG           OFF CACHE BOOL "Enable DEBUG mode")
+set(INLINE_POST     OFF CACHE BOOL "Enable inline post")
+set(MULTI_GASES     OFF CACHE BOOL "Enable MULTI_GASES")
+set(OPENMP          ON  CACHE BOOL "Enable OpenMP threading")
+set(PARALLEL_NETCDF OFF CACHE BOOL "Enable parallel NetCDF")
+set(QUAD_PRECISION  ON  CACHE BOOL "Enable QUAD_PRECISION (for certain grid metric terms in dycore)")
+set(REPRO           OFF CACHE BOOL "Enable REPRO mode")
+set(WW3             OFF CACHE BOOL "Enable WW3")
+
+if(CMAKE_Platform)
+  message("")
+  if(EXISTS ${PROJECT_SOURCE_DIR}/cmake/configure_${CMAKE_Platform}.cmake)
+    message("Setting configuration for ${CMAKE_Platform}")
+    include(${PROJECT_SOURCE_DIR}/cmake/configure_${CMAKE_Platform}.cmake)
+  else()
+    message("Platform '${CMAKE_Platform}' configuration file does not exist")
+  endif()
+endif()
+
+message("")
+message("32BIT ............ ${32BIT}")
+message("AVX2 ............. ${AVX2}")
+message("CCPP ............. ${CCPP}")
+message("DEBUG ............ ${DEBUG}")
+message("INLINE_POST ...... ${INLINE_POST}")
+message("MULTI_GASES ...... ${MULTI_GASES}")
+message("OPENMP ........... ${OPENMP}")
+message("PARALLEL_NETCDF .. ${PARALLEL_NETCDF}")
+message("QUAD_PRECISION ... ${QUAD_PRECISION}")
+message("REPRO ............ ${REPRO}")
+message("WW3 .............. ${WW3}")
+message("")
+
+get_filename_component (C_COMPILER_NAME ${CMAKE_C_COMPILER} NAME)
+get_filename_component (CXX_COMPILER_NAME ${CMAKE_CXX_COMPILER} NAME)
+get_filename_component (Fortran_COMPILER_NAME ${CMAKE_Fortran_COMPILER} NAME)
+message("C       compiler: ${CMAKE_C_COMPILER_ID} ${CMAKE_C_COMPILER_VERSION} (${C_COMPILER_NAME})")
+message("CXX     compiler: ${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION} (${CXX_COMPILER_NAME})")
+message("Fortran compiler: ${CMAKE_Fortran_COMPILER_ID} ${CMAKE_Fortran_COMPILER_VERSION} (${Fortran_COMPILER_NAME})")
+message("")
+
+include(cmake/${CMAKE_Fortran_COMPILER_ID}.cmake)
+message("")
+
+find_package(NetCDF REQUIRED C Fortran)
+find_package(MPI REQUIRED)
+find_package(ESMF MODULE REQUIRED)
+
+find_package(bacio REQUIRED)
+find_package(nemsio REQUIRED)
+find_package(sp REQUIRED)
+find_package(w3emc REQUIRED)
+find_package(w3nco REQUIRED)
+if(INLINE_POST)
+  find_package(nceppost REQUIRED)
+endif()
+# See https://github.com/NOAA-EMC/NCEPLIBS-nemsio/pull/22
+target_link_libraries(nemsio::nemsio INTERFACE w3emc::w3emc_d bacio::bacio_4)
 
 add_definitions(-Duse_libMPI)
 add_definitions(-Duse_netCDF)
@@ -36,7 +92,7 @@ add_definitions(-DUSE_COND)
 add_definitions(-DNEW_TAUCTMAX)
 add_definitions(-DINTERNAL_FILE_NML)
 
-if (CMAKE_Fortran_COMPILER_ID MATCHES "GNU" AND CMAKE_Fortran_COMPILER_VERSION VERSION_LESS 9.0.0)        
+if (CMAKE_Fortran_COMPILER_ID MATCHES "GNU" AND CMAKE_Fortran_COMPILER_VERSION VERSION_LESS 9.0.0)
   message(FATAL_ERROR "GNU Compiler >= 9 is required")
 endif()
 
@@ -95,103 +151,16 @@ endif()
 ###############################################################################
 ### FMS
 ###############################################################################
-add_library(
-    fms
-
-    FMS/amip_interp/amip_interp.F90
-    FMS/astronomy/astronomy.F90
-    FMS/axis_utils/axis_utils.F90
-    FMS/axis_utils/axis_utils2.F90
-    FMS/block_control/block_control.F90
-    FMS/column_diagnostics/column_diagnostics.F90
-    FMS/constants/constants.F90
-    FMS/coupler/atmos_ocean_fluxes.F90
-    FMS/coupler/coupler_types.F90
-    FMS/coupler/ensemble_manager.F90
-    FMS/data_override/data_override.F90
-    FMS/diag_integral/diag_integral.F90
-    FMS/diag_manager/diag_axis.F90
-    FMS/diag_manager/diag_data.F90
-    FMS/diag_manager/diag_grid.F90
-    FMS/diag_manager/diag_manager.F90
-    FMS/diag_manager/diag_output.F90
-    FMS/diag_manager/diag_table.F90
-    FMS/diag_manager/diag_util.F90
-    FMS/drifters/cloud_interpolator.F90
-    FMS/drifters/drifters.F90
-    FMS/drifters/drifters_comm.F90
-    FMS/drifters/drifters_core.F90
-    FMS/drifters/drifters_input.F90
-    FMS/drifters/drifters_io.F90
-    FMS/drifters/quicksort.F90
-    FMS/exchange/stock_constants.F90
-    FMS/exchange/xgrid.F90
-    FMS/field_manager/field_manager.F90
-    FMS/field_manager/fm_util.F90
-    FMS/fms/fms.F90
-    FMS/fms/fms_io.F90
-    FMS/fms2_io/blackboxio.F90
-    FMS/fms2_io/fms2_io.F90
-    FMS/fms2_io/fms_io_utils.F90
-    FMS/fms2_io/fms_netcdf_domain_io.F90
-    FMS/fms2_io/fms_netcdf_unstructured_domain_io.F90
-    FMS/fms2_io/netcdf_io.F90
-    FMS/horiz_interp/horiz_interp.F90
-    FMS/horiz_interp/horiz_interp_bicubic.F90
-    FMS/horiz_interp/horiz_interp_bilinear.F90
-    FMS/horiz_interp/horiz_interp_conserve.F90
-    FMS/horiz_interp/horiz_interp_spherical.F90
-    FMS/horiz_interp/horiz_interp_type.F90
-    FMS/interpolator/interpolator.F90
-    FMS/memutils/memutils.F90
-    FMS/monin_obukhov/monin_obukhov.F90
-    FMS/monin_obukhov/monin_obukhov_kernel.F90
-    FMS/mosaic/gradient.F90
-    FMS/mosaic/grid.F90
-    FMS/mosaic/mosaic.F90
-    FMS/mosaic/mosaic2.F90
-    FMS/mpp/mpp.F90
-    FMS/mpp/mpp_data.F90
-    FMS/mpp/mpp_domains.F90
-    FMS/mpp/mpp_efp.F90
-    FMS/mpp/mpp_io.F90
-    FMS/mpp/mpp_memutils.F90
-    FMS/mpp/mpp_parameter.F90
-    FMS/mpp/mpp_pset.F90
-    FMS/mpp/mpp_utilities.F90
-    FMS/platform/platform.F90
-    FMS/random_numbers/MersenneTwister.F90
-    FMS/random_numbers/random_numbers.F90
-    FMS/sat_vapor_pres/sat_vapor_pres.F90
-    FMS/sat_vapor_pres/sat_vapor_pres_k.F90
-    FMS/time_interp/time_interp.F90
-    FMS/time_interp/time_interp_external.F90
-    FMS/time_interp/time_interp_external2.F90
-    FMS/time_manager/get_cal_time.F90
-    FMS/time_manager/time_manager.F90
-    FMS/topography/gaussian_topog.F90
-    FMS/topography/topography.F90
-    FMS/tracer_manager/tracer_manager.F90
-    FMS/tridiagonal/tridiagonal.F90
-
-    FMS/mosaic/create_xgrid.c
-    FMS/mosaic/gradient_c2l.c
-    FMS/mosaic/interp.c
-    FMS/mosaic/mosaic_util.c
-    FMS/mosaic/read_mosaic.c
-    FMS/mpp/mpp_memuse.c
-    FMS/mpp/nsclock.c
-    FMS/mpp/threadloc.c
-)
-# stupid cmake can not figure out dependency of fft.F90 on fft99.F90 because 'use fft99_mod' is inside ifdefs
-set_property(SOURCE FMS/fft/fft.F90 APPEND_STRING PROPERTY COMPILE_FLAGS "-DSGICRAY=0 -DNAGFFT=0")
-
-target_include_directories(fms PUBLIC ${PROJECT_SOURCE_DIR}/FMS/include FMS/include FMS/fms FMS/fms2_io/include FMS/mpp/include)
-target_include_directories(fms PRIVATE ${NETCDF_INC_DIR})
-
+include(fms_files.cmake)
+add_library(fms ${fms_src_files})
+target_include_directories(fms PUBLIC $<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/FMS/include>
+                                      $<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/FMS/fms>
+                                      $<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/FMS/fms2_io/include>
+                                      $<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/FMS/mpp/include>)
+target_link_libraries(fms PRIVATE NetCDF::NetCDF_Fortran)
 set_target_properties(fms PROPERTIES ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/FMS)
 set_target_properties(fms PROPERTIES Fortran_MODULE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/FMS/mod)
-target_include_directories(fms PUBLIC ${CMAKE_CURRENT_BINARY_DIR}/FMS/mod)
+target_include_directories(fms PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/FMS/mod>)
 
 ###############################################################################
 ### FV3
@@ -210,34 +179,24 @@ if(WW3)
     set(WW3_COMP "orion")
   endif()
   message("Build WW3:")
-  message("   run: ${CMAKE_BUILD_TOOL} WW3_PARCOMPN=4 WW3_COMP=${WW3_COMP} ww3_nems")
+  message("   run: ${CMAKE_BUILD_TOOL} WW3_PARCOMPN=4 WW3_COMP=${WW3_COMP} ww3_nemslibonly")
   message("   in:  ${PROJECT_SOURCE_DIR}/WW3/model/esmf")
   message("")
 
   add_custom_target(ww3_nems
     WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/WW3/model/esmf
-    COMMAND ${CMAKE_BUILD_TOOL} WW3_PARCOMPN=4 WW3_COMP=${WW3_COMP} ww3_nems > make.log 2>&1
+    COMMAND ${CMAKE_BUILD_TOOL} WW3_PARCOMPN=4 WW3_COMP=${WW3_COMP} ww3_nemslibonly > ${PROJECT_BINARY_DIR}/ww3_make.log 2>&1
   )
+
+    set(WW3_LIBS ${PROJECT_SOURCE_DIR}/WW3/model/obj/libww3_multi_esmf.a)
 endif()
 
 ###############################################################################
-### NEMS
+### NEMS (ufs-weather-model-lib)
 ###############################################################################
-configure_file(${PROJECT_SOURCE_DIR}/NEMS/src/ESMFVersionDefine_ESMF_NUOPC.h
-               ${PROJECT_BINARY_DIR}/NEMS/src/ESMFVersionDefine.h
-               COPYONLY)
-# until we remove "../ESMFVersionDefine.h" from ENS_Cpl/ENS_CplComp_ESMFMod_STUB.F90
-configure_file(${PROJECT_SOURCE_DIR}/NEMS/src/ESMFVersionDefine_ESMF_NUOPC.h
-               ${PROJECT_BINARY_DIR}/NEMS/ESMFVersionDefine.h
-               COPYONLY)
-configure_file(${PROJECT_SOURCE_DIR}/NEMS/src/ESMFConvenienceMacros.h
-               ${PROJECT_BINARY_DIR}/NEMS/src/ESMFConvenienceMacros.h
-               COPYONLY)
-
-add_executable(
-    NEMS.exe
+add_library(
+    ufs-weather-model-lib
 
-    NEMS/src/MAIN_NEMS.F90
     NEMS/src/module_NEMS_UTILS.F90
     NEMS/src/module_MEDIATOR_methods.F90
     NEMS/src/module_MEDIATOR.F90
@@ -250,41 +209,79 @@ add_executable(
     NEMS/src/nems_c_rusage.c
     NEMS/src/ENS_Cpl/ENS_CplComp_ESMFMod_STUB.F90
 )
+add_dependencies(ufs-weather-model-lib fv3atm)
+target_link_libraries(ufs-weather-model-lib PRIVATE esmf)
 
-target_compile_definitions(NEMS.exe PRIVATE -DESMF_VERSION_MAJOR=${ESMF_VERSION_MAJOR})
-target_compile_definitions(NEMS.exe PRIVATE -DFRONT_FMS)
-target_compile_definitions(NEMS.exe PRIVATE -DFRONT_FV3=fv3gfs_cap_mod)
+target_compile_definitions(ufs-weather-model-lib PRIVATE -DESMF_VERSION_MAJOR=${ESMF_VERSION_MAJOR})
+target_compile_definitions(ufs-weather-model-lib PRIVATE -DFRONT_FMS)
+target_compile_definitions(ufs-weather-model-lib PRIVATE -DFRONT_FV3=fv3gfs_cap_mod)
+
+target_include_directories(ufs-weather-model-lib PRIVATE ${PROJECT_BINARY_DIR}/NEMS/src)
+target_include_directories(ufs-weather-model-lib PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/FV3/mod)
+target_include_directories(ufs-weather-model-lib PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/FMS/mod)
+
+if(WW3)
+    target_compile_definitions(ufs-weather-model-lib PRIVATE -DFRONT_WW3=WMESMFMD)
+    target_include_directories(ufs-weather-model-lib PRIVATE ${PROJECT_SOURCE_DIR}/WW3/model/mod)
+    set_target_properties(ufs-weather-model-lib PROPERTIES Fortran_MODULE_DIRECTORY ${PROJECT_SOURCE_DIR}/WW3/model/mod)
+    add_dependencies(ufs-weather-model-lib ww3_nems)
+endif()
+
+set_target_properties(ufs-weather-model-lib PROPERTIES Fortran_MODULE_DIRECTORY ${PROJECT_BINARY_DIR}/mod)
+target_include_directories(ufs-weather-model-lib PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/mod>
+                                       $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/FV3/mod>
+                                       $<INSTALL_INTERFACE:${CMAKE_INSTALL_PREFIX}/mod>)
+
+configure_file(${PROJECT_SOURCE_DIR}/NEMS/src/ESMFVersionDefine_ESMF_NUOPC.h
+               ${PROJECT_BINARY_DIR}/NEMS/src/ESMFVersionDefine.h
+               COPYONLY)
+# until we remove "../ESMFVersionDefine.h" from ENS_Cpl/ENS_CplComp_ESMFMod_STUB.F90
+configure_file(${PROJECT_SOURCE_DIR}/NEMS/src/ESMFVersionDefine_ESMF_NUOPC.h
+               ${PROJECT_BINARY_DIR}/NEMS/ESMFVersionDefine.h
+               COPYONLY)
+configure_file(${PROJECT_SOURCE_DIR}/NEMS/src/ESMFConvenienceMacros.h
+               ${PROJECT_BINARY_DIR}/NEMS/src/ESMFConvenienceMacros.h
+               COPYONLY)
 
-target_include_directories(NEMS.exe PRIVATE ${ESMF_MOD} )
+target_link_libraries(ufs-weather-model-lib PUBLIC fv3atm)
+
+###############################################################################
+### UFS executable
+###############################################################################
+add_executable(NEMS.exe
+               NEMS/src/MAIN_NEMS.F90
+)
+add_dependencies(NEMS.exe ufs-weather-model-lib)
+
+target_compile_definitions(NEMS.exe PRIVATE -DESMF_VERSION_MAJOR=${ESMF_VERSION_MAJOR})
 target_include_directories(NEMS.exe PRIVATE ${PROJECT_BINARY_DIR}/NEMS/src)
 
-set_target_properties(NEMS.exe PROPERTIES Fortran_MODULE_DIRECTORY ${PROJECT_BINARY_DIR}/NEMS/mod)
+set_target_properties(NEMS.exe PROPERTIES Fortran_MODULE_DIRECTORY ${PROJECT_BINARY_DIR}/mod)
 
 if(WW3)
     target_compile_definitions(NEMS.exe PRIVATE -DFRONT_WW3=WMESMFMD)
     set_target_properties(NEMS.exe PROPERTIES Fortran_MODULE_DIRECTORY ${PROJECT_SOURCE_DIR}/WW3/model/mod)
-    set(WW3_LIBS ${PROJECT_SOURCE_DIR}/WW3/model/obj/libww3_multi_esmf.a)
     add_dependencies(NEMS.exe ww3_nems)
 endif()
 
-if(CCPP)
-    set(CCPP_LIBRARIES ccppdriver ccppphys ccpp)
-else()
-    set(IPD_LIBRARIES ipd)
-endif()
-
 target_link_libraries(NEMS.exe
-                      fv3cap
-                      fv3core
-                      io
-                      ${IPD_LIBRARIES}
-                      gfsphysics
-                      ${CCPP_LIBRARIES}
-                      fv3cpl
-                      stochastic_physics
-                      fms
+                      ufs-weather-model-lib
                       ${WW3_LIBS}
-                      ${NCEP_LIBS} ${ESMF_LIBS} ${NETCDF_LIBS} ${MKL_LIB})
+                      esmf NetCDF::NetCDF_Fortran ${MKL_LIB})
+
+###############################################################################
+### Install
+###############################################################################
+install(
+  TARGETS ufs-weather-model-lib fms
+  EXPORT ufs-weather-model-lib-config
+  LIBRARY DESTINATION lib
+  ARCHIVE DESTINATION lib)
+install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/mod DESTINATION ${CMAKE_INSTALL_PREFIX})
+
+install(EXPORT ufs-weather-model-lib-config
+  DESTINATION lib/cmake
+)
 
 ###############################################################################
 ### done

diff --git a/FV3 b/FV3
+72 −38		CMakeLists.txt
+1 −1		ccpp/driver/CMakeLists.txt
+4 −5		cpl/CMakeLists.txt
+4 −3		gfsphysics/CMakeLists.txt
+10 −9		io/CMakeLists.txt
+3 −2		ipd/CMakeLists.txt
+4 −3		stochastic_physics/CMakeLists.txt