diff --git a/tools/contrib/build_esmf_on_gadi.sh b/tools/contrib/build_esmf_on_gadi.sh new file mode 100755 index 0000000..24bd55a --- /dev/null +++ b/tools/contrib/build_esmf_on_gadi.sh @@ -0,0 +1,27 @@ +#!/bin/bash +set -x +set -e + +git archive --remote=git://git.code.sf.net/p/esmf/esmf --format=tar --prefix=esmf/ ESMF_7_1_0r | tar xf - +#module load netcdf/4.7.4 +module load netcdf/4.6.3 +#module load intel-compiler/2020.2.254 +module load intel-compiler/2019.3.199 +#module load gcc +mkdir -p bin +cd esmf +export ESMF_DIR=$(pwd) +export ESMF_F90COMPILER=ifort +export ESMF_F90LINKER=ifort +export ESMF_NETCDF=nc-config +#export ESMF_NETCDF="split" +export ESMF_NETCDF_INCLUDE=$NETCDF_ROOT/include +export ESMF_NETCDF_LIBPATH=$NETCDF_ROOT/lib +export ESMF_NETCDF_LIBS="-lnetcdff -lnetcdf" +make clean +make +cd src/apps/ESMF_RegridWeightGen +make +cd ../../../../ +cp esmf/apps/appsO/*/ESMF_RegridWeightGen bin/ +export PATH=$(pwd)/bin:$PATH diff --git a/tools/contrib/build_fixed_esmf_on_gadi.sh b/tools/contrib/build_fixed_esmf_on_gadi.sh new file mode 100755 index 0000000..971cefa --- /dev/null +++ b/tools/contrib/build_fixed_esmf_on_gadi.sh @@ -0,0 +1,30 @@ +#!/bin/bash +set -x +set -e + +# use Russ' fixed code - see https://github.com/COSIMA/access-om2/issues/216 +git clone https://github.com/COSIMA/esmf.git || true +cd esmf +git checkout f536c3e12d501e2ee1d4caf95cb425f5be6e84d3 + +#module load netcdf/4.7.4 +module load netcdf/4.6.3 +#module load intel-compiler/2020.2.254 +module load intel-compiler/2019.3.199 +#module load gcc +export ESMF_DIR=$(pwd) +export ESMF_F90COMPILER=ifort +export ESMF_F90LINKER=ifort +export ESMF_NETCDF=nc-config +#export ESMF_NETCDF="split" +export ESMF_NETCDF_INCLUDE=$NETCDF_ROOT/include +export ESMF_NETCDF_LIBPATH=$NETCDF_ROOT/lib +export ESMF_NETCDF_LIBS="-lnetcdff -lnetcdf" +make clean +make +cd src/apps/ESMF_RegridWeightGen +make +cd ../../../../ +mkdir -p bin +cp esmf/apps/appsO/*/ESMF_RegridWeightGen bin/ESMF_RegridWeightGen_f536c3e12d +export PATH=$(pwd)/bin:$PATH diff --git a/tools/esmgrids b/tools/esmgrids index 130c8d0..514b560 160000 --- a/tools/esmgrids +++ b/tools/esmgrids @@ -1 +1 @@ -Subproject commit 130c8d00774c624c196fc2ca15f5cad78c4cdbc1 +Subproject commit 514b560adf7b9094081408da49ecb3e19ef0353b diff --git a/tools/make_remap_weights.py b/tools/make_remap_weights.py index 18b5850..fbd6058 100755 --- a/tools/make_remap_weights.py +++ b/tools/make_remap_weights.py @@ -19,16 +19,15 @@ from esmgrids.mom_grid import MomGrid # noqa from esmgrids.core2_grid import Core2Grid # noqa from esmgrids.jra55_grid import Jra55Grid # noqa +from esmgrids.era5_grid import Era5Grid # noqa from esmgrids.jra55_river_grid import Jra55RiverGrid # noqa from esmgrids.daitren_runoff_grid import DaitrenRunoffGrid # noqa """ This script makes all of the remapping weights for ACCESS-OM2. -Run example: +See make_remap_weights.sh for run examples -./make_remap_weights.py /short/x77/nah599/access-om2/input/ \ -/g/data/ua8/JRA55-do/RYF/v1-3/ """ def convert_to_scrip_output(weights): @@ -97,11 +96,14 @@ def create_weights(src_grid, dest_grid, npes, method, ignore_unmapped = [] try: - cmd = ['mpirun', '-np', str(npes), 'ESMF_RegridWeightGen'] + \ - ['--netcdf4', + cmd = ['mpirun', '-np', str(npes), + '/g/data/ik11/inputs/access-om2/bin/ESMF_RegridWeightGen_f536c3e12d', + '--netcdf4', '-s', src_grid_scrip, - '-d', dest_grid_scrip, '-m', method, - '-w', regrid_weights] + ignore_unmapped + '-d', dest_grid_scrip, + '-m', method, + '-w', regrid_weights] \ + + ignore_unmapped print(cmd) sp.check_output(cmd) except sp.CalledProcessError as e: @@ -121,9 +123,9 @@ def create_weights(src_grid, dest_grid, npes, method, return regrid_weights -def find_grid_defs(input_dir, jra55_input, core_input): +def find_ocean_grid_defs(input_dir): """ - Return a dictionary containing the grid definition files. + Return a dictionary containing ocean grid definition files. """ d = {} @@ -133,51 +135,40 @@ def find_grid_defs(input_dir, jra55_input, core_input): os.path.join(input_dir, 'mom_025deg', 'ocean_mask.nc')) d['MOM01'] = (os.path.join(input_dir, 'mom_01deg', 'ocean_hgrid.nc'), os.path.join(input_dir, 'mom_01deg', 'ocean_mask.nc')) - d['CORE2'] = os.path.join(core_input, 't_10.0001.nc') - d['JRA55'] = os.path.join(jra55_input, 'RYF.t_10.1990_1991.nc') - d['JRA55_runoff'] = os.path.join(jra55_input, - 'RYF.runoff_all.1990_1991.nc') - d['Daitren_runoff'] = os.path.join(core_input, 'runoff.daitren.clim.10FEB2011.nc') - return d def main(): parser = argparse.ArgumentParser() - parser.add_argument('input_dir', help=""" + parser.add_argument('--accessom2_input_dir', required=True, help=""" The ACCESS-OM2 input directory.""") - parser.add_argument('jra55_input', help=""" - The JRA55 input directory.""") - parser.add_argument('core_input', help=""" - The CORE input directory.""") - parser.add_argument('--atm', default=None, help=""" + parser.add_argument('--atm_forcing_file', required=True, help=""" + An atmospheric or runoff forcing field.""") + parser.add_argument('--atm', required=True, default=None, help=""" Atmosphere grid to regrid from, can be one of: - CORE2, JRA55, JRA55_runoff, Daitren_runoff""") + CORE2, JRA55, JRA55_runoff, Daitren_runoff, ERA5""") parser.add_argument('--ocean', default=None, help=""" Ocean grid to regrid to, can be one of: - MOM1, MOM01, MOM025""") + MOM1, MOM01, MOM025; default is all three""") parser.add_argument('--method', default=None, help=""" - The interpolation method to use, can be patch, conserve or conserve2nd""") + The interpolation method to use, can be patch or conserve; default is both""") parser.add_argument('--npes', default=None, help=""" The number of PEs to use.""") - parser.add_argument('--unmask_dest', + parser.add_argument('--unmask_dest', action='store_true', help='Ignore destination grid mask') args = parser.parse_args() - atm_options = ['JRA55', 'JRA55_runoff', 'CORE2', 'Daitren_runoff'] + atm_options = ['JRA55', 'JRA55_runoff', 'CORE2', 'Daitren_runoff', 'ERA5'] ocean_options = ['MOM1', 'MOM025', 'MOM01'] - method_options = ['patch', 'conserve', 'conserve2nd'] +# method_options = ['patch', 'conserve', 'conserve2nd'] + method_options = ['patch', 'conserve'] - if args.atm is None: - args.atm = atm_options - else: - if args.atm not in atm_options: - print("Error: bad atm grid.", file=sys.stderr) - parser.print_help() - return 1 - args.atm = [args.atm] + if args.atm not in atm_options: + print("Error: bad atm grid.", file=sys.stderr) + parser.print_help() + return 1 if args.ocean is None: args.ocean = ocean_options @@ -197,37 +188,38 @@ def main(): import multiprocessing as mp args.npes = mp.cpu_count() // 2 - grid_file_dict = find_grid_defs(args.input_dir, args.jra55_input, args.core_input) + ocean_grid_file_dict = find_ocean_grid_defs(args.accessom2_input_dir) for ocean in args.ocean: - umask_file = grid_file_dict[ocean][1] - dest_grid = MomGrid.fromfile(grid_file_dict[ocean][0], + umask_file = ocean_grid_file_dict[ocean][1] + dest_grid = MomGrid.fromfile(ocean_grid_file_dict[ocean][0], mask_file=umask_file) - for atm in args.atm: - - if atm == 'CORE2': - src_grid = Core2Grid(grid_file_dict[atm]) - elif atm == 'Daitren_runoff': - src_grid = DaitrenRunoffGrid(grid_file_dict[atm]) - elif atm == 'JRA55': - src_grid = Jra55Grid(grid_file_dict[atm]) - elif atm == 'JRA55_runoff': - src_grid = Jra55RiverGrid(grid_file_dict[atm], calc_areas=False) - else: - print('Unrecognised atmosphere grid: {}'.format(atm)) - return 1 - for method in args.method: + if args.atm == 'CORE2': + src_grid = Core2Grid(args.atm_forcing_file) + elif args.atm == 'Daitren_runoff': + src_grid = DaitrenRunoffGrid(args.atm_forcing_file) + elif args.atm == 'JRA55': + src_grid = Jra55Grid(args.atm_forcing_file) + elif args.atm == 'JRA55_runoff': + src_grid = Jra55RiverGrid(args.atm_forcing_file, calc_areas=False) + elif args.atm == 'ERA5': + src_grid = Era5Grid(args.atm_forcing_file) + else: + print('Unrecognised atmosphere grid: {}'.format(args.atm)) + return 1 + + for method in args.method: - weights = create_weights(src_grid, dest_grid, args.npes, - method, unmasked_dest=args.unmask_dest) - if not weights: - return 1 - weights = convert_to_scrip_output(weights) - if not weights: - return 1 + weights = create_weights(src_grid, dest_grid, args.npes, + method, unmasked_dest=args.unmask_dest) + if not weights: + return 1 + weights = convert_to_scrip_output(weights) + if not weights: + return 1 - shutil.move(weights, '{}_{}_{}.nc'.format(atm, ocean, method)) + shutil.move(weights, '{}_{}_{}.nc'.format(args.atm, ocean, method)) return 0 diff --git a/tools/make_remap_weights.sh b/tools/make_remap_weights.sh index 0496a9c..2ccabe2 100644 --- a/tools/make_remap_weights.sh +++ b/tools/make_remap_weights.sh @@ -1,21 +1,26 @@ #!/bin/bash #PBS -P x77 #PBS -q normal -#PBS -l ncpus=256,mem=512GB,walltime=05:00:00,jobfs=100GB +#PBS -l ncpus=288,mem=512GB,walltime=05:00:00,jobfs=100GB #PBS -l wd +#PBS -lstorage=scratch/v45+gdata/hh5+gdata/ik11+gdata/ua8+gdata/rt52 module purge -module load openmpi +module load python3-as-python module load nco -module load esmf/7.1.0r-intel module use /g/data/hh5/public/modules module load conda/analysis3 +module unload openmpi +module load openmpi/4.0.2 # Make all 1 deg weights. -time ./make_remap_weights.py /short/x77/nah599/access-om2/input/ /g/data/ua8/JRA55-do/RYF/v1-3/ /short/x77/nah599/access-om2/input/yatm_1deg/ --ocean MOM1 --npes 256 +time ./make_remap_weights.py --accessom2_input_dir /g/data/ik11/inputs/access-om2/input_20201102 --atm_forcing_file /g/data/ua8/JRA55-do/RYF/v1-3/RYF.t_10.1990_1991.nc --ocean MOM1 --npes 288 --atm JRA55 +time ./make_remap_weights.py --accessom2_input_dir /g/data/ik11/inputs/access-om2/input_20201102 --atm_forcing_file /g/data/rt52/era5/single-levels/reanalysis/2t/1980/2t_era5_oper_sfc_19800101-19800131.nc --ocean MOM1 --npes 288 --atm ERA5 # Make all 0.25 deg weights. -time ./make_remap_weights.py /short/x77/nah599/access-om2/input/ /g/data/ua8/JRA55-do/RYF/v1-3/ /short/x77/nah599/access-om2/input/yatm_1deg/ --ocean MOM025 --npes 256 +time ./make_remap_weights.py --accessom2_input_dir /g/data/ik11/inputs/access-om2/input_20201102 --atm_forcing_file /g/data/ua8/JRA55-do/RYF/v1-3/RYF.t_10.1990_1991.nc --ocean MOM025 --npes 288 --atm JRA55 +time ./make_remap_weights.py --accessom2_input_dir /g/data/ik11/inputs/access-om2/input_20201102 --atm_forcing_file /g/data/rt52/era5/single-levels/reanalysis/2t/1980/2t_era5_oper_sfc_19800101-19800131.nc --ocean MOM025 --npes 288 --atm ERA5 # Make all 0.1 deg weights. -time ./make_remap_weights.py /short/x77/nah599/access-om2/input/ /g/data/ua8/JRA55-do/RYF/v1-3/ /short/x77/nah599/access-om2/input/yatm_1deg/ --ocean MOM01 --npes 256 +time ./make_remap_weights.py --accessom2_input_dir /g/data/ik11/inputs/access-om2/input_20201102 --atm_forcing_file /g/data/ua8/JRA55-do/RYF/v1-3/RYF.t_10.1990_1991.nc --ocean MOM01 --npes 288 --atm JRA55 +time ./make_remap_weights.py --accessom2_input_dir /g/data/ik11/inputs/access-om2/input_20201102 --atm_forcing_file /g/data/rt52/era5/single-levels/reanalysis/2t/1980/2t_era5_oper_sfc_19800101-19800131.nc --ocean MOM01 --npes 288 --atm ERA5