diff --git a/CMakeLists.txt b/CMakeLists.txt index 3a644ef583..f6db881f48 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -540,6 +540,12 @@ set(NCIO_MINBLOCKSIZE 256 CACHE STRING "Minimum I/O Blocksize for netCDF classic if(NETCDF_ENABLE_NETCDF4) set(USE_NETCDF4 ON CACHE BOOL "") set(NETCDF_ENABLE_NETCDF4 ON CACHE BOOL "") + + # Configure the nc4dispatch.c file to H5Linclude plugin path at + # library initialization. + configure_file("${CMAKE_SOURCE_DIR}/libsrc4/nc4dispatch.c.in" + "${CMAKE_BINARY_DIR}/libsrc4/nc4dispatch.c" @ONLY NEWLINE_STYLE LF) + else() set(USE_HDF4_FILE_TESTS OFF) set(USE_HDF4 OFF) diff --git a/Makefile.am b/Makefile.am index 8ccfd6a4d2..560817496d 100644 --- a/Makefile.am +++ b/Makefile.am @@ -25,7 +25,7 @@ TEST_EXTENSIONS = .sh # comment thread at https://github.com/Unidata/netcdf-c/pull/2348 # for more information. ## -AM_DISTCHECK_CONFIGURE_FLAGS = --without-plugin-dir +#AM_DISTCHECK_CONFIGURE_FLAGS = --without-plugin-dir # These files get added to the distribution. EXTRA_DIST = README.md COPYRIGHT INSTALL.md test_prog.c lib_flags.am \ diff --git a/configure.ac b/configure.ac index 9888d0af65..ea735169c7 100644 --- a/configure.ac +++ b/configure.ac @@ -1618,7 +1618,7 @@ if test "x$enable_hdf5" = xyes; then # H5Pset_fapl_mpiposix and H5Pget_fapl_mpiposix have been removed since HDF5 1.8.12. # Use H5Pset_fapl_mpio and H5Pget_fapl_mpio, instead. - AC_CHECK_FUNCS([H5Pget_fapl_mpio H5Pset_deflate H5Z_SZIP H5Pset_all_coll_metadata_ops H5Literate H5Literate2]) + AC_CHECK_FUNCS([H5Pget_fapl_mpio H5Pset_deflate H5Z_SZIP H5Pset_all_coll_metadata_ops H5Literate H5Literate2 H5PLappend]) # Check to see if HDF5 library has collective metadata APIs, (HDF5 >= 1.10.0) if test "x$ac_cv_func_H5Pset_all_coll_metadata_ops" = xyes; then @@ -2094,9 +2094,14 @@ AC_SUBST(STD_FILTERS,[$std_filters]) # If user wants, then install selected standard filters AC_MSG_CHECKING([whether and where we should install plugins]) -AC_ARG_WITH([plugin-dir], [AS_HELP_STRING([--with-plugin-dir=|no|--without-plugin-dir], +AC_ARG_WITH([plugin-dir], [AS_HELP_STRING([--with-plugin-dir=|yes|no|--without-plugin-dir], [Install selected standard filters in specified or default directory])], [],[with_plugin_dir=no]) +if test "x$have_zstd" = xyes; then + if test "x$with_plugin_dir" = xno; then + with_plugin_dir=yes + fi +fi AC_MSG_RESULT([$with_plugin_dir]) if test "x$with_plugin_dir" = xno ; then # option missing|disabled with_plugin_dir=no @@ -2237,6 +2242,7 @@ AC_CONFIG_FILES(dap4_test/findtestserver4.c:ncdap_test/findtestserver.c.in) AC_CONFIG_FILES(dap4_test/pingurl4.c:ncdap_test/pingurl.c) AC_CONFIG_FILES([h5_test/run_par_tests.sh], [chmod ugo+x h5_test/run_par_tests.sh]) AC_CONFIG_FILES([nc_test4/run_par_test.sh], [chmod ugo+x nc_test4/run_par_test.sh]) +AC_CONFIG_FILES([libsrc4/nc4dispatch.c], [chmod ugo+x nc_test4/run_par_test.sh]) AC_CONFIG_FILES([nc_perf/run_par_bm_test.sh], [chmod ugo+x nc_perf/run_par_bm_test.sh]) AC_CONFIG_FILES([nc_perf/run_gfs_test.sh], [chmod ugo+x nc_perf/run_gfs_test.sh]) AC_CONFIG_FILES(nczarr_test/timer_utils.h:unit_test/timer_utils.h) diff --git a/libsrc4/CMakeLists.txt b/libsrc4/CMakeLists.txt index 09227318e6..75313f1122 100644 --- a/libsrc4/CMakeLists.txt +++ b/libsrc4/CMakeLists.txt @@ -6,7 +6,7 @@ # See netcdf-c/COPYRIGHT file for more info. # Process these files with m4. -set(libsrc4_SOURCES nc4dispatch.c nc4attr.c nc4dim.c nc4grp.c +set(libsrc4_SOURCES ${netCDF_BINARY_DIR}/libsrc4/nc4dispatch.c nc4attr.c nc4dim.c nc4grp.c nc4internal.c nc4type.c nc4var.c ncfunc.c ncindex.c nc4cache.c) add_library(netcdf4 OBJECT ${libsrc4_SOURCES}) diff --git a/libsrc4/nc4dispatch.c b/libsrc4/nc4dispatch.c.in similarity index 88% rename from libsrc4/nc4dispatch.c rename to libsrc4/nc4dispatch.c.in index 2aa5987824..58aa436f16 100644 --- a/libsrc4/nc4dispatch.c +++ b/libsrc4/nc4dispatch.c.in @@ -15,6 +15,9 @@ #include "nc4internal.h" #include "nc4dispatch.h" #include "nc.h" +#ifdef HAVE_H5PLAPPEND +#include "hdf5.h" +#endif /* If user-defined formats are in use, we need to declare their * dispatch tables. */ @@ -52,6 +55,13 @@ NC4_initialize(void) return ret; #endif /* USE_UDF0 */ +#ifdef HAVE_H5PLAPPEND + /* Add the place where plugins were installed to the plugin + * path. */ + if ((ret = H5PLappend("@PLUGIN_INSTALL_DIR@"))) + return ret; +#endif + #ifdef LOGGING if(getenv(NCLOGLEVELENV) != NULL) { char* slevel = getenv(NCLOGLEVELENV); diff --git a/nc_test4/CMakeLists.txt b/nc_test4/CMakeLists.txt index 33a105f5b2..bd56f3e6cd 100644 --- a/nc_test4/CMakeLists.txt +++ b/nc_test4/CMakeLists.txt @@ -102,6 +102,11 @@ IF(HAVE_H5Z_SZIP) ENDIF() ENDIF() +IF(HAVE_ZSTD) + BUILD_BIN_TEST(tst_zstd) + add_sh_test(nc_test4 run_zstd_test) +ENDIF() + # Copy some test files from current source dir to out-of-tree build dir. FILE(GLOB COPY_FILES ${CMAKE_CURRENT_SOURCE_DIR}/*.nc ${CMAKE_CURRENT_SOURCE_DIR}/ref_bzip2.c ${CMAKE_CURRENT_SOURCE_DIR}/*.sh ${CMAKE_CURRENT_SOURCE_DIR}/*.h5 ${CMAKE_CURRENT_SOURCE_DIR}/*.cdl) diff --git a/nc_test4/Makefile.am b/nc_test4/Makefile.am index 6d1c0e2e44..4776403801 100644 --- a/nc_test4/Makefile.am +++ b/nc_test4/Makefile.am @@ -11,10 +11,10 @@ include $(top_srcdir)/lib_flags.am # Un comment to use a more verbose test driver -#SH_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose -#sh_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose -#LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose -#TESTS_ENVIRONMENT = export SETX=1; +SH_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose +sh_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose +LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose +TESTS_ENVIRONMENT = export SETX=1; TEST_EXTENSIONS = .sh @@ -113,6 +113,12 @@ endif # NETCDF_ENABLE_PLUGINS endif # USE_HDF5 endif # NETCDF_BUILD_UTILITIES +# Zstandard tests. +if HAVE_ZSTD +check_PROGRAMS += tst_zstd +TESTS += run_zstd_test.sh +endif # HAVE_ZSTD + # This are extra tests that will only be run if netcdf-4 is configured # with --enable-parallel-tests. if TEST_PARALLEL4 diff --git a/nc_test4/run_zstd_test.sh b/nc_test4/run_zstd_test.sh new file mode 100755 index 0000000000..cd234c43a9 --- /dev/null +++ b/nc_test4/run_zstd_test.sh @@ -0,0 +1,32 @@ +#!/bin/sh + +# This .in file is processed at build time into a shell that runs some +# zstd for netCDF. + +# Ed Hartnett, 8/10/24 + +set -e + +if test "x$srcdir" = x ; then srcdir=`pwd`; fi +. ../test_common.sh + +# Load the findplugins function +. ${builddir}/findplugin.sh +echo "findplugin.sh loaded" +echo "${HDF5_PLUGIN_DIR}" + +findplugin h5zstd +echo "HDF5_PLUGIN_DIR=$HDF5_PLUGIN_DIR" +./tst_zstd + + +echo +echo "Testing Zstandard compression..." + +export HDF5_PLUGIN_PATH="${HDF5_PLUGIN_DIR}" +echo "set HDF5_PLUGIN_PATH: ${HDF5_PLUGIN_PATH}" + +echo "SUCCESS!" + + + diff --git a/nc_test4/tst_zstd.c b/nc_test4/tst_zstd.c new file mode 100644 index 0000000000..7ae201fbf6 --- /dev/null +++ b/nc_test4/tst_zstd.c @@ -0,0 +1,78 @@ +/* This is part of the netCDF package. + Copyright 2018 University Corporation for Atmospheric Research/Unidata + See COPYRIGHT file for conditions of use. + + Test netcdf-4 variables. + Ed Hartnett, 8/7/24 +*/ + +#include "nc_tests.h" +#include "err_macros.h" +#include "netcdf.h" + +#define FILE_NAME "tst_zstd.nc" +#define VAR_FLOAT_NAME "GALE_data" +#define VAR_DOUBLE_NAME "ERICA_data" +#define NDIM2 2 +#define DIM1_NAME "x" +#define DIM1_LEN 2 +#define DIM2_NAME "y" +#define DIM2_LEN 3 + +int +main(int argc, char **argv) +{ + int ncid, dimids[NDIM2]; + int float_varid, double_varid; + float float_in[DIM1_LEN][DIM2_LEN], float_out[DIM1_LEN][DIM2_LEN] = {{-.1f, 9999.99f, 100.001f},{-.1f, 9999.99f, 100.001f}}; + double double_in[DIM1_LEN][DIM2_LEN], double_out[DIM1_LEN][DIM2_LEN] = {{0.02, .1128, 1090.1},{0.02, .1128, 1090.1}}; + int i, j, ret; + + + printf("\n*** Testing netcdf-4 zstd compression.\n"); + + printf("*** testing netcdf-4 zstd with float..."); + { + int nvars_in, varids_in[2]; + + /* Create a netcdf file with one dim and two vars. */ + if (nc_create(FILE_NAME, NC_NETCDF4, &ncid)) ERR; + if (nc_def_dim(ncid, DIM1_NAME, DIM1_LEN, &dimids[0])) ERR; + if (nc_def_dim(ncid, DIM2_NAME, DIM2_LEN, &dimids[1])) ERR; + if (nc_def_var(ncid, VAR_FLOAT_NAME, NC_FLOAT, 2, dimids, &float_varid)) ERR; + if (nc_def_var(ncid, VAR_DOUBLE_NAME, NC_DOUBLE, 2, dimids, &double_varid)) ERR; + if ((ret = nc_def_var_zstandard(ncid, float_varid, 4))) + { + printf("error: %s\n", nc_strerror(ret)); + ERR; + } + if (nc_def_var_zstandard(ncid, double_varid, 4)) ERR; + if (nc_put_var_float(ncid, float_varid, (float *)float_out)) ERR; + if (nc_put_var_double(ncid, double_varid, (double *)double_out)) ERR; + if (nc_close(ncid)) ERR; + + /* Open the file and make sure nc_inq_varids yields correct + * result. */ + if (nc_open(FILE_NAME, NC_NOWRITE, &ncid)) ERR; + if (nc_inq_varids(ncid, &nvars_in, varids_in)) ERR; + if (nvars_in != 2 || varids_in[0] != 0 || varids_in[1] != 1) ERR; + if (nc_get_var_float(ncid, float_varid, (float *)float_in)) ERR; + if (nc_get_var_double(ncid, double_varid, (double *)double_in)) ERR; + for (i = 0; i < DIM1_LEN; i++) + { + for (j = 0; j < DIM2_LEN; j++) + { + if (float_in[i][j] != float_out[i][j]) ERR; + if (double_in[i][j] != double_out[i][j]) ERR; + } + } + + + if (nc_close(ncid)) ERR; + } + + SUMMARIZE_ERR; + SUMMARIZE_ERR; + + FINAL_RESULTS; +} diff --git a/nczarr_test/run_interop.sh b/nczarr_test/run_interop.sh index e7a22f9ba7..b0cb1690fd 100755 --- a/nczarr_test/run_interop.sh +++ b/nczarr_test/run_interop.sh @@ -45,7 +45,7 @@ testcasezip() { } testcases3() { - set -x + #set -x echo " o Running S3 Testcase: $1 $2" zext=s3 base=$1