adding more zstd testing

CMakeLists.txt

@@ -1840,7 +1840,7 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/s3gc.in ${CMAKE_CURRENT_BINARY_DIR}/s
 #####
 # Build and copy nc_test4/findplugin.sh to various places
 #####
-foreach(CC nc_test4 nczarr_test v3_nczarr_test plugins examples/C)
+foreach(CC nc_test4 nczarr_test v3_nczarr_test plugins h5_test examples/C)
     configure_file(${CMAKE_CURRENT_SOURCE_DIR}/plugins/findplugin.in ${CMAKE_CURRENT_BINARY_DIR}/${CC}/findplugin.sh @ONLY NEWLINE_STYLE LF)
 endforeach()
 

configure.ac

@@ -2238,7 +2238,7 @@ AC_MSG_NOTICE([generating header files and makefiles])
 AC_CONFIG_FILES(test_common.sh:test_common.in)
 AC_CONFIG_FILES(s3cleanup.sh:s3cleanup.in, [chmod ugo+x s3cleanup.sh])
 AC_CONFIG_FILES(s3gc.sh:s3gc.in, [chmod ugo+x s3gc.sh])
-for FP in plugins nc_test4 nczarr_test examples/C ; do
+for FP in plugins nc_test4 nczarr_test h5_test examples/C ; do
 AC_CONFIG_FILES(${FP}/findplugin.sh:plugins/findplugin.in, [chmod ugo+x ${FP}/findplugin.sh])
 done
 AC_CONFIG_FILES(ncdap_test/findtestserver.c:ncdap_test/findtestserver.c.in, [chmod ugo+x ncdap_test/findtestserver.c])

h5_test/CMakeLists.txt

@@ -24,3 +24,17 @@ IF(TEST_PARALLEL4)
   build_bin_test(tst_h_par_compress)
   add_sh_test(h5_test run_par_tests)
 ENDIF()
+
+# If zstd is enabled, run a test for it.
+if(NETCDF_ENABLE_PLUGINS)
+  if(NETCDF_ENABLE_FILTER_ZSTD)
+    build_bin_test(tst_h_zstd)
+    add_sh_test(h5_test run_h5_zstd_tests)    
+  endif()
+endif()
+
+# Copy the shell file to build directory.
+file(COPY run_h5_zstd_tests.sh DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/ FILE_PERMISSIONS OWNER_WRITE OWNER_READ OWNER_EXECUTE)
+
+
+

h5_test/Makefile.am

@@ -2,16 +2,24 @@
 # Copyright 2018 University Corporation for Atmospheric Research/Unidata
 # See COPYRIGHT file for conditions of use.
 #
-# This entire directory will be skipped, unless the configure script
-# is run with --enable-netcdf-4. This directory contains tests that
-# only use HDF5; these tests don't use netCDF at all.
+# This directory contains tests that only use HDF5; these tests don't
+# use netCDF at all. These tests will only be run if netcdf-4 is
+# enabled.
 #
 # If one of these tests fails, then netCDF-4 will not work correctly.
 # Ed Hartnett, Ward Fisher
 
 # Set AM_CPPFLAGS and AM_LDFLAGS based on user choices.
 include $(top_srcdir)/lib_flags.am
 
+# Un comment to use a more verbose test driver
+# SH_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose
+# sh_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose
+# LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose
+# TESTS_ENVIRONMENT = export SETX=1;
+
+TEST_EXTENSIONS = .sh
+
 # These programs test HDF5 to make sure it will do (some of) the stuff
 # that netCDF-4 needs.
 H5TESTS = tst_h_files tst_h_files2 tst_h_files4 tst_h_atts		\
@@ -21,7 +29,7 @@ tst_h_strings tst_h_strings1 tst_h_strings2 tst_h_ints			\
 tst_h_dimscales tst_h_dimscales1 tst_h_dimscales2 tst_h_dimscales3	\
 tst_h_enums tst_h_dimscales4 tst_h_rename
 
-# If benchmarks were turned on, build and run a bunch more tests.
+# If benchmarks were turned on, build and run more tests.
 if BUILD_BENCHMARKS
 H5TESTS += tst_h_mem
 endif # BUILD_BENCHMARKS
@@ -32,6 +40,14 @@ check_PROGRAMS = $(H5TESTS)
 # List of tests to run.
 TESTS = $(H5TESTS)
 
+# If zstd is enabled, run a test for it.
+if NETCDF_ENABLE_PLUGINS
+if HAVE_ZSTD
+check_PROGRAMS += tst_h_zstd
+TESTS += run_h5_zstd_tests.sh
+endif # HAVE_ZSTD
+endif # NETCDF_ENABLE_PLUGINS
+
 # The parallel program is run from a script.
 if TEST_PARALLEL4
 check_PROGRAMS += tst_h_par tst_h_par_compress
@@ -41,7 +57,7 @@ endif
 # We must include these files in the distribution.
 EXTRA_DIST = run_par_tests.sh.in ref_tst_h_compounds.h5		\
 ref_tst_h_compounds2.h5 run_par_tests.sh ref_tst_compounds.nc	\
-h5_err_macros.h CMakeLists.txt
+h5_err_macros.h CMakeLists.txt run_h5_zstd_tests.sh
 
 # Clean up test results.
 CLEANFILES = tst_h_*.h5

h5_test/run_h5_zstd_tests.sh

@@ -0,0 +1,24 @@
+#!/bin/sh
+# Copyright 2024 University Corporation for Atmospheric
+# Research/Unidata. See netcdf-c/COPYRIGHT file for more info.
+
+# This shell runs the zstandard tests.
+
+# Ed Hartnett
+
+if test "x$srcdir" = x ; then srcdir=`pwd`; fi
+. ../test_common.sh
+
+# Load the findplugins function
+. ${builddir}/findplugin.sh
+export HDF5_PLUGIN_PATH="${HDF5_PLUGIN_DIR}"
+echo "HDF5_PLUGIN_PATH=$HDF5_PLUGIN_PATH"
+
+findplugin h5zstd
+echo ""
+echo "Testing zstandard I/O with HDF5..."
+
+./tst_h_zstd
+echo "SUCCESS!!!"
+
+exit 0

h5_test/tst_h_zstd.c

@@ -0,0 +1,111 @@
+/* This is part of the netCDF package. Copyright 2024 University
+   Corporation for Atmospheric Research/Unidata See COPYRIGHT file for
+   conditions of use.
+
+   Test HDF5 file code. These are not intended to be exhaustive tests,
+   but they use HDF5 the same way that netCDF-4 does, so if these
+   tests don't work, than netCDF-4 won't work either.
+
+   This files tests dataset creation and writing with zstd
+   compression. This test is only run if zstd compression is enabled
+   in the build.
+
+   Ed Hartnett
+*/
+
+#include "h5_err_macros.h"
+#include <hdf5.h>
+
+#define FILE_NAME "tst_h_zstd.h5"
+#define GRP_NAME "Bebbanburg"
+#define VAR_BOOL_NAME "Uhtred"
+#define DIM1_LEN 3
+#define H5Z_FILTER_ZSTD 32015
+
+int
+main()
+{
+    hid_t fileid, grpid, spaceid, datasetid;
+    int bool_out[DIM1_LEN] = {0, 1, 0};
+    hsize_t dims[1];
+
+    printf("\n*** Checking HDF5 variable functions with zstandard compression.\n");
+    printf("*** Checking HDF5 deflate filter setting and getting...");
+#define DEFLATE_LEVEL 9
+#define MAX_NAME 100
+#define NUM_CD_ELEM 10
+/* HDF5 defines this... */
+#define DEFLATE_NAME "deflate"
+#define ZSTD_NAME "zstd"
+    {
+        H5Z_filter_t filter;
+        int num_filters;
+        hid_t propid;
+        unsigned int flags, cd_values[NUM_CD_ELEM], filter_config;
+        size_t cd_nelems = NUM_CD_ELEM;
+        size_t namelen = MAX_NAME;
+        char name[MAX_NAME + 1];
+        htri_t avail = -1;
+        unsigned int id = H5Z_FILTER_ZSTD;
+        unsigned int ulevel = 1;
+        herr_t code;
+
+        /* Check that zstandard filter is available. */
+        if((avail = H5Zfilter_avail(id)) < 0) ERR;
+        if (!avail)
+        {
+            printf("zstandard filter not available, but expected to be.\n");
+            ERR;
+        }
+
+        /* Open file and create group. */
+        if ((fileid = H5Fcreate(FILE_NAME, H5F_ACC_TRUNC, H5P_DEFAULT,
+                                H5P_DEFAULT)) < 0) ERR;
+        if ((grpid = H5Gcreate1(fileid, GRP_NAME, 0)) < 0) ERR;
+
+        /* Write an array of bools, with zstandard compression. */
+        dims[0] = DIM1_LEN;
+        if ((propid = H5Pcreate(H5P_DATASET_CREATE)) < 0) ERR;
+        if (H5Pset_layout(propid, H5D_CHUNKED)) ERR;
+        if (H5Pset_chunk(propid, 1, dims)) ERR;
+        if ((code = H5Pset_filter(propid, id, H5Z_FLAG_OPTIONAL, 1, &ulevel)))
+            ERR;
+        if ((spaceid = H5Screate_simple(1, dims, dims)) < 0) ERR;
+        if ((datasetid = H5Dcreate1(grpid, VAR_BOOL_NAME, H5T_NATIVE_HBOOL,
+                                   spaceid, propid)) < 0) ERR;
+        if (H5Dwrite(datasetid, H5T_NATIVE_HBOOL, H5S_ALL, H5S_ALL, H5P_DEFAULT,
+                     bool_out) < 0) ERR;
+        if (H5Dclose(datasetid) < 0 ||
+            H5Pclose(propid) < 0 ||
+            H5Sclose(spaceid) < 0 ||
+            H5Gclose(grpid) < 0 ||
+            H5Fclose(fileid) < 0)
+            ERR;
+
+        /* Now reopen the file and check. */
+        if ((fileid = H5Fopen(FILE_NAME, H5F_ACC_RDONLY, H5P_DEFAULT)) < 0) ERR;
+        if ((grpid = H5Gopen1(fileid, GRP_NAME)) < 0) ERR;
+        if ((datasetid = H5Dopen1(grpid, VAR_BOOL_NAME)) < 0) ERR;
+        if ((propid = H5Dget_create_plist(datasetid)) < 0) ERR;
+
+        /* The possible values of filter (which is just an int) can be
+         * found in H5Zpublic.h. */
+        if ((num_filters = H5Pget_nfilters(propid)) < 0) ERR;
+        printf("num_filters %d\n", num_filters);
+        if (num_filters != 1) ERR;
+        if ((filter = H5Pget_filter2(propid, 0, &flags, &cd_nelems, cd_values,
+                                     namelen, name, &filter_config)) < 0) ERR;
+        if (filter != H5Z_FILTER_ZSTD || cd_nelems != 1 ||
+            cd_values[0] != ulevel) ERR;
+        if (strcmp(name, ZSTD_NAME)) ERR;
+
+        if (H5Dclose(datasetid) < 0 ||
+            H5Pclose(propid) < 0 ||
+            H5Gclose(grpid) < 0 ||
+            H5Fclose(fileid) < 0)
+            ERR;
+    }
+
+    SUMMARIZE_ERR;
+    FINAL_RESULTS;
+}

libhdf5/hdf5filter.c

@@ -124,6 +124,22 @@ PRINTFILTER(spec,"free");
     return NC_NOERR;
 }
 
+/**
+ * Add filter to netCDF's list of filters for a variable.
+ * @internal
+ *
+ * This function handles necessary filter ordering for shuffle and
+ * fletcher32 filters.
+ *
+ * @param var Pointer to the NC_VAR_INFO_T for a variable.
+ * @param id HDF5 filter ID.
+ * @param nparams Number of parameters in filter parameter list.
+ * @param params Array that holds nparams unsigned ints - the filter parameters.
+ * @param flags NetCDF flags about the filter.
+ *
+ * @return ::NC_NOERR on success, error code otherwise.
+ * @author Dennis Heimbigner
+ */
 int
 NC4_hdf5_addfilter(NC_VAR_INFO_T* var, unsigned int id, size_t nparams, const unsigned int* params, int flags)
 {
@@ -180,6 +196,16 @@ PRINTFILTERLIST(var,"add");
     return THROW(stat);
 }
 
+/**
+ * @internal
+ * Remove a filter from netCDF's list of filters for a variable.
+ *
+ * @param var Pointer to the NC_VAR_INFO_T for a variable.
+ * @param id HDF5 filter ID.
+ *
+ * @return ::NC_NOERR on success, error code otherwise.
+ * @author Dennis Heimbigner
+ */
 int
 NC4_hdf5_filter_remove(NC_VAR_INFO_T* var, unsigned int id)
 {
@@ -203,6 +229,17 @@ fprintf(stderr,"\tid=%s\n",id);
     return NC_ENOFILTER;
 }
 
+/**
+ * @internal
+ * Find a filter in netCDF's list of filters for a variable.
+ *
+ * @param var Pointer to the NC_VAR_INFO_T for a variable.
+ * @param id HDF5 filter ID.
+ * @param specp Filter specification.
+ *
+ * @return ::NC_NOERR on success, error code otherwise.
+ * @author Dennis Heimbigner
+ */
 int
 NC4_hdf5_filter_lookup(NC_VAR_INFO_T* var, unsigned int id, struct NC_HDF5_Filter** specp)
 {
@@ -224,6 +261,19 @@ NC4_hdf5_filter_lookup(NC_VAR_INFO_T* var, unsigned int id, struct NC_HDF5_Filte
     return NC_ENOFILTER;
 }
 
+/**
+ * @internal
+ * Add a filter for a variable in a netCDF/HDF5 file.
+ *
+ * @param ncid File ID.
+ * @param varid Variable ID.
+ * @param id HDF5 filter ID.
+ * @param nparams Number of parameters in filter parameter list.
+ * @param params Array that holds nparams unsigned ints - the filter parameters.
+ *
+ * @return ::NC_NOERR on success, error code otherwise.
+ * @author Dennis Heimbigner
+ */
 int
 NC4_hdf5_def_var_filter(int ncid, int varid, unsigned int id, size_t nparams,
                    const unsigned int* params)
@@ -365,6 +415,18 @@ NC4_hdf5_def_var_filter(int ncid, int varid, unsigned int id, size_t nparams,
     return stat;
 }
 
+/**
+ * @internal
+ * Inquire about a variable's filters in a netCDF/HDF5 file.
+ *
+ * @param ncid File ID.
+ * @param varid Variable ID.
+ * @param nfiltersp A pointer that gets the number of filters for the variable.
+ * @param ids A pointer that gets the ids of the filters.
+ *
+ * @return ::NC_NOERR on success, error code otherwise.
+ * @author Dennis Heimbigner
+ */
 int
 NC4_hdf5_inq_var_filter_ids(int ncid, int varid, size_t* nfiltersp, unsigned int* ids)
 {
@@ -404,6 +466,19 @@ NC4_hdf5_inq_var_filter_ids(int ncid, int varid, size_t* nfiltersp, unsigned int
 
 }
 
+/**
+ * @internal
+ * Inquire about a filter for a variable in a netCDF/HDF5 file.
+ *
+ * @param ncid File ID.
+ * @param varid Variable ID.
+ * @param id The HDF5 filter ID.
+ * @param nparamsp A pointer that gets the number of parameters for the filter.
+ * @param params A pointer that gets nparamsp parameters for the filter.
+ *
+ * @return ::NC_NOERR on success, error code otherwise.
+ * @author Dennis Heimbigner
+ */
 int
 NC4_hdf5_inq_var_filter_info(int ncid, int varid, unsigned int id, size_t* nparamsp, unsigned int* params)
 {
@@ -465,7 +540,16 @@ NC4_hdf5_filter_finalize(void)
     return NC_NOERR;
 }
 
-/* Test if filter available */
+/**
+ * @internal
+ * Test if a filter is available in HDF5.
+ *
+ * @param ncid File ID (ignored).
+ * @param id The HDF5 filter ID.
+ *
+ * @return ::NC_NOERR on success, error code otherwise.
+ * @author Dennis Heimbigner
+ */
 int
 NC4_hdf5_inq_filter_avail(int ncid, unsigned id)
 {