Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

adding zstd test #2965

Closed
wants to merge 22 commits into from
Closed
Show file tree
Hide file tree
Changes from 7 commits
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 6 additions & 2 deletions configure.ac
Original file line number Diff line number Diff line change
Expand Up @@ -1618,7 +1618,7 @@ if test "x$enable_hdf5" = xyes; then
# H5Pset_fapl_mpiposix and H5Pget_fapl_mpiposix have been removed since HDF5 1.8.12.
# Use H5Pset_fapl_mpio and H5Pget_fapl_mpio, instead.

AC_CHECK_FUNCS([H5Pget_fapl_mpio H5Pset_deflate H5Z_SZIP H5Pset_all_coll_metadata_ops H5Literate H5Literate2])
AC_CHECK_FUNCS([H5Pget_fapl_mpio H5Pset_deflate H5Z_SZIP H5Pset_all_coll_metadata_ops H5Literate H5Literate2 H5PLappend])

# Check to see if HDF5 library has collective metadata APIs, (HDF5 >= 1.10.0)
if test "x$ac_cv_func_H5Pset_all_coll_metadata_ops" = xyes; then
Expand Down Expand Up @@ -2094,9 +2094,12 @@ AC_SUBST(STD_FILTERS,[$std_filters])

# If user wants, then install selected standard filters
AC_MSG_CHECKING([whether and where we should install plugins])
AC_ARG_WITH([plugin-dir], [AS_HELP_STRING([--with-plugin-dir=<absolute directory>|no|--without-plugin-dir],
AC_ARG_WITH([plugin-dir], [AS_HELP_STRING([--with-plugin-dir=<absolute directory>|yes|no|--without-plugin-dir],
[Install selected standard filters in specified or default directory])],
[],[with_plugin_dir=no])
if test "x$have_zstd" = xyes ; then
with_plugin_dir=yes
fi
AC_MSG_RESULT([$with_plugin_dir])
if test "x$with_plugin_dir" = xno ; then # option missing|disabled
with_plugin_dir=no
Expand Down Expand Up @@ -2237,6 +2240,7 @@ AC_CONFIG_FILES(dap4_test/findtestserver4.c:ncdap_test/findtestserver.c.in)
AC_CONFIG_FILES(dap4_test/pingurl4.c:ncdap_test/pingurl.c)
AC_CONFIG_FILES([h5_test/run_par_tests.sh], [chmod ugo+x h5_test/run_par_tests.sh])
AC_CONFIG_FILES([nc_test4/run_par_test.sh], [chmod ugo+x nc_test4/run_par_test.sh])
AC_CONFIG_FILES([libsrc4/nc4dispatch.c], [chmod ugo+x nc_test4/run_par_test.sh])
AC_CONFIG_FILES([nc_perf/run_par_bm_test.sh], [chmod ugo+x nc_perf/run_par_bm_test.sh])
AC_CONFIG_FILES([nc_perf/run_gfs_test.sh], [chmod ugo+x nc_perf/run_gfs_test.sh])
AC_CONFIG_FILES(nczarr_test/timer_utils.h:unit_test/timer_utils.h)
Expand Down
7 changes: 7 additions & 0 deletions libsrc4/nc4dispatch.c → libsrc4/nc4dispatch.c.in
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,13 @@ NC4_initialize(void)
return ret;
#endif /* USE_UDF0 */

#ifdef HAVE_H5PLAPPEND
/* Add the place where plugins were installed to the plugin
* path. */
if ((ret = H5PLappend("@PLUGIN_INSTALL_DIR@")))
return ret;
#endif

#ifdef LOGGING
if(getenv(NCLOGLEVELENV) != NULL) {
char* slevel = getenv(NCLOGLEVELENV);
Expand Down
4 changes: 4 additions & 0 deletions nc_test4/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,10 @@ IF(HAVE_H5Z_SZIP)
ENDIF()
ENDIF()

IF(HAVE_ZSTD)
BUILD_BIN_TEST(tst_zstd)
ENDIF()


# Copy some test files from current source dir to out-of-tree build dir.
FILE(GLOB COPY_FILES ${CMAKE_CURRENT_SOURCE_DIR}/*.nc ${CMAKE_CURRENT_SOURCE_DIR}/ref_bzip2.c ${CMAKE_CURRENT_SOURCE_DIR}/*.sh ${CMAKE_CURRENT_SOURCE_DIR}/*.h5 ${CMAKE_CURRENT_SOURCE_DIR}/*.cdl)
Expand Down
6 changes: 6 additions & 0 deletions nc_test4/Makefile.am
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,12 @@ endif # NETCDF_ENABLE_PLUGINS
endif # USE_HDF5
endif # NETCDF_BUILD_UTILITIES

# Zstandard tests.
if HAVE_ZSTD
check_PROGRAMS += tst_zstd
TESTS += tst_zstd
endif # HAVE_ZSTD

# This are extra tests that will only be run if netcdf-4 is configured
# with --enable-parallel-tests.
if TEST_PARALLEL4
Expand Down
74 changes: 74 additions & 0 deletions nc_test4/tst_zstd.c
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
/* This is part of the netCDF package.
Copyright 2018 University Corporation for Atmospheric Research/Unidata
See COPYRIGHT file for conditions of use.

Test netcdf-4 variables.
Ed Hartnett, 8/7/24
*/

#include "nc_tests.h"
#include "err_macros.h"
#include "netcdf.h"

#define FILE_NAME "tst_zstd.nc"
#define VAR_FLOAT_NAME "GALE_data"
#define VAR_DOUBLE_NAME "ERICA_data"
#define NDIM2 2
#define DIM1_NAME "x"
#define DIM1_LEN 2
#define DIM2_NAME "y"
#define DIM2_LEN 3

int
main(int argc, char **argv)
{
int ncid, dimids[NDIM2];
int float_varid, double_varid;
float float_in[DIM1_LEN][DIM2_LEN], float_out[DIM1_LEN][DIM2_LEN] = {{-.1f, 9999.99f, 100.001f},{-.1f, 9999.99f, 100.001f}};
double double_in[DIM1_LEN][DIM2_LEN], double_out[DIM1_LEN][DIM2_LEN] = {{0.02, .1128, 1090.1},{0.02, .1128, 1090.1}};
int i, j;


printf("\n*** Testing netcdf-4 zstd compression.\n");

printf("*** testing netcdf-4 zstd with float...");
{
int nvars_in, varids_in[2];

/* Create a netcdf file with one dim and two vars. */
if (nc_create(FILE_NAME, NC_NETCDF4, &ncid)) ERR;
if (nc_def_dim(ncid, DIM1_NAME, DIM1_LEN, &dimids[0])) ERR;
if (nc_def_dim(ncid, DIM2_NAME, DIM2_LEN, &dimids[1])) ERR;
if (nc_def_var(ncid, VAR_FLOAT_NAME, NC_FLOAT, 2, dimids, &float_varid)) ERR;
if (nc_def_var(ncid, VAR_DOUBLE_NAME, NC_DOUBLE, 2, dimids, &double_varid)) ERR;
if (nc_def_var_zstandard(ncid, float_varid, 4)) ERR;
if (nc_def_var_zstandard(ncid, double_varid, 4)) ERR;
if (nc_put_var_float(ncid, float_varid, (float *)float_out)) ERR;
if (nc_put_var_double(ncid, double_varid, (double *)double_out)) ERR;
if (nc_close(ncid)) ERR;

/* Open the file and make sure nc_inq_varids yields correct
* result. */
if (nc_open(FILE_NAME, NC_NOWRITE, &ncid)) ERR;
if (nc_inq_varids(ncid, &nvars_in, varids_in)) ERR;
if (nvars_in != 2 || varids_in[0] != 0 || varids_in[1] != 1) ERR;
if (nc_get_var_float(ncid, float_varid, (float *)float_in)) ERR;
if (nc_get_var_double(ncid, double_varid, (double *)double_in)) ERR;
for (i = 0; i < DIM1_LEN; i++)
{
for (j = 0; j < DIM2_LEN; j++)
{
if (float_in[i][j] != float_out[i][j]) ERR;
if (double_in[i][j] != double_out[i][j]) ERR;
}
}


if (nc_close(ncid)) ERR;
}

SUMMARIZE_ERR;
SUMMARIZE_ERR;

FINAL_RESULTS;
}
2 changes: 1 addition & 1 deletion nczarr_test/run_interop.sh
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ testcasezip() {
}

testcases3() {
set -x
#set -x
echo " o Running S3 Testcase: $1 $2"
zext=s3
base=$1
Expand Down
Loading