diff --git a/.github/workflows/run_tests_osx.yml b/.github/workflows/run_tests_osx.yml index fc73521c85..11864a8012 100644 --- a/.github/workflows/run_tests_osx.yml +++ b/.github/workflows/run_tests_osx.yml @@ -7,7 +7,7 @@ name: Run macOS-based netCDF Tests -on: [pull_request, workflow_dispatch] +on: [pull_request,workflow_dispatch] jobs: diff --git a/.github/workflows/run_tests_ubuntu.yml b/.github/workflows/run_tests_ubuntu.yml index 618c13ed2f..9df8b7048e 100644 --- a/.github/workflows/run_tests_ubuntu.yml +++ b/.github/workflows/run_tests_ubuntu.yml @@ -1,5 +1,5 @@ ### -# Build hdf4, hdf5 dependencies and cache them in a combined directory. +# Build hdf5 dependencies and cache them in a combined directory. ### name: Run Ubuntu/Linux netCDF Tests @@ -25,7 +25,7 @@ jobs: run: sudo apt update && sudo apt install -y libaec-dev zlib1g-dev automake autoconf libcurl4-openssl-dev libjpeg-dev wget curl bzip2 m4 flex bison cmake libzip-dev doxygen ### - # Installing libhdf4 and libhdf5 + # Installing libhdf5 ### - name: Cache libhdf5-${{ matrix.hdf5 }} id: cache-hdf5 @@ -39,13 +39,15 @@ jobs: if: steps.cache-hdf5.outputs.cache-hit != 'true' run: | set -x + wget https://support.hdfgroup.org/ftp/HDF/releases/HDF4.2.15/src/hdf-4.2.15.tar.bz2 tar -jxf hdf-4.2.15.tar.bz2 pushd hdf-4.2.15 - ./configure --prefix=${HOME}/environments/${{ matrix.hdf5 }} --disable-static --enable-shared --disable-fortran --disable-netcdf --with-szlib + ./configure --prefix=${HOME}/environments/${{ matrix.hdf5 }} --disable-static --enable-shared --disable-fortran --disable-netcdf --with-szlib --enable-hdf4-xdr make -j make install -j popd + wget https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-$(echo ${{ matrix.hdf5 }} | cut -d. -f 1,2)/hdf5-${{ matrix.hdf5 }}/src/hdf5-${{ matrix.hdf5 }}.tar.bz2 tar -jxf hdf5-${{ matrix.hdf5 }}.tar.bz2 pushd hdf5-${{ matrix.hdf5 }} @@ -72,7 +74,7 @@ jobs: run: sudo apt update && sudo apt install -y libaec-dev zlib1g-dev automake autoconf libcurl4-openssl-dev libjpeg-dev wget curl bzip2 m4 flex bison cmake libzip-dev mpich libmpich-dev ### - # Installing libhdf4 and libhdf5 + # Installing libhdf5 ### - name: Cache libhdf5-parallel-${{ matrix.hdf5 }} id: cache-hdf5 @@ -86,13 +88,15 @@ jobs: if: steps.cache-hdf5.outputs.cache-hit != 'true' run: | set -x + wget https://support.hdfgroup.org/ftp/HDF/releases/HDF4.2.15/src/hdf-4.2.15.tar.bz2 tar -jxf hdf-4.2.15.tar.bz2 pushd hdf-4.2.15 - CC=mpicc ./configure --prefix=${HOME}/environments/${{ matrix.hdf5 }} --disable-static --enable-shared --disable-fortran --disable-netcdf --with-szlib --enable-parallel + CC=mpicc ./configure --prefix=${HOME}/environments/${{ matrix.hdf5 }} --disable-static --enable-shared --disable-fortran --disable-netcdf --with-szlib --enable-parallel --enable-hdf4-xdr make -j make install -j popd + wget https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-$(echo ${{ matrix.hdf5 }} | cut -d. -f 1,2)/hdf5-${{ matrix.hdf5 }}/src/hdf5-${{ matrix.hdf5 }}.tar.bz2 tar -jxf hdf5-${{ matrix.hdf5 }}.tar.bz2 pushd hdf5-${{ matrix.hdf5 }} @@ -164,7 +168,7 @@ jobs: - name: Configure shell: bash -l {0} - run: CFLAGS=${CFLAGS} LDFLAGS=${LDFLAGS} LD_LIBRARY_PATH=${LD_LIBRARY_PATH} ./configure --enable-hdf4 --enable-hdf5 --enable-dap --disable-dap-remote-tests --enable-doxygen --enable-external-server-tests + run: CFLAGS=${CFLAGS} LDFLAGS=${LDFLAGS} LD_LIBRARY_PATH=${LD_LIBRARY_PATH} ./configure --enable-hdf5 --enable-dap --disable-dap-remote-tests --enable-doxygen --enable-external-server-tests if: ${{ success() }} - name: Look at config.log if error @@ -240,7 +244,7 @@ jobs: - name: Configure shell: bash -l {0} - run: CFLAGS=${CFLAGS} LDFLAGS=${LDFLAGS} LD_LIBRARY_PATH=${LD_LIBRARY_PATH} CC=mpicc ./configure --enable-hdf4 --enable-hdf5 --enable-dap --disable-dap-remote-tests --enable-parallel-tests --enable-pnetcdf + run: CFLAGS=${CFLAGS} LDFLAGS=${LDFLAGS} LD_LIBRARY_PATH=${LD_LIBRARY_PATH} CC=mpicc ./configure --enable-hdf5 --enable-dap --disable-dap-remote-tests --enable-parallel-tests --enable-pnetcdf if: ${{ success() }} - name: Look at config.log if error @@ -322,7 +326,7 @@ jobs: run: | mkdir build cd build - LD_LIBRARY_PATH=${LD_LIBRARY_PATH} cmake .. -DENABLE_HDF4=TRUE -DCMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH} -DENABLE_DAP=TRUE -DENABLE_HDF5=TRUE -DENABLE_NCZARR=TRUE -D ENABLE_DAP_LONG_TESTS=TRUE + LD_LIBRARY_PATH=${LD_LIBRARY_PATH} cmake .. -DCMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH} -DENABLE_DAP=TRUE -DENABLE_HDF5=TRUE -DENABLE_NCZARR=TRUE -D ENABLE_DAP_LONG_TESTS=TRUE - name: Print Summary shell: bash -l {0} @@ -402,7 +406,7 @@ jobs: run: | mkdir build cd build - LD_LIBRARY_PATH=${LD_LIBRARY_PATH} cmake .. -DCMAKE_C_COMPILER=mpicc -DENABLE_HDF4=TRUE -DCMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH} -DENABLE_DAP=TRUE -DENABLE_HDF5=TRUE -DENABLE_NCZARR=TRUE -D ENABLE_DAP_LONG_TESTS=TRUE -DENABLE_PNETCDF=TRUE + LD_LIBRARY_PATH=${LD_LIBRARY_PATH} cmake .. -DCMAKE_C_COMPILER=mpicc -DCMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH} -DENABLE_DAP=TRUE -DENABLE_HDF5=TRUE -DENABLE_NCZARR=TRUE -D ENABLE_DAP_LONG_TESTS=TRUE -DENABLE_PNETCDF=TRUE - name: Print Summary shell: bash -l {0} @@ -458,11 +462,9 @@ jobs: - run: echo "LDFLAGS=-L${HOME}/environments/${{ matrix.hdf5 }}/lib" >> $GITHUB_ENV - run: echo "LD_LIBRARY_PATH=${HOME}/environments/${{ matrix.hdf5 }}/lib" >> $GITHUB_ENV - run: | - echo "ENABLE_HDF4=--disable-hdf4" >> $GITHUB_ENV echo "ENABLE_HDF5=--disable-hdf5" >> $GITHUB_ENV if: matrix.use_nc4 == 'nc3' - run: | - echo "ENABLE_HDF4=--enable-hdf4" >> $GITHUB_ENV echo "ENABLE_HDF5=--enable-hdf5" >> $GITHUB_ENV if: matrix.use_nc4 == 'nc4' - run: echo "ENABLE_DAP=--disable-dap" >> $GITHUB_ENV @@ -499,7 +501,7 @@ jobs: - name: Configure shell: bash -l {0} - run: CFLAGS=${CFLAGS} LDFLAGS=${LDFLAGS} LD_LIBRARY_PATH=${LD_LIBRARY_PATH} ./configure ${ENABLE_HDF4} ${ENABLE_HDF5} ${ENABLE_DAP} ${ENABLE_NCZARR} + run: CFLAGS=${CFLAGS} LDFLAGS=${LDFLAGS} LD_LIBRARY_PATH=${LD_LIBRARY_PATH} ./configure ${ENABLE_HDF5} ${ENABLE_DAP} ${ENABLE_NCZARR} if: ${{ success() }} - name: Look at config.log if error @@ -526,18 +528,6 @@ jobs: run: CFLAGS=${CFLAGS} LDFLAGS=${LDFLAGS} LD_LIBRARY_PATH=${LD_LIBRARY_PATH} make check -j if: ${{ success() }} - # - name: Make Distcheck - # shell: bash -l {0} - # run: CFLAGS=${CFLAGS} LDFLAGS=${LDFLAGS} LD_LIBRARY_PATH=${LD_LIBRARY_PATH} DISTCHECK_CONFIGURE_FLAGS="${ENABLE_HDF4} ${ENABLE_HDF5} ${ENABLE_DAP} ${ENABLE_NCZARR}" make distcheck - # if: ${{ success() }} - - #- name: Start SSH Debug - # uses: luchihoratiu/debug-via-ssh@main - # with: - # NGROK_AUTH_TOKEN: ${{ secrets.NGROK_AUTH_TOKEN }} - # SSH_PASS: ${{ secrets.SSH_PASS }} - # if: ${{ failure() }} - nc-cmake: needs: [ nc-cmake-tests-oneoff-serial, nc-ac-tests-oneoff-serial, nc-cmake-tests-oneoff-parallel, nc-ac-tests-oneoff-parallel ] @@ -564,11 +554,9 @@ jobs: - run: echo "CMAKE_PREFIX_PATH=${HOME}/environments/${{ matrix.hdf5 }}/" >> $GITHUB_ENV - run: echo "LD_LIBRARY_PATH=${HOME}/environments/${{ matrix.hdf5 }}/lib" >> $GITHUB_ENV - run: | - echo "ENABLE_HDF4=OFF" >> $GITHUB_ENV echo "ENABLE_HDF5=OFF" >> $GITHUB_ENV if: matrix.use_nc4 == 'nc3' - run: | - echo "ENABLE_HDF4=ON" >> $GITHUB_ENV echo "ENABLE_HDF5=ON" >> $GITHUB_ENV if: matrix.use_nc4 == 'nc4' - run: echo "ENABLE_DAP=OFF" >> $GITHUB_ENV @@ -605,7 +593,7 @@ jobs: run: | mkdir build cd build - LD_LIBRARY_PATH=${LD_LIBRARY_PATH} cmake .. -DENABLE_HDF4=${ENABLE_HDF4} -DCMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH} -DENABLE_DAP=${ENABLE_DAP} -DENABLE_HDF5=${ENABLE_HDF5} -DENABLE_NCZARR=${ENABLE_NCZARR} + LD_LIBRARY_PATH=${LD_LIBRARY_PATH} cmake .. -DCMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH} -DENABLE_DAP=${ENABLE_DAP} -DENABLE_HDF5=${ENABLE_HDF5} -DENABLE_NCZARR=${ENABLE_NCZARR} - name: Print Summary shell: bash -l {0} diff --git a/.github/workflows/run_tests_win_cygwin.yml b/.github/workflows/run_tests_win_cygwin.yml index 361e6265b7..d3e4f5c880 100644 --- a/.github/workflows/run_tests_win_cygwin.yml +++ b/.github/workflows/run_tests_win_cygwin.yml @@ -1,6 +1,6 @@ name: Run Cygwin-based tests -on: [pull_request, workflow_dispatch] +on: [pull_request,workflow_dispatch] env: SHELLOPTS: igncr @@ -29,7 +29,7 @@ jobs: git automake libtool autoconf2.5 make libhdf5-devel libhdf4-devel zipinfo libxml2-devel perl zlib-devel libzstd-devel libbz2-devel libaec-devel libzip-devel - libdeflate-devel gcc-core + libdeflate-devel gcc-core libcurl-devel libiconv-devel - name: (Autotools) Run autoconf and friends run: | diff --git a/.github/workflows/run_tests_win_mingw.yml b/.github/workflows/run_tests_win_mingw.yml index 5ff67e1a16..d872128597 100644 --- a/.github/workflows/run_tests_win_mingw.yml +++ b/.github/workflows/run_tests_win_mingw.yml @@ -9,7 +9,7 @@ name: Run MSYS2, MinGW64-based Tests env: CPPFLAGS: "-D_BSD_SOURCE" -on: [pull_request, workflow_dispatch] +on: [pull_request,workflow_dispatch] jobs: diff --git a/CMakeLists.txt b/CMakeLists.txt index 400a54dc22..9b0573111b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -519,8 +519,8 @@ OPTION(ENABLE_CDF5 "Enable CDF5 support" ON) # Netcdf-4 support (i.e. libsrc4) is required by more than just HDF5 (e.g. NCZarr) # So depending on what above formats are enabled, enable netcdf-4 if(ENABLE_HDF5 OR ENABLE_HDF4 OR ENABLE_NCZARR) -SET(ENABLE_NETCDF_4 ON CACHE BOOL "Enable netCDF-4 API" FORCE) -SET(ENABLE_NETCDF4 ON CACHE BOOL "Enable netCDF4 Alias" FORCE) + SET(ENABLE_NETCDF_4 ON CACHE BOOL "Enable netCDF-4 API" FORCE) + SET(ENABLE_NETCDF4 ON CACHE BOOL "Enable netCDF4 Alias" FORCE) endif() IF(ENABLE_HDF4) @@ -1062,11 +1062,14 @@ ENDIF() IF(ENABLE_DAP) SET(USE_DAP ON CACHE BOOL "") SET(ENABLE_DAP2 ON CACHE BOOL "") - SET(ENABLE_DAP4 OFF CACHE BOOL "") - - IF(NOT ENABLE_HDF5) + + IF(ENABLE_HDF5) + MESSAGE(STATUS "Enabling DAP4") + SET(ENABLE_DAP4 ON CACHE BOOL "") + ELSE() + MESSAGE(STATUS "Disabling DAP4") SET(ENABLE_DAP4 OFF CACHE BOOL "") - ENDIF(NOT ENABLE_HDF5) + ENDIF(ENABLE_HDF5) ELSE() SET(ENABLE_DAP2 OFF CACHE BOOL "") @@ -1074,7 +1077,7 @@ ELSE() ENDIF() # Option to support byte-range reading of remote datasets -OPTION(ENABLE_BYTERANGE "Enable byte-range access to remote datasets.." OFF) +OPTION(ENABLE_BYTERANGE "Enable byte-range access to remote datasets.." ON) # Check for the math library so it can be explicitly linked. IF(NOT WIN32) @@ -1309,7 +1312,7 @@ ENDIF() IF(NOT ENABLE_S3_SDK) IF(ENABLE_NCZARR_S3 OR ENABLE_NCZARR_S3_TESTS) - message(FATAL_ERROR "S3 support library not found; please specify option DENABLE_NCZARR_S3=NO") + message(FATAL_ERROR "S3 support library not found; please specify option -DENABLE_NCZARR_S3=NO") SET(ENABLE_NCZARR_S3 OFF CACHE BOOL "NCZARR S3 support" FORCE) SET(ENABLE_NCZARR_S3_TESTS OFF CACHE BOOL "S3 tests" FORCE) ENDIF() @@ -2535,6 +2538,7 @@ is_enabled(ENABLE_ZERO_LENGTH_COORD_BOUND RELAX_COORD_BOUND) is_enabled(USE_CDF5 HAS_CDF5) is_enabled(ENABLE_ERANGE_FILL HAS_ERANGE_FILL) is_enabled(HDF5_HAS_PAR_FILTERS HAS_PAR_FILTERS) +is_enabled(ENABLE_NCZARR_S3 HAS_NCZARR_S3) is_enabled(ENABLE_NCZARR HAS_NCZARR) is_enabled(ENABLE_NCZARR_S3_TESTS DO_NCZARR_S3_TESTS) is_enabled(ENABLE_MULTIFILTERS HAS_MULTIFILTERS) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index f810060538..662e800b4a 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -7,7 +7,7 @@ This file contains a high-level description of this package's evolution. Release ## 4.9.1 - T.B.D. -### 4.9.1 - Release Candidate 2 - TBD +### 4.9.1 - Release Candidate 2 - November 21, 2022 #### Known Issues @@ -18,6 +18,10 @@ This file contains a high-level description of this package's evolution. Release * [Bug Fix] Fix a race condition when testing missing filters. See [Github #2557](https://github.com/Unidata/netcdf-c/pull/2557). * [Bug Fix] Fix some race conditions due to use of a common file in multiple shell scripts . See [Github #2552](https://github.com/Unidata/netcdf-c/pull/2552). + + +### 4.9.1 - Release Candidate 1 - October 24, 2022 + * [Enhancement][Documentation] Add Plugins Quick Start Guide. See [GitHub #2524](https://github.com/Unidata/netcdf-c/pull/2524) for more information. * [Enhancement] Add new entries in `netcdf_meta.h`, `NC_HAS_BLOSC` and `NC_HAS_BZ2`. See [Github #2511](https://github.com/Unidata/netcdf-c/issues/2511) and [Github #2512](https://github.com/Unidata/netcdf-c/issues/2512) for more information. * [Enhancement] Add new options to `nc-config`: `--has-multifilters`, `--has-stdfilters`, `--has-quantize`, `--plugindir`. See [Github #2509](https://github.com/Unidata/netcdf-c/pull/2509) for more information. diff --git a/configure.ac b/configure.ac index 3a38069449..81a75ea319 100644 --- a/configure.ac +++ b/configure.ac @@ -97,7 +97,7 @@ AC_CONFIG_LINKS([nc_test4/ref_hdf5_compat3.nc:nc_test4/ref_hdf5_compat3.nc]) AC_CONFIG_LINKS([hdf4_test/ref_chunked.hdf4:hdf4_test/ref_chunked.hdf4]) AC_CONFIG_LINKS([hdf4_test/ref_contiguous.hdf4:hdf4_test/ref_contiguous.hdf4]) AM_INIT_AUTOMAKE([foreign dist-zip subdir-objects]) - +AM_MAINTAINER_MODE() # Check for the existence of this file before proceeding. AC_CONFIG_SRCDIR([include/netcdf.h]) @@ -1282,9 +1282,9 @@ fi # Does the user want to allow reading of remote data via range headers? AC_MSG_CHECKING([whether byte range support is enabled]) AC_ARG_ENABLE([byterange], - [AS_HELP_STRING([--enable-byterange], + [AS_HELP_STRING([--disable-byterange], [allow byte-range I/O])]) -test "x$enable_byterange" = xyes || enable_byterange=no +test "x$enable_byterange" = xno || enable_byterange=yes AC_MSG_RESULT($enable_byterange) # Need curl for byte ranges if test "x$found_curl" = xno && test "x$enable_byterange" = xyes ; then @@ -1929,6 +1929,7 @@ AC_SUBST(HAS_ERANGE_FILL,[$enable_erange_fill]) AC_SUBST(HAS_BYTERANGE,[$enable_byterange]) AC_SUBST(RELAX_COORD_BOUND,[yes]) AC_SUBST([HAS_PAR_FILTERS], [$hdf5_supports_par_filters]) +AC_SUBST(HAS_NCZARR_S3,[$enable_nczarr_s3]) AC_SUBST(HAS_NCZARR,[$enable_nczarr]) AC_SUBST(DO_NCZARR_S3_TESTS,[$enable_nczarr_s3_tests]) AC_SUBST(HAS_MULTIFILTERS,[$has_multifilters]) @@ -2064,12 +2065,12 @@ AX_SET_META([NC_HAS_SZIP],[$enable_hdf5_szip],[yes]) AX_SET_META([NC_HAS_ZSTD],[$have_zstd],[yes]) AX_SET_META([NC_HAS_BLOSC],[$have_blosc],[yes]) AX_SET_META([NC_HAS_BZ2],[$have_bz2],[yes]) + # This is the version of the dispatch table. If the dispatch table is # changed, this should be incremented, so that user-defined format # applications like PIO can determine whether they have an appropriate # dispatch table to submit. If this is changed, make sure the value in # CMakeLists.txt also changes to match. - AC_SUBST([NC_DISPATCH_VERSION], [5]) AC_DEFINE_UNQUOTED([NC_DISPATCH_VERSION], [${NC_DISPATCH_VERSION}], [Dispatch table version.]) diff --git a/docs/dispatch.md b/docs/dispatch.md index 55346503f8..16689c629b 100644 --- a/docs/dispatch.md +++ b/docs/dispatch.md @@ -499,6 +499,31 @@ The code in *hdf4var.c* does an *nc_get_vara()* on the HDF4 SD dataset. This is all that is needed for all the nc_get_* functions to work. +# Appendix A. Changing NC_DISPATCH_VERSION + +When new entries are added to the *struct NC_Dispatch* type `located in include/netcdf_dispatch.h.in` it is necessary to do two things. + +1. Bump the NC_DISPATCH_VERSION number +2. Modify the existing dispatch tables to include the new entries. +It if often the case that the new entries do not mean anything for +a given dispatch table. In that case, the new entries may be set to +some variant of *NC_RO_XXX* or *NC_NOTNC4_XXX* *NC_NOTNC3_XXX*. + +Modifying the dispatch version requires two steps: +1. Modify the version number in *netcdf-c/configure.ac*, and +2. Modify the version number in *netcdf-c/CMakeLists.txt*. + +The two should agree in value. + +### NC_DISPATCH_VERSION Incompatibility + +When dynamically adding a dispatch table +-- in nc_def_user_format (see libdispatch/dfile.c) -- +the version of the new table is compared with that of the built-in +NC_DISPATCH_VERSION; if they differ, then an error is returned from +that function. + + # Point of Contact {#dispatch_poc} *Author*: Dennis Heimbigner
diff --git a/docs/static-pages/orgs.html b/docs/static-pages/orgs.html new file mode 100644 index 0000000000..2e5c84df66 --- /dev/null +++ b/docs/static-pages/orgs.html @@ -0,0 +1,417 @@ + + + + +Organizations in which NetCDF is Used + + + + + + + + + + + + + +

Organizations in which NetCDF is Used

+The following list of organizations was created by sorting the organizational +affiliations of authors of questions or comments about netCDF sent to support@unidata.ucar.edu. + + + + + + \ No newline at end of file diff --git a/docs/static-pages/standards.html b/docs/static-pages/standards.html new file mode 100644 index 0000000000..b0ea4edfa9 --- /dev/null +++ b/docs/static-pages/standards.html @@ -0,0 +1,72 @@ + + + + +NetCDF Standards +

Status of standards body endorsements of netCDF and related conventions

+ +

The netCDF format has been endorsed by several standards bodies: +

+ + +
+
+ + + \ No newline at end of file diff --git a/libdispatch/dfile.c b/libdispatch/dfile.c index ae756a8f27..a53be47893 100644 --- a/libdispatch/dfile.c +++ b/libdispatch/dfile.c @@ -1839,19 +1839,16 @@ NC_create(const char *path0, int cmode, size_t initialsz, TRACE(nc_create); if(path0 == NULL) - return NC_EINVAL; + {stat = NC_EINVAL; goto done;} /* Check mode flag for sanity. */ - if ((stat = check_create_mode(cmode))) - return stat; + if ((stat = check_create_mode(cmode))) goto done; /* Initialize the library. The available dispatch tables * will depend on how netCDF was built * (with/without netCDF-4, DAP, CDMREMOTE). */ - if(!NC_initialized) - { - if ((stat = nc_initialize())) - return stat; + if(!NC_initialized) { + if ((stat = nc_initialize())) goto done; } { @@ -1863,10 +1860,7 @@ NC_create(const char *path0, int cmode, size_t initialsz, memset(&model,0,sizeof(model)); newpath = NULL; - if((stat = NC_infermodel(path,&cmode,1,useparallel,NULL,&model,&newpath))) { - nullfree(newpath); - goto done; - } + if((stat = NC_infermodel(path,&cmode,1,useparallel,NULL,&model,&newpath))) goto done; if(newpath) { nullfree(path); path = newpath; @@ -1918,7 +1912,7 @@ NC_create(const char *path0, int cmode, size_t initialsz, dispatcher = NC3_dispatch_table; break; default: - return NC_ENOTNC; + {stat = NC_ENOTNC; goto done;} } /* Create the NC* instance and insert its dispatcher and model */ @@ -1937,6 +1931,7 @@ NC_create(const char *path0, int cmode, size_t initialsz, } done: nullfree(path); + nullfree(newpath); return stat; } @@ -1980,12 +1975,12 @@ NC_open(const char *path0, int omode, int basepe, size_t *chunksizehintp, TRACE(nc_open); if(!NC_initialized) { stat = nc_initialize(); - if(stat) return stat; + if(stat) goto done; } /* Check inputs. */ if (!path0) - return NC_EINVAL; + {stat = NC_EINVAL; goto done;} /* Capture the inmemory related flags */ mmap = ((omode & NC_MMAP) == NC_MMAP); diff --git a/libdispatch/dinfermodel.c b/libdispatch/dinfermodel.c index 74fd55a4fc..ff3e8e9770 100644 --- a/libdispatch/dinfermodel.c +++ b/libdispatch/dinfermodel.c @@ -143,7 +143,15 @@ static const struct MACRODEF { {NULL,NULL,{NULL}} }; -/* Mode inferences: if mode contains key, then add the inference and infer again */ +/* +Mode inferences: if mode contains key value, then add the inferred value; +Warning: be careful how this list is constructed to avoid infinite inferences. +In order to (mostly) avoid that consequence, any attempt to +infer a value that is already present will be ignored. +This effectively means that the inference graph +must be a DAG and may not have cycles. +You have been warned. +*/ static const struct MODEINFER { char* key; char* inference; @@ -151,6 +159,7 @@ static const struct MODEINFER { {"zarr","nczarr"}, {"xarray","zarr"}, {"noxarray","nczarr"}, +{"noxarray","zarr"}, {NULL,NULL} }; @@ -202,6 +211,7 @@ static int processmacros(NClist** fraglistp); static char* envvlist2string(NClist* pairs, const char*); static void set_default_mode(int* cmodep); static int parseonchar(const char* s, int ch, NClist* segments); +static int mergelist(NClist** valuesp); static int openmagic(struct MagicFile* file); static int readmagic(struct MagicFile* file, long pos, char* magic); @@ -217,8 +227,9 @@ static int parsepair(const char* pair, char** keyp, char** valuep); static NClist* parsemode(const char* modeval); static const char* getmodekey(const NClist* envv); static int replacemode(NClist* envv, const char* newval); -static int inferone(const char* mode, NClist* newmodes); +static void infernext(NClist* current, NClist* next); static int negateone(const char* mode, NClist* modes); +static void cleanstringlist(NClist* strs, int caseinsensitive); /* If the path looks like a URL, then parse it, reformat it. @@ -416,28 +427,6 @@ envvlist2string(NClist* envv, const char* delim) return result; } -/* Convert a list into a comma'd string */ -static char* -list2string(NClist* list) -{ - int i; - NCbytes* buf = NULL; - char* result = NULL; - - if(list == NULL || nclistlength(list)==0) return strdup(""); - buf = ncbytesnew(); - for(i=0;i 0) ncbytescat(buf,","); - ncbytescat(buf,m); - } - result = ncbytesextract(buf); - ncbytesfree(buf); - if(result == NULL) result = strdup(""); - return result; -} - /* Given a mode= argument, fill in the impl */ static int processmodearg(const char* arg, NCmodel* model) @@ -504,9 +493,10 @@ processinferences(NClist* fraglenv) { int stat = NC_NOERR; const char* modeval = NULL; - NClist* modes = NULL; NClist* newmodes = nclistnew(); - int i,inferred = 0; + NClist* currentmodes = NULL; + NClist* nextmodes = nclistnew(); + int i; char* newmodeval = NULL; if(fraglenv == NULL || nclistlength(fraglenv) == 0) goto done; @@ -515,22 +505,53 @@ processinferences(NClist* fraglenv) if((modeval = getmodekey(fraglenv))==NULL) goto done; /* Get the mode as list */ - modes = parsemode(modeval); - - /* Repeatedly walk the mode list until no more new positive inferences */ - do { - for(i=0;ikey;tests++) { - if(strcasecmp(tests->key,mode)==0) { - /* Append the inferred mode; dups removed later */ - nclistpush(newmodes,strdup(tests->inference)); - changed = 1; + int i; + for(i=0;ikey;tests++) { + if(strcasecmp(tests->key,cur)==0) { + /* Append the inferred mode unless dup */ + if(!nclistmatch(next,tests->inference,1)) + nclistpush(next,strdup(tests->inference)); + } } } - return changed; } +/* +Given a list of strings, remove nulls and duplicates +*/ static int -mergekey(NClist** valuesp) +mergelist(NClist** valuesp) { int i,j; int stat = NC_NOERR; @@ -686,12 +714,12 @@ cleanfragments(NClist** fraglenvp) /* collect all unique keys */ collectallkeys(fraglenv,allkeys); - /* Collect all values for same key across all fragments */ + /* Collect all values for same key across all fragment pairs */ for(i=0;i 0) ncbytescat(buf,","); + ncbytescat(buf,m); + } + result = ncbytesextract(buf); + ncbytesfree(buf); + if(result == NULL) result = strdup(""); + return result; +} + +#if 0 +/* Given a comma separated string, remove duplicates; mostly used to cleanup mode list */ +static char* +cleancommalist(const char* commalist, int caseinsensitive) +{ + NClist* tmp = nclistnew(); + char* newlist = NULL; + if(commalist == NULL || strlen(commalist)==0) return nulldup(commalist); + (void)parseonchar(commalist,',',tmp);/* split on commas */ + cleanstringlist(tmp,caseinsensitive); + newlist = list2string(tmp); + nclistfreeall(tmp); + return newlist; +} +#endif + +/* Given a list of strings, remove nulls and duplicated */ +static void +cleanstringlist(NClist* strs, int caseinsensitive) +{ + int i,j; + if(nclistlength(strs) == 0) return; + /* Remove nulls */ + for(i=nclistlength(strs)-1;i>=0;i--) { + if(nclistget(strs,i)==NULL) nclistremove(strs,i); + } + /* Remove duplicates*/ + for(i=0;ii;j--) { + int match; + const char* candidate = nclistget(strs,j); + if(caseinsensitive) + match = (strcasecmp(value,candidate) == 0); + else + match = (strcmp(value,candidate) == 0); + if(match) {char* dup = nclistremove(strs,j); nullfree(dup);} + } + } +} + + /**************************************************/ /** * @internal Given an existing file, figure out its format and return @@ -1502,8 +1595,10 @@ printlist(NClist* list, const char* tag) { int i; fprintf(stderr,"%s:",tag); - for(i=0;ilength == 0) return ncbytesfail(); diff --git a/libdispatch/nclist.c b/libdispatch/nclist.c index 49f0dded45..b5f815864c 100644 --- a/libdispatch/nclist.c +++ b/libdispatch/nclist.c @@ -183,6 +183,7 @@ nclistremove(NClist* l, size_t i) return elem; } +/* Match on == */ int nclistcontains(NClist* l, void* elem) { @@ -193,7 +194,7 @@ nclistcontains(NClist* l, void* elem) return 0; } -/* Return 1/0 */ +/* Match on str(case)cmp */ int nclistmatch(NClist* l, const char* elem, int casesensitive) { @@ -230,7 +231,6 @@ nclistelemremove(NClist* l, void* elem) return found; } - /* Extends nclist to include a unique operator which remove duplicate values; NULL values removed return value is always 1. diff --git a/libnczarr/zsync.c b/libnczarr/zsync.c index b3a93ee767..f237cd2b61 100644 --- a/libnczarr/zsync.c +++ b/libnczarr/zsync.c @@ -1429,6 +1429,7 @@ define_vars(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NClist* varnames) char* varpath = NULL; char* key = NULL; NCZ_FILE_INFO_T* zinfo = NULL; + NC_VAR_INFO_T* var = NULL; NCZ_VAR_INFO_T* zvar = NULL; NCZMAP* map = NULL; NCjson* jvar = NULL; @@ -1460,7 +1461,6 @@ define_vars(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NClist* varnames) /* Load each var in turn */ for(i = 0; i < nclistlength(varnames); i++) { - NC_VAR_INFO_T* var; const char* varname = nclistget(varnames,i); if((stat = nc4_var_list_add2(grp, varname, &var))) goto done; @@ -1477,10 +1477,6 @@ define_vars(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NClist* varnames) /* Indicate we do not have quantizer yet */ var->quantize_mode = -1; - /* Set filter list */ - assert(var->filters == NULL); - var->filters = (void*)nclistnew(); - /* Construct var path */ if((stat = NCZ_varkey(var,&varpath))) goto done; @@ -1697,9 +1693,9 @@ define_vars(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NClist* varnames) object MUST contain a "id" key identifying the codec to be used. */ /* Do filters key before compressor key so final filter chain is in correct order */ { +#ifdef ENABLE_NCZARR_FILTERS if(var->filters == NULL) var->filters = (void*)nclistnew(); if(zvar->incompletefilters == NULL) zvar->incompletefilters = (void*)nclistnew(); -#ifdef ENABLE_NCZARR_FILTERS { int k; chainindex = 0; /* track location of filter in the chain */ if((stat = NCZ_filter_initialize())) goto done; @@ -1722,8 +1718,8 @@ define_vars(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NClist* varnames) /* From V2 Spec: A JSON object identifying the primary compression codec and providing configuration parameters, or ``null`` if no compressor is to be used. */ { - if(var->filters == NULL) var->filters = (void*)nclistnew(); #ifdef ENABLE_NCZARR_FILTERS + if(var->filters == NULL) var->filters = (void*)nclistnew(); if((stat = NCZ_filter_initialize())) goto done; if((stat = NCJdictget(jvar,"compressor",&jfilter))) goto done; if(jfilter != NULL && NCJsort(jfilter) != NCJ_NULL) { @@ -1752,6 +1748,7 @@ define_vars(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NClist* varnames) nullfree(shapes); shapes = NULL; if(formatv1) {NCJreclaim(jncvar); jncvar = NULL;} NCJreclaim(jvar); jvar = NULL; + var = NULL; } done: diff --git a/libnczarr/zvar.c b/libnczarr/zvar.c index 12014bae87..2a98646e36 100644 --- a/libnczarr/zvar.c +++ b/libnczarr/zvar.c @@ -391,9 +391,11 @@ NCZ_def_var(int ncid, const char *name, nc_type xtype, int ndims, var->meta_read = NC_TRUE; var->atts_read = NC_TRUE; +#ifdef ENABLE_NCZARR_FILTERS /* Set the filter list */ assert(var->filters == NULL); var->filters = (void*)nclistnew(); +#endif /* Point to the type, and increment its ref. count */ var->type_info = type; @@ -558,10 +560,12 @@ ncz_def_var_extra(int ncid, int varid, int *shuffle, int *unused1, /* Can't turn on parallel and deflate/fletcher32/szip/shuffle * before HDF5 1.10.3. */ +#ifdef ENABLE_NCZARR_FILTERS #ifndef HDF5_SUPPORTS_PAR_FILTERS if (h5->parallel == NC_TRUE) if (nclistlength(((NClist*)var->filters)) > 0 || fletcher32 || shuffle) {retval = NC_EINVAL; goto done;} +#endif #endif /* If the HDF5 dataset has already been created, then it is too @@ -628,8 +632,10 @@ ncz_def_var_extra(int ncid, int varid, int *shuffle, int *unused1, * no filters in use for this data. */ if (storage != NC_CHUNKED) { +#ifdef NCZARR_FILTERS if (nclistlength(((NClist*)var->filters)) > 0) {retval = NC_EINVAL; goto done;} +#endif for (d = 0; d < var->ndims; d++) if (var->dim[d]->unlimited) {retval = NC_EINVAL; goto done;} diff --git a/libnetcdf.settings.in b/libnetcdf.settings.in index 6dea12d5c7..b25d3b603a 100644 --- a/libnetcdf.settings.in +++ b/libnetcdf.settings.in @@ -27,22 +27,27 @@ XML Parser: @XMLPARSER@ # Features -------- +Benchmarks: @HAS_BENCHMARKS@ NetCDF-2 API: @HAS_NC2@ HDF4 Support: @HAS_HDF4@ HDF5 Support: @HAS_HDF5@ NetCDF-4 API: @HAS_NC4@ +CDF5 Support: @HAS_CDF5@ NC-4 Parallel Support: @HAS_PARALLEL4@ PnetCDF Support: @HAS_PNETCDF@ + DAP2 Support: @HAS_DAP2@ DAP4 Support: @HAS_DAP4@ Byte-Range Support: @HAS_BYTERANGE@ +NCZarr Support: @HAS_NCZARR@ +NCZarr S3 Support: @HAS_NCZARR_S3@ + Diskless Support: @HAS_DISKLESS@ MMap Support: @HAS_MMAP@ JNA Support: @HAS_JNA@ -CDF5 Support: @HAS_CDF5@ ERANGE Fill Support: @HAS_ERANGE_FILL@ Relaxed Boundary Check: @RELAX_COORD_BOUND@ -Parallel Filters: @HAS_PAR_FILTERS@ + NCZarr Support: @HAS_NCZARR@ Multi-Filter Support: @HAS_MULTIFILTERS@ Quantization: @HAS_QUANTIZE@ @@ -50,4 +55,5 @@ Logging: @HAS_LOGGING@ SZIP Write Support: @HAS_SZLIB_WRITE@ Standard Filters: @STD_FILTERS@ ZSTD Support: @HAS_ZSTD@ -Benchmarks: @HAS_BENCHMARKS@ +Parallel Filters: @HAS_PAR_FILTERS@ +