Merge branch 'master' into dispatchversion.dmh

Author: WardF

CMakeLists.txt

@@ -661,7 +661,6 @@ IF(USE_HDF5)
     IF(HDF5_VERSION_STRING AND NOT HDF5_VERSION)
       SET(HDF5_VERSION ${HDF5_VERSION_STRING})
     ENDIF()
-
     IF("${HDF5_VERSION}" STREQUAL "")
       MESSAGE(STATUS "Unable to determine hdf5 version.  NetCDF requires at least version ${HDF5_VERSION_REQUIRED}")
     ELSE()
@@ -723,7 +722,7 @@ IF(USE_HDF5)
   FIND_PACKAGE(Threads)
 
   # There is a missing case in the above code so default it
-  IF(NOT HDF5_C_LIBRARY_HDF5 OR "${HDF5_C_LIBRARY_hdf5}" STREQUAL "" )
+  IF(NOT HDF5_C_LIBRARY_hdf5 OR "${HDF5_C_LIBRARY_hdf5}" STREQUAL "" )
     SET(HDF5_C_LIBRARY_hdf5 "${HDF5_C_LIBRARY}")
   ENDIF()
 
@@ -802,6 +801,15 @@ IF(USE_HDF5)
     SET(HAS_PAR_FILTERS no CACHE STRING "")
   ENDIF()
 
+  # Check to see if HDF5 library is 1.10.6 or greater.
+  # Used to control path name conversion
+  IF(${HDF5_VERSION} VERSION_LESS "1.10.6")
+    SET(HDF5_UTF8_PATHS FALSE)
+  ELSE()
+    SET(HDF5_UTF8_PATHS TRUE)
+  ENDIF()
+  MESSAGE("-- Checking for HDF5 version 1.10.6 or later: ${HDF5_UTF8_PATHS}")
+
   SET(H5_USE_16_API 1)
   OPTION(NC_ENABLE_HDF_16_API "Enable HDF5 1.6.x Compatibility(Required)" ON)
   IF(NOT NC_ENABLE_HDF_16_API)
@@ -833,7 +841,7 @@ IF(USE_HDF5)
   #option to include HDF5 High Level header file (hdf5_hl.h) in case we are not doing a make install
   INCLUDE_DIRECTORIES(${HDF5_HL_INCLUDE_DIR})
 
-  # Check to see if we have libcurl 7.66 or later
+  # Check HDF5 version
   CHECK_C_SOURCE_COMPILES("
   #include <H5public.h>
   int main() {
@@ -941,6 +949,17 @@ OPTION(ENABLE_DAP_LONG_TESTS "Enable DAP long tests." OFF)
 OPTION(ENABLE_DAP_REMOTE_TESTS "Enable DAP remote tests." ON)
 SET(REMOTETESTSERVERS "remotetest.unidata.ucar.edu" CACHE STRING "test servers to use for remote test")
 
+# See if we have libzip
+FIND_PACKAGE(Zip)
+
+# Define a test flag for have curl library
+IF(Zip_FOUND)
+  INCLUDE_DIRECTORIES(${Zip_INCLUDE_DIRS})
+  SET(ENABLE_NCZARR_ZIP TRUE)
+ELSE()
+  SET(ENABLE_NCZARR_ZIP FALSE)
+ENDIF()
+
 # Enable some developer-only tests
 OPTION(ENABLE_EXTRA_TESTS "Enable Extra tests. Some may not work because of known issues. Developers only." OFF)
 IF(ENABLE_EXTRA_TESTS)
@@ -1016,6 +1035,10 @@ IF(NOT ENABLE_S3_SDK)
   ENDIF()
 ENDIF()
 
+IF(ENABLE_NCZARR_S3_TESTS)
+  message(WARNING "**** DO NOT ENABLE_NCZARR_S3_TESTS UNLESS YOU HAVE ACCESS TO THE UNIDATA S3 BUCKET! ***")
+ENDIF()
+
 # Start disabling if curl not found
 IF(NOT FOUND_CURL)
   IF(ENABLE_BYTERANGE)
@@ -1035,6 +1058,11 @@ IF(NOT FOUND_CURL)
   ENDIF()
 ENDIF(NOT FOUND_CURL)
 
+IF(NOT ENABLE_BYTERANGE AND ENABLE_HDF5_ROS3)
+    MESSAGE(WARNING "ROS3 support requires ENABLE_BYTERANGE=TRUE; disabling ROS3 support")
+    SET(ENABLE_HDF5_ROS3 OFF CACHE BOOL "ROS3 support" FORCE)
+ENDIF()
+
 ##
 # Enable Tests
 ##
@@ -2144,8 +2172,9 @@ is_enabled(ENABLE_ERANGE_FILL HAS_ERANGE_FILL)
 is_enabled(HAVE_H5Z_SZIP HAS_SZLIB)
 is_enabled(HDF5_HAS_PAR_FILTERS HAS_PAR_FILTERS)
 is_enabled(ENABLE_NCZARR HAS_NCZARR)
-is_enabled(ENABLE_NCZARR_S3_TESTS DO_S3_TESTS)
+is_enabled(ENABLE_NCZARR_S3_TESTS DO_NCZARR_S3_TESTS)
 is_enabled(ENABLE_MULTIFILTERS HAS_MULTIFILTERS)
+is_enabled(ENABLE_NCZARR_ZIP DO_NCZARR_ZIP_TESTS)
 
 # Generate file from template.
 CONFIGURE_FILE("${CMAKE_CURRENT_SOURCE_DIR}/libnetcdf.settings.in"
@@ -2189,6 +2218,7 @@ configure_file(
 SET(EXTRA_DIST ${EXTRA_DIST} ${CMAKE_CURRENT_SOURCE_DIR}/test_common.in)
 SET(TOPSRCDIR "${CMAKE_CURRENT_SOURCE_DIR}")
 SET(TOPBUILDDIR "${CMAKE_CURRENT_BINARY_DIR}")
+SET(ISMSVC "${MSVC}")
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/test_common.in ${CMAKE_CURRENT_BINARY_DIR}/test_common.sh @ONLY NEWLINE_STYLE LF)
 
 

README.md

@@ -47,7 +47,7 @@ software for various languages:
 
 * [C library and utilities](http://github.com/Unidata/netcdf-c)
 * [Fortran](http://github.com/Unidata/netcdf-fortran)
-* [Java](http://www.unidata.ucar.edu/downloads/netcdf/netcdf-java-4/)
+* [Java](https://www.unidata.ucar.edu/downloads/netcdf-java/
 * [Python](http://github.com/Unidata/netcdf4-python)
 * [C++](http://github.com/Unidata/netcdf-cxx4)
 

RELEASE_NOTES.md

@@ -8,6 +8,9 @@ This file contains a high-level description of this package's evolution. Release
 ## 4.8.0 - TBD
 
 * [Enhancement] Bump the NC_DISPATCH_VERSION from 2 to 3, and as a side effect, unify the definition of NC_DISPATCH_VERSION so it only needs to be defined in CMakeLists.txt and configure.ac. See [Github #1945](https://github.com/Unidata/netcdf-c/pull/1945) for more information.
+* [Enhancement] Provide better cross platform path name management. This converts paths for various platforms (e.g. Windows, MSYS, etc.) so that they are in the proper format for the executing platform. See [Github #1958](https://github.com/Unidata/netcdf-c/pull/1958) for more information.
+* [Bug Fixes] The nccopy program was treating -d0 as turning deflation on rather than interpreting it as "turn off deflation". See [Github #1944](https://github.com/Unidata/netcdf-c/pull/1944) for more information.
+* [Enhancement] Add support for storing NCZarr data in zip files. See [Github #1942](https://github.com/Unidata/netcdf-c/pull/1942) for more information.
 * [Bug Fixes] Make fillmismatch the default for DAP2 and DAP4; too many servers ignore this requirement.
 * [Bug Fixes] Fix some memory leaks in NCZarr, fix a bug with long strides in NCZarr. See [Github #1913](https://github.com/Unidata/netcdf-c/pull/1913) for more information.
 * [Enhancement] Add some optimizations to NCZarr, dosome cleanup of code cruft, add some NCZarr test cases, add a performance test to NCZarr. See [Github #1908](https://github.com/Unidata/netcdf-c/pull/1908) for more information.

cmake/modules/FindZip.cmake

@@ -0,0 +1,64 @@
+# Searches for an installation of the zip library. On success, it sets the following variables:
+#
+#   Zip_FOUND              Set to true to indicate the zip library was found
+#   Zip_INCLUDE_DIRS       The directory containing the header file zip/zip.h
+#   Zip_LIBRARIES          The libraries needed to use the zip library
+#
+# To specify an additional directory to search, set Zip_ROOT.
+#
+# Author: Siddhartha Chaudhuri, 2009
+#
+
+# Look for the header, first in the user-specified location and then in the system locations
+SET(Zip_INCLUDE_DOC "The directory containing the header file zip/zip.h")
+FIND_PATH(Zip_INCLUDE_DIRS NAMES zip.h zip/zip.h PATHS ${Zip_ROOT} ${Zip_ROOT}/include DOC ${Zip_INCLUDE_DOC} NO_DEFAULT_PATH)
+IF(NOT Zip_INCLUDE_DIRS)  # now look in system locations
+  FIND_PATH(Zip_INCLUDE_DIRS NAMES zip.h zip/zip.h DOC ${Zip_INCLUDE_DOC})
+ENDIF(NOT Zip_INCLUDE_DIRS)
+
+SET(Zip_FOUND FALSE)
+
+IF(Zip_INCLUDE_DIRS)
+  SET(Zip_LIBRARY_DIRS ${Zip_INCLUDE_DIRS})
+
+  IF("${Zip_LIBRARY_DIRS}" MATCHES "/include$")
+    # Strip off the trailing "/include" in the path.
+    GET_FILENAME_COMPONENT(Zip_LIBRARY_DIRS ${Zip_LIBRARY_DIRS} PATH)
+  ENDIF("${Zip_LIBRARY_DIRS}" MATCHES "/include$")
+
+  IF(EXISTS "${Zip_LIBRARY_DIRS}/lib")
+    SET(Zip_LIBRARY_DIRS ${Zip_LIBRARY_DIRS}/lib)
+  ENDIF(EXISTS "${Zip_LIBRARY_DIRS}/lib")
+
+  # Find Zip libraries
+  FIND_LIBRARY(Zip_DEBUG_LIBRARY NAMES zipd zip_d libzipd libzip_d libzip
+               PATH_SUFFIXES Debug ${CMAKE_LIBRARY_ARCHITECTURE} ${CMAKE_LIBRARY_ARCHITECTURE}/Debug
+               PATHS ${Zip_LIBRARY_DIRS} NO_DEFAULT_PATH)
+  FIND_LIBRARY(Zip_RELEASE_LIBRARY NAMES zip libzip
+               PATH_SUFFIXES Release ${CMAKE_LIBRARY_ARCHITECTURE} ${CMAKE_LIBRARY_ARCHITECTURE}/Release
+               PATHS ${Zip_LIBRARY_DIRS} NO_DEFAULT_PATH)
+
+  SET(Zip_LIBRARIES )
+  IF(Zip_DEBUG_LIBRARY AND Zip_RELEASE_LIBRARY)
+    SET(Zip_LIBRARIES debug ${Zip_DEBUG_LIBRARY} optimized ${Zip_RELEASE_LIBRARY})
+  ELSEIF(Zip_DEBUG_LIBRARY)
+    SET(Zip_LIBRARIES ${Zip_DEBUG_LIBRARY})
+  ELSEIF(Zip_RELEASE_LIBRARY)
+    SET(Zip_LIBRARIES ${Zip_RELEASE_LIBRARY})
+  ENDIF(Zip_DEBUG_LIBRARY AND Zip_RELEASE_LIBRARY)
+
+  IF(Zip_LIBRARIES)
+    SET(Zip_FOUND TRUE)
+  ENDIF(Zip_LIBRARIES)
+ENDIF(Zip_INCLUDE_DIRS)
+
+IF(Zip_FOUND)
+#  IF(NOT Zip_FIND_QUIETLY)
+    MESSAGE(STATUS "Found Zip: headers at ${Zip_INCLUDE_DIRS}, libraries at ${Zip_LIBRARY_DIRS}")
+    MESSAGE(STATUS "	library is ${Zip_LIBRARIES}")    
+#  ENDIF(NOT Zip_FIND_QUIETLY)
+ELSE(Zip_FOUND)
+  IF(Zip_FIND_REQUIRED)
+    MESSAGE(FATAL_ERROR "Zip library not found")
+  ENDIF(Zip_FIND_REQUIRED)
+ENDIF(Zip_FOUND)

config.h.cmake.in

@@ -156,6 +156,9 @@ are set when opening a binary file on Windows. */
 /* if true, enable S3 support */
 #cmakedefine ENABLE_NCZARR_S3 1
 
+/* if true, enable nczarr zip support */
+#cmakedefine ENABLE_NCZARR_ZIP 1
+
 /* if true, enable S3 testing*/
 #cmakedefine ENABLE_NCZARR_S3_TESTS 1
 
@@ -310,6 +313,9 @@ are set when opening a binary file on Windows. */
 /* Define to 1 if you have the `MPI_Comm_f2c' function. */
 #cmakedefine HAVE_MPI_COMM_F2C 1
 
+/* Define to 1 if you have the `MPI_Info_f2c' function. */
+#cmakedefine HAVE_MPI_INFO_F2C 1
+
 /* Define to 1 if you have the `mremap' function. */
 #cmakedefine HAVE_MREMAP 1
 

configure.ac

@@ -79,9 +79,9 @@ AC_MSG_NOTICE([checking supported formats])
 # An explicit disable of netcdf-4 | netcdf4 is treated as if it was disable-hdf5
 AC_MSG_CHECKING([whether we should build with netcdf4 (alias for HDF5)])
 AC_ARG_ENABLE([netcdf4], [AS_HELP_STRING([--disable-netcdf4],
-              [(synonym for --enable-hdf5)])])
+              [(deprecated synonym for --enable-hdf5)])])
 test "x$enable_netcdf4" = xno || enable_netcdf4=yes
-AC_MSG_RESULT([$enable_netcdf4])
+AC_MSG_RESULT([$enable_netcdf4 (deprecated; Please use with --disable-hdf5)])
 AC_MSG_CHECKING([whether we should build with netcdf-4 (alias for HDF5)])
 AC_ARG_ENABLE([netcdf-4], [AS_HELP_STRING([--disable-netcdf-4],
               [(synonym for --disable-netcdf4)])])
@@ -629,6 +629,27 @@ if test "x$enable_nczarr_s3_tests" = xyes ; then
 fi
 AM_CONDITIONAL(ENABLE_NCZARR_S3_TESTS, [test "x$enable_nczarr_s3_tests" = xyes])
 
+if test "x$enable_nczarr_s3_tests" = xyes ; then
+  AC_MSG_WARN([*** DO NOT ENABLE_NCZARR_S3_TESTS UNLESS YOU HAVE ACCESS TO THE UNIDATA S3 BUCKET! ***])
+fi
+
+# Set default
+# See if we have libzip
+AC_CHECK_LIB([zip],[zip_open],[have_zip=yes],[have_zip=no])
+if test "x$have_zip" = "xyes" ; then
+   AC_SEARCH_LIBS([zip_open],[zip zip.dll cygzip.dll], [],
+      [AC_MSG_ERROR([libzip search failed.])])
+fi
+AC_MSG_CHECKING([whether libzip library is available])
+AC_MSG_RESULT([${have_zip}])
+
+enable_nczarr_zip=${have_zip} # alias
+
+if test "x$enable_nczarr_zip" = xyes ; then
+AC_DEFINE([ENABLE_NCZARR_ZIP], [1], [If true, then libzip found])
+fi
+AM_CONDITIONAL(ENABLE_NCZARR_ZIP, [test "x$enable_nczarr_zip" = xyes])
+
 # Did the user specify a default cache size for NCZarr?
 AC_MSG_CHECKING([whether a default file cache size for NCZarr was specified])
 AC_ARG_WITH([chunk-cache-size-nczarr],
@@ -1589,6 +1610,7 @@ AC_SUBST([HAS_PAR_FILTERS], [$hdf5_supports_par_filters])
 AC_SUBST(HAS_NCZARR,[$enable_nczarr])
 AC_SUBST(DO_NCZARR_S3_TESTS,[$enable_nczarr_s3_tests])
 AC_SUBST(HAS_MULTIFILTERS,[$has_multifilters])
+AC_SUBST(DO_NCZARR_ZIP_TESTS,[$enable_nczarr_zip])
 
 # Include some specifics for netcdf on windows.
 #AH_VERBATIM([_WIN32_STRICMP],

dap4_test/Makefile.am

@@ -73,7 +73,7 @@ clean-local: clean-local-check
 
 clean-local-check:
 	-rm -rf results results_test_parse results_test_data \
-                results_test_hyrax results_test_meta
+                results_test_hyrax results_test_meta results_test_raw
 	-rm -f .dodsrc .daprc
 
 # The shell file maketests.sh is used to build the testdata

dap4_test/dump.c

@@ -21,12 +21,12 @@
 #endif
 
 #include "netcdf.h"
+#include "netcdf_aux.h"
 #include "ncbytes.h"
 #include "ncpathmgr.h"
 
 extern void NCD4_dumpbytes(size_t size, const void* data0, int swap);
 extern void NCD4_tagdump(size_t size, const void* data0, int swap, const char* tag);
-extern int NC_readfile(const char* filename, NCbytes* content);
 
 static char* progname = NULL;
 
@@ -42,7 +42,6 @@ main(int argc, char *argv[])
 {
     int c;
     char* fname = NULL;
-    NCbytes* contents = ncbytesnew();
     char* tag = NULL;
     size_t offset = 0;
     size_t len = 0;
@@ -81,12 +80,10 @@ main(int argc, char *argv[])
     }
 
     if(tag == NULL) tag = strdup(progname);    
-    if(NC_readfile(fname,contents)) usage();
-    len = ncbyteslength(contents) - offset;
-    data = (char*)ncbytescontents(contents);
+    if(ncaux_readfile(fname,&len,&((void*)data))) usage();
     data += offset;
     NCD4_tagdump(len,data,swap,tag);
-
+    nullfree(data);
     return 0;
 }