This PR primarily addresses Issue #725.

After a long discussion, I implemented the rules at the end of that issue.
They are documented in nccopy.1.

Additionally, I added a new, per-variable, -c flag that allows
for the direct setting of the chunking parameters for a variable.
The form is
    -c var:c1,c2,...ck
where var is the name of the variable (possibly a fully qualified name)
and the ci are the chunksizes for that variable. It must be the case
that the rank of the variable is k. If the new form is used as well
as the old form, then the new form overrides the old form for the
specified variable. Note that multiple occurrences of the new form
-c flag may be specified.

Misc. Other fixes
1. Added -M <size> option to nccopy to specify the minimum
   allowable chunksize.
2. Removed the unused variables from bigmeta.c
   (Issue #1079)
3. Fixed failure of nc_test4/tst_filter.sh by using the new -M
   flag (#1) to allow filter test on a small chunk size.

Author: DennisHeimbigner

cf

@@ -1,6 +1,6 @@
 #!/bin/bash
 #NB=1
-DB=1
+#DB=1
 #X=-x
 
 #FAST=1
@@ -118,7 +118,7 @@ FLAGS="$FLAGS --disable-diskless"
 #FLAGS="$FLAGS --enable-jna"
 #FLAGS="$FLAGS --disable-properties-attribute"
 #FLAGS="$FLAGS --disable-silent-rules"
-FLAGS="$FLAGS --disable-filter-testing"
+#FLAGS="$FLAGS --disable-filter-testing"
 #FLAGS="$FLAGS --enable-metadata-perf"
 
 if test "x$TESTSERVERS" != x ; then

cf.cmake

@@ -60,7 +60,7 @@ FLAGS="$FLAGS -DENABLE_DAP_REMOTE_TESTS=true"
 FLAGS="$FLAGS -DENABLE_LOGGING=true"
 #FLAGS="$FLAGS -DENABLE_DOXYGEN=true -DENABLE_INTERNAL_DOCS=true"
 #FLAGS="$FLAGS -DENABLE_LARGE_FILE_TESTS=true"
-#FLAGS="$FLAGS -DENABLE_FILTER_TESTING=true"
+FLAGS="$FLAGS -DENABLE_FILTER_TESTING=true"
 
 # Disables
 FLAGS="$FLAGS -DENABLE_EXAMPLES=false"

libdap4/d4data.c

@@ -99,8 +99,7 @@ NCD4_processdata(NCD4meta* meta)
         for(i=0;i<nclistlength(toplevel);i++) {
 	    NCD4node* var = (NCD4node*)nclistget(toplevel,i);
 	    if(var->data.localchecksum != var->data.remotechecksum) {
-		fprintf(stderr,"Checksum mismatch: %s\n",var->name);
-		fflush(stderr);
+		nclog(NCLOGERR,"Checksum mismatch: %s\n",var->name);
 		ret = NC_EDAP;
 		goto done;
 	    }

nc_test4/Make0

@@ -1,5 +1,5 @@
 # Test c output
-T=tst_varsperf
+T=bigmeta
 
 #SRC=hdf5plugins/H5Zmisc.c
 
@@ -12,7 +12,8 @@ T=tst_varsperf
 #PAR=1
 #SZIP=1
 
-CFLAGS = -Wall -Wno-unused-variable -Wno-unused-function -g -O0 -I.. -I../include
+#CFLAGS = -Wall -Wno-unused-variable -Wno-unused-function -g -O0 -I.. -I../include
+CFLAGS = -Wall -g -O0 -I.. -I../include
 
 LDFLAGS = ../liblib/.libs/libnetcdf.a -L/usr/local/lib -lhdf5_hl -lhdf5 -lz -ldl -lcurl -lm -lmfhdf -ldf
 
@@ -31,7 +32,8 @@ LLP=/usr/local/lib:${LD_LIBRARY_PATH}
 
 all:: cmp
 	export LD_LIBRARY_PATH=${LLP}; export CFLAGS; export LDFLAGS; \
-	${CMD} ./t ${ARGS}
+
+#	${CMD} ./t ${ARGS}
 
 cmp::
 	export LD_LIBRARY_PATH=${LLP}; export CFLAGS; export LDFLAGS; \

nc_test4/bigmeta.c

@@ -134,7 +134,6 @@ static void
 buildatts(int grpid, int varid)
 {
     char name[NC_MAX_NAME+1];
-    int attid;
     int i, count;
 
     count = (varid == NC_GLOBAL? ngroupattrs : nvarattrs);
@@ -149,7 +148,7 @@ static void
 buildgroup(int parent, int grpindex, int depth)
 {
     char name[NC_MAX_NAME+1];
-    int i, grpid, dimid, typid, varid, attid;
+    int i, grpid, varid;
     int dimids[NDIMS];
 
     if(depth == 0) return;
@@ -193,9 +192,7 @@ buildgroup(int parent, int grpindex, int depth)
 int
 main(int argc, char **argv)
 {
-    int stat = NC_NOERR;
     int i, ncid;
-    int grpid, dimid, varid, typid, attid;
     time_t starttime, endtime;
     long long delta;
     int tag;

nc_test4/findplugin.in

@@ -99,3 +99,9 @@ HDF5_PLUGIN_PATH="$FP_PLUGIN_PATH"
 
 return 0
 }
+
+# debug
+if test "x$1" != x ; then
+findplugin
+echo "HDF5_PLUGIN_PATH=|$FP_PLUGIN_PATH|"
+fi

nc_test4/tst_filter.sh

@@ -110,7 +110,8 @@ echo "	*** Pass: nccopy simple filter"
 
 echo "	*** Testing pass-thru of filters"
 rm -f ./tst_filter.txt tst_filter2.txt ./tst_filter2.nc
-${NCCOPY} ./filtered.nc ./tst_filter2.nc
+# Prevent failure by allowing any chunk size
+${NCCOPY} -M0 ./filtered.nc ./tst_filter2.nc
 ${NCDUMP} -s tst_filter2.nc > ./tst_filter.txt
 sed -e '/_Filter/p' -e d < ./tst_filter.txt >tst_filter2.txt
 test -s tst_filter2.txt

ncdump/CMakeLists.txt

@@ -4,7 +4,7 @@ IF(BUILD_SHARED_LIBS AND WIN32)
 ENDIF()
 
 SET(ncdump_FILES ncdump.c vardata.c dumplib.c indent.c nctime0.c utils.c nciter.c)
-SET(nccopy_FILES nccopy.c nciter.c chunkspec.c utils.c dimmap.c)
+SET(nccopy_FILES nccopy.c nciter.c chunkspec.c utils.c dimmap.c list.c)
 
 IF(USE_X_GETOPT)
   SET(ncdump_FILES ${ncdump_FILES} XGetopt.c)
@@ -211,6 +211,11 @@ ENDIF(MSVC)
       SET_TESTS_PROPERTIES(ncdump_tst_nccopy4 PROPERTIES RUN_SERIAL TRUE)
     ENDIF(HAVE_BASH)
 
+    add_sh_test(ncdump tst_nccopy5)
+    IF(HAVE_BASH)
+      SET_TESTS_PROPERTIES(ncdump_tst_nccopy5 PROPERTIES RUN_SERIAL TRUE)
+    ENDIF(HAVE_BASH)
+
     IF(USE_NETCDF4)
       add_sh_test(ncdump tst_ncgen4)
     ENDIF(USE_NETCDF4)

ncdump/Make0

@@ -1,12 +1,12 @@
 # Test c output
-T=tst_unicode
+T=tst_chunking
 
 #ARGS=./x.nc
 
 #TF=test_atomic_array.cdl
 
 #CMD=valgrind --leak-check=full
-CMD=gdb --args
+#CMD=gdb --args
 
 #HDF4=1
 #PAR=1
@@ -36,11 +36,11 @@ LLP=/usr/local/lib:${LD_LIBRARY_PATH}
 
 all:: comp
 	export LD_LIBRARY_PATH=${LLP}; export CFLAGS; export LDFLAGS; \
-	${CMD} ./t ${ARGS}
+	${CMD} ./$T ${ARGS}
 
 comp::
 	export LD_LIBRARY_PATH=${LLP}; export CFLAGS; export LDFLAGS; \
-	${CC} -o t ${CFLAGS} ${T}.c ${SRC} ${LDFLAGS}; \
+	${CC} -o $T ${CFLAGS} ${T}.c ${SRC} ${LDFLAGS}; \
 
 x.nc: x.cdl
 	ncgen -4 x.cdl

ncdump/Makefile.am

@@ -26,7 +26,7 @@ utils.c nciter.h nciter.c nccomps.h
 # netCDF API
 bin_PROGRAMS += nccopy
 nccopy_SOURCES = nccopy.c nciter.c nciter.h chunkspec.h chunkspec.c     \
-utils.h utils.c dimmap.h dimmap.c
+utils.h utils.c dimmap.h dimmap.c list.c list.h
 
 # A simple netcdf-4 metadata -> xml printer. Do not install.
 if USE_NETCDF4
@@ -80,14 +80,14 @@ tst_unicode tst_fillbug tst_compress tst_chunking tst_h_scalar
 # Tests for netCDF-4 behavior.
 TESTS += tst_fileinfo.sh tst_hdf5_offset.sh tst_inttags4.sh		\
 tst_netcdf4.sh tst_fillbug.sh tst_netcdf4_4.sh tst_nccopy4.sh		\
-tst_grp_spec.sh tst_mud.sh tst_h_scalar.sh tst_formatx4.sh		\
+tst_nccopy5.sh tst_grp_spec.sh tst_mud.sh tst_h_scalar.sh tst_formatx4.sh		\
 run_utf8_nc4_tests.sh run_back_comp_tests.sh run_ncgen_nc4_tests.sh	\
 tst_ncgen4.sh
 
-# The tst_nccopy4.sh test script depends on the output of a bunch of
-# other tests. Record dependencies so parallel builds work.
+# Record interscript dependencies so parallel builds work.
 tst_nccopy4.log: run_ncgen_tests.log tst_output.log tst_ncgen4.log	\
 tst_fillbug.log tst_netcdf4_4.log tst_h_scalar.log
+tst_nccopy5.log: tst_nccopy4.log
 endif #!USE_NETCDF4
 
 TESTS += tst_inmemory_nc3.sh
@@ -111,7 +111,7 @@ ref_tst_noncoord.cdl ref_tst_compounds2.nc ref_tst_compounds2.cdl	\
 ref_tst_compounds3.nc ref_tst_compounds3.cdl ref_tst_compounds4.nc	\
 ref_tst_compounds4.cdl ref_tst_group_data_v23.cdl tst_mslp.cdl		\
 tst_bug321.cdl ref_tst_format_att.cdl ref_tst_format_att_64.cdl		\
-tst_nccopy3.sh tst_nccopy4.sh ref_nc_test_netcdf4_4_0.nc		\
+tst_nccopy3.sh tst_nccopy4.sh tst_nccopy5.sh ref_nc_test_netcdf4_4_0.nc		\
 run_back_comp_tests.sh ref_nc_test_netcdf4.cdl				\
 ref_tst_special_atts3.cdl tst_brecs.cdl ref_tst_grp_spec0.cdl		\
 ref_tst_grp_spec.cdl tst_grp_spec.sh ref_tst_charfill.cdl		\