Skip to content

Commit b02703a

Browse files
This PR primarily addresses Issue #725.
After a long discussion, I implemented the rules at the end of that issue. They are documented in nccopy.1. Additionally, I added a new, per-variable, -c flag that allows for the direct setting of the chunking parameters for a variable. The form is -c var:c1,c2,...ck where var is the name of the variable (possibly a fully qualified name) and the ci are the chunksizes for that variable. It must be the case that the rank of the variable is k. If the new form is used as well as the old form, then the new form overrides the old form for the specified variable. Note that multiple occurrences of the new form -c flag may be specified. Misc. Other fixes 1. Added -M <size> option to nccopy to specify the minimum allowable chunksize. 2. Removed the unused variables from bigmeta.c (Issue #1079) 3. Fixed failure of nc_test4/tst_filter.sh by using the new -M flag (#1) to allow filter test on a small chunk size.
1 parent 000e7a0 commit b02703a

20 files changed

+1306
-294
lines changed

cf

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
#!/bin/bash
22
#NB=1
3-
DB=1
3+
#DB=1
44
#X=-x
55

66
#FAST=1
@@ -118,7 +118,7 @@ FLAGS="$FLAGS --disable-diskless"
118118
#FLAGS="$FLAGS --enable-jna"
119119
#FLAGS="$FLAGS --disable-properties-attribute"
120120
#FLAGS="$FLAGS --disable-silent-rules"
121-
FLAGS="$FLAGS --disable-filter-testing"
121+
#FLAGS="$FLAGS --disable-filter-testing"
122122
#FLAGS="$FLAGS --enable-metadata-perf"
123123

124124
if test "x$TESTSERVERS" != x ; then

cf.cmake

+1-1
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@ FLAGS="$FLAGS -DENABLE_DAP_REMOTE_TESTS=true"
6060
FLAGS="$FLAGS -DENABLE_LOGGING=true"
6161
#FLAGS="$FLAGS -DENABLE_DOXYGEN=true -DENABLE_INTERNAL_DOCS=true"
6262
#FLAGS="$FLAGS -DENABLE_LARGE_FILE_TESTS=true"
63-
#FLAGS="$FLAGS -DENABLE_FILTER_TESTING=true"
63+
FLAGS="$FLAGS -DENABLE_FILTER_TESTING=true"
6464

6565
# Disables
6666
FLAGS="$FLAGS -DENABLE_EXAMPLES=false"

libdap4/d4data.c

+1-2
Original file line numberDiff line numberDiff line change
@@ -99,8 +99,7 @@ NCD4_processdata(NCD4meta* meta)
9999
for(i=0;i<nclistlength(toplevel);i++) {
100100
NCD4node* var = (NCD4node*)nclistget(toplevel,i);
101101
if(var->data.localchecksum != var->data.remotechecksum) {
102-
fprintf(stderr,"Checksum mismatch: %s\n",var->name);
103-
fflush(stderr);
102+
nclog(NCLOGERR,"Checksum mismatch: %s\n",var->name);
104103
ret = NC_EDAP;
105104
goto done;
106105
}

nc_test4/Make0

+5-3
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
# Test c output
2-
T=tst_varsperf
2+
T=bigmeta
33

44
#SRC=hdf5plugins/H5Zmisc.c
55

@@ -12,7 +12,8 @@ T=tst_varsperf
1212
#PAR=1
1313
#SZIP=1
1414

15-
CFLAGS = -Wall -Wno-unused-variable -Wno-unused-function -g -O0 -I.. -I../include
15+
#CFLAGS = -Wall -Wno-unused-variable -Wno-unused-function -g -O0 -I.. -I../include
16+
CFLAGS = -Wall -g -O0 -I.. -I../include
1617

1718
LDFLAGS = ../liblib/.libs/libnetcdf.a -L/usr/local/lib -lhdf5_hl -lhdf5 -lz -ldl -lcurl -lm -lmfhdf -ldf
1819

@@ -31,7 +32,8 @@ LLP=/usr/local/lib:${LD_LIBRARY_PATH}
3132

3233
all:: cmp
3334
export LD_LIBRARY_PATH=${LLP}; export CFLAGS; export LDFLAGS; \
34-
${CMD} ./t ${ARGS}
35+
36+
# ${CMD} ./t ${ARGS}
3537

3638
cmp::
3739
export LD_LIBRARY_PATH=${LLP}; export CFLAGS; export LDFLAGS; \

nc_test4/bigmeta.c

+1-4
Original file line numberDiff line numberDiff line change
@@ -134,7 +134,6 @@ static void
134134
buildatts(int grpid, int varid)
135135
{
136136
char name[NC_MAX_NAME+1];
137-
int attid;
138137
int i, count;
139138

140139
count = (varid == NC_GLOBAL? ngroupattrs : nvarattrs);
@@ -149,7 +148,7 @@ static void
149148
buildgroup(int parent, int grpindex, int depth)
150149
{
151150
char name[NC_MAX_NAME+1];
152-
int i, grpid, dimid, typid, varid, attid;
151+
int i, grpid, varid;
153152
int dimids[NDIMS];
154153

155154
if(depth == 0) return;
@@ -193,9 +192,7 @@ buildgroup(int parent, int grpindex, int depth)
193192
int
194193
main(int argc, char **argv)
195194
{
196-
int stat = NC_NOERR;
197195
int i, ncid;
198-
int grpid, dimid, varid, typid, attid;
199196
time_t starttime, endtime;
200197
long long delta;
201198
int tag;

nc_test4/findplugin.in

+6
Original file line numberDiff line numberDiff line change
@@ -99,3 +99,9 @@ HDF5_PLUGIN_PATH="$FP_PLUGIN_PATH"
9999

100100
return 0
101101
}
102+
103+
# debug
104+
if test "x$1" != x ; then
105+
findplugin
106+
echo "HDF5_PLUGIN_PATH=|$FP_PLUGIN_PATH|"
107+
fi

nc_test4/tst_filter.sh

+2-1
Original file line numberDiff line numberDiff line change
@@ -110,7 +110,8 @@ echo " *** Pass: nccopy simple filter"
110110

111111
echo " *** Testing pass-thru of filters"
112112
rm -f ./tst_filter.txt tst_filter2.txt ./tst_filter2.nc
113-
${NCCOPY} ./filtered.nc ./tst_filter2.nc
113+
# Prevent failure by allowing any chunk size
114+
${NCCOPY} -M0 ./filtered.nc ./tst_filter2.nc
114115
${NCDUMP} -s tst_filter2.nc > ./tst_filter.txt
115116
sed -e '/_Filter/p' -e d < ./tst_filter.txt >tst_filter2.txt
116117
test -s tst_filter2.txt

ncdump/CMakeLists.txt

+6-1
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ IF(BUILD_SHARED_LIBS AND WIN32)
44
ENDIF()
55

66
SET(ncdump_FILES ncdump.c vardata.c dumplib.c indent.c nctime0.c utils.c nciter.c)
7-
SET(nccopy_FILES nccopy.c nciter.c chunkspec.c utils.c dimmap.c)
7+
SET(nccopy_FILES nccopy.c nciter.c chunkspec.c utils.c dimmap.c list.c)
88

99
IF(USE_X_GETOPT)
1010
SET(ncdump_FILES ${ncdump_FILES} XGetopt.c)
@@ -211,6 +211,11 @@ ENDIF(MSVC)
211211
SET_TESTS_PROPERTIES(ncdump_tst_nccopy4 PROPERTIES RUN_SERIAL TRUE)
212212
ENDIF(HAVE_BASH)
213213

214+
add_sh_test(ncdump tst_nccopy5)
215+
IF(HAVE_BASH)
216+
SET_TESTS_PROPERTIES(ncdump_tst_nccopy5 PROPERTIES RUN_SERIAL TRUE)
217+
ENDIF(HAVE_BASH)
218+
214219
IF(USE_NETCDF4)
215220
add_sh_test(ncdump tst_ncgen4)
216221
ENDIF(USE_NETCDF4)

ncdump/Make0

+4-4
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,12 @@
11
# Test c output
2-
T=tst_unicode
2+
T=tst_chunking
33

44
#ARGS=./x.nc
55

66
#TF=test_atomic_array.cdl
77

88
#CMD=valgrind --leak-check=full
9-
CMD=gdb --args
9+
#CMD=gdb --args
1010

1111
#HDF4=1
1212
#PAR=1
@@ -36,11 +36,11 @@ LLP=/usr/local/lib:${LD_LIBRARY_PATH}
3636

3737
all:: comp
3838
export LD_LIBRARY_PATH=${LLP}; export CFLAGS; export LDFLAGS; \
39-
${CMD} ./t ${ARGS}
39+
${CMD} ./$T ${ARGS}
4040

4141
comp::
4242
export LD_LIBRARY_PATH=${LLP}; export CFLAGS; export LDFLAGS; \
43-
${CC} -o t ${CFLAGS} ${T}.c ${SRC} ${LDFLAGS}; \
43+
${CC} -o $T ${CFLAGS} ${T}.c ${SRC} ${LDFLAGS}; \
4444

4545
x.nc: x.cdl
4646
ncgen -4 x.cdl

ncdump/Makefile.am

+5-5
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ utils.c nciter.h nciter.c nccomps.h
2626
# netCDF API
2727
bin_PROGRAMS += nccopy
2828
nccopy_SOURCES = nccopy.c nciter.c nciter.h chunkspec.h chunkspec.c \
29-
utils.h utils.c dimmap.h dimmap.c
29+
utils.h utils.c dimmap.h dimmap.c list.c list.h
3030

3131
# A simple netcdf-4 metadata -> xml printer. Do not install.
3232
if USE_NETCDF4
@@ -80,14 +80,14 @@ tst_unicode tst_fillbug tst_compress tst_chunking tst_h_scalar
8080
# Tests for netCDF-4 behavior.
8181
TESTS += tst_fileinfo.sh tst_hdf5_offset.sh tst_inttags4.sh \
8282
tst_netcdf4.sh tst_fillbug.sh tst_netcdf4_4.sh tst_nccopy4.sh \
83-
tst_grp_spec.sh tst_mud.sh tst_h_scalar.sh tst_formatx4.sh \
83+
tst_nccopy5.sh tst_grp_spec.sh tst_mud.sh tst_h_scalar.sh tst_formatx4.sh \
8484
run_utf8_nc4_tests.sh run_back_comp_tests.sh run_ncgen_nc4_tests.sh \
8585
tst_ncgen4.sh
8686

87-
# The tst_nccopy4.sh test script depends on the output of a bunch of
88-
# other tests. Record dependencies so parallel builds work.
87+
# Record interscript dependencies so parallel builds work.
8988
tst_nccopy4.log: run_ncgen_tests.log tst_output.log tst_ncgen4.log \
9089
tst_fillbug.log tst_netcdf4_4.log tst_h_scalar.log
90+
tst_nccopy5.log: tst_nccopy4.log
9191
endif #!USE_NETCDF4
9292

9393
TESTS += tst_inmemory_nc3.sh
@@ -111,7 +111,7 @@ ref_tst_noncoord.cdl ref_tst_compounds2.nc ref_tst_compounds2.cdl \
111111
ref_tst_compounds3.nc ref_tst_compounds3.cdl ref_tst_compounds4.nc \
112112
ref_tst_compounds4.cdl ref_tst_group_data_v23.cdl tst_mslp.cdl \
113113
tst_bug321.cdl ref_tst_format_att.cdl ref_tst_format_att_64.cdl \
114-
tst_nccopy3.sh tst_nccopy4.sh ref_nc_test_netcdf4_4_0.nc \
114+
tst_nccopy3.sh tst_nccopy4.sh tst_nccopy5.sh ref_nc_test_netcdf4_4_0.nc \
115115
run_back_comp_tests.sh ref_nc_test_netcdf4.cdl \
116116
ref_tst_special_atts3.cdl tst_brecs.cdl ref_tst_grp_spec0.cdl \
117117
ref_tst_grp_spec.cdl tst_grp_spec.sh ref_tst_charfill.cdl \

0 commit comments

Comments
 (0)