Skip to content
This repository was archived by the owner on Sep 1, 2022. It is now read-only.

Commit a409257

Browse files
lesserwhirlslesserwhirls
committed
Use the data_word_size from the level 2 message
Backport of Unidata/netcdf-java#514 Previously, the level 2 code used a fixed word size of 8 bits for the data blocks, with a special exception for the differential phase product, which was hardcoded to use 16 bits. This PR enables the IOSP and Level2 Record parser to use the actual word size as encoded in the message.
1 parent 34784e6 commit a409257

File tree

3 files changed

+132
-11
lines changed

3 files changed

+132
-11
lines changed

‎cdm-test/src/test/java/ucar/nc2/iosp/nexrad2/Test16BitDataWidth.java

+52
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
package ucar.nc2.iosp.nexrad2;
2+
3+
import java.io.IOException;
4+
import org.junit.Assert;
5+
import org.junit.Test;
6+
import org.junit.experimental.categories.Category;
7+
import ucar.ma2.Array;
8+
import ucar.ma2.InvalidRangeException;
9+
import ucar.ma2.MAMath;
10+
import ucar.ma2.MAMath.MinMax;
11+
import ucar.nc2.NetcdfFile;
12+
import ucar.nc2.Variable;
13+
import ucar.nc2.dataset.NetcdfDataset;
14+
import ucar.unidata.util.test.TestDir;
15+
import ucar.unidata.util.test.category.NeedsCdmUnitTest;
16+
17+
@Category(NeedsCdmUnitTest.class)
18+
public class Test16BitDataWidth {
19+
20+
private static final String filename =
21+
TestDir.cdmUnitTestDir + "formats/nexrad/newLevel2/testfiles/Level2_KDDC_20201007_1914.ar2v";
22+
private static final double comparisonTolerance = 1e-6;
23+
24+
25+
@Test
26+
public void testNonPhiVar() throws IOException, InvalidRangeException {
27+
try (NetcdfFile ncf = NetcdfFile.open(filename)) {
28+
// verified against metpy
29+
MinMax expectedMinMax = new MinMax(0, 1058);
30+
Variable var = ncf.findVariable("DifferentialReflectivity_HI");
31+
Array data = var.read("0,:,:");
32+
MinMax minMax = MAMath.getMinMax(data);
33+
34+
Assert.assertTrue(Math.abs(minMax.min - expectedMinMax.min) < comparisonTolerance);
35+
Assert.assertTrue(Math.abs(minMax.max - expectedMinMax.max) < comparisonTolerance);
36+
}
37+
}
38+
39+
@Test
40+
public void testNonPhiVarEnhanced() throws IOException, InvalidRangeException {
41+
try (NetcdfDataset ncf = NetcdfDataset.openDataset(filename)) {
42+
// verified against metpy
43+
MinMax expectedMinMax = new MinMax(-13, 20);
44+
Variable var = ncf.findVariable("DifferentialReflectivity_HI");
45+
Array data = var.read("0,:,:");
46+
MinMax minMax = MAMath.getMinMax(data);
47+
48+
Assert.assertTrue(Math.abs(minMax.min - expectedMinMax.min) < comparisonTolerance);
49+
Assert.assertTrue(Math.abs(minMax.max - expectedMinMax.max) < comparisonTolerance);
50+
}
51+
}
52+
}

‎cdm/src/main/java/ucar/nc2/iosp/nexrad2/Level2Record.java

+79-10
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,8 @@
3434

3535
package ucar.nc2.iosp.nexrad2;
3636

37+
import java.util.ArrayList;
38+
import java.util.List;
3739
import ucar.unidata.io.RandomAccessFile;
3840
import ucar.ma2.IndexIterator;
3941
import ucar.ma2.Range;
@@ -541,6 +543,10 @@ static public java.util.Date getDate(int julianDays, int msecs) {
541543
short phiHR_first_gate = 0;
542544
short rhoHR_first_gate = 0;
543545

546+
byte reflectHR_data_word_size, velocityHR_data_word_size, spectrumHR_data_word_size;
547+
byte zdrHR_data_word_size, phiHR_data_word_size, rhoHR_data_word_size;
548+
549+
private static List<String> missingProductWarned = new ArrayList<>();
544550

545551
public static Level2Record factory(RandomAccessFile din, int record, long message_offset31) throws IOException {
546552
long offset = record * RADAR_DATA_SIZE + FILE_HEADER_SIZE + message_offset31;
@@ -639,6 +645,10 @@ public Level2Record(RandomAccessFile din, int record, long message_offset31) thr
639645
int dbpp8 = 0;
640646
int dbpp9 = 0;
641647

648+
int missingPointerNumber = -999;
649+
int missingPointerValue = -999;
650+
String missingName = "";
651+
642652
if (dbp4 > 0) {
643653
String tname = getDataBlockStringValue(din, (short) dbp4, 1, 3);
644654
if (tname.startsWith("REF")) {
@@ -660,7 +670,9 @@ public Level2Record(RandomAccessFile din, int record, long message_offset31) thr
660670
hasHighResRHOData = true;
661671
dbpp9 = dbp4;
662672
} else {
663-
logger.warn("Missing radial product dbp4={} tname={}", dbp4, tname);
673+
missingName = tname;
674+
missingPointerNumber = 4;
675+
missingPointerValue = dbp4;
664676
}
665677

666678
}
@@ -686,7 +698,9 @@ public Level2Record(RandomAccessFile din, int record, long message_offset31) thr
686698
hasHighResRHOData = true;
687699
dbpp9 = dbp5;
688700
} else {
689-
logger.warn("Missing radial product dbp5={} tname={}", dbp5, tname);
701+
missingName = tname;
702+
missingPointerNumber = 5;
703+
missingPointerValue = dbp5;
690704
}
691705
}
692706
if (dbp6 > 0) {
@@ -711,7 +725,9 @@ public Level2Record(RandomAccessFile din, int record, long message_offset31) thr
711725
hasHighResRHOData = true;
712726
dbpp9 = dbp6;
713727
} else {
714-
logger.warn("Missing radial product dbp6={} tname={}", dbp6, tname);
728+
missingName = tname;
729+
missingPointerNumber = 6;
730+
missingPointerValue = dbp6;
715731
}
716732
}
717733

@@ -737,7 +753,9 @@ public Level2Record(RandomAccessFile din, int record, long message_offset31) thr
737753
hasHighResRHOData = true;
738754
dbpp9 = dbp7;
739755
} else {
740-
logger.warn("Missing radial product dbp7={} tname={}", dbp7, tname);
756+
missingName = tname;
757+
missingPointerNumber = 7;
758+
missingPointerValue = dbp7;
741759
}
742760
}
743761

@@ -763,7 +781,9 @@ public Level2Record(RandomAccessFile din, int record, long message_offset31) thr
763781
hasHighResRHOData = true;
764782
dbpp9 = dbp8;
765783
} else {
766-
logger.warn("Missing radial product dbp8={} tname={}", dbp8, tname);
784+
missingName = tname;
785+
missingPointerNumber = 8;
786+
missingPointerValue = dbp8;
767787
}
768788
}
769789

@@ -789,17 +809,31 @@ public Level2Record(RandomAccessFile din, int record, long message_offset31) thr
789809
hasHighResRHOData = true;
790810
dbpp9 = dbp9;
791811
} else {
792-
logger.warn("Missing radial product dbp9={} tname={}", dbp9, tname);
812+
missingName = tname;
813+
missingPointerNumber = 9;
814+
missingPointerValue = dbp9;
815+
}
816+
}
817+
818+
if (missingPointerNumber != -999) {
819+
// warn once per type per pointer location
820+
String missingKey = String.format("%s%d", missingName, missingPointerNumber);
821+
if (!missingProductWarned.contains(missingKey)) {
822+
String msg = String.format("Unknown radial product dbp%d=%d tname=%s "
823+
+ "(this is the only message you will see about this.)", missingPointerNumber, missingPointerValue,
824+
missingName);
825+
logger.warn(msg);
826+
missingProductWarned.add(missingKey);
793827
}
794828
}
795-
//hasHighResREFData = (dbp4 > 0);
796829

797830
if (hasHighResREFData) {
798831
reflectHR_gate_count = getDataBlockValue(din, (short) dbpp4, 8);
799832
reflectHR_first_gate = getDataBlockValue(din, (short) dbpp4, 10);
800833
reflectHR_gate_size = getDataBlockValue(din, (short) dbpp4, 12);
801834
ref_rf_threshold = getDataBlockValue(din, (short) dbpp4, 14);
802835
ref_snr_threshold = getDataBlockValue(din, (short) dbpp4, 16);
836+
reflectHR_data_word_size = getDataBlockByte(din, (short) dbpp4, 19);
803837
reflectHR_scale = getDataBlockValue1(din, (short) dbpp4, 20);
804838
reflectHR_addoffset = getDataBlockValue1(din, (short) dbpp4, 24);
805839
reflectHR_offset = (short) (dbpp4 + 28);
@@ -812,6 +846,7 @@ public Level2Record(RandomAccessFile din, int record, long message_offset31) thr
812846
velocityHR_gate_size = getDataBlockValue(din, (short) dbpp5, 12);
813847
vel_rf_threshold = getDataBlockValue(din, (short) dbpp5, 14);
814848
vel_snr_threshold = getDataBlockValue(din, (short) dbpp5, 16);
849+
velocityHR_data_word_size = getDataBlockByte(din, (short) dbpp5, 19);
815850
velocityHR_scale = getDataBlockValue1(din, (short) dbpp5, 20);
816851
velocityHR_addoffset = getDataBlockValue1(din, (short) dbpp5, 24);
817852
velocityHR_offset = (short) (dbpp5 + 28);
@@ -824,6 +859,7 @@ public Level2Record(RandomAccessFile din, int record, long message_offset31) thr
824859
spectrumHR_gate_size = getDataBlockValue(din, (short) dbpp6, 12);
825860
sw_rf_threshold = getDataBlockValue(din, (short) dbpp6, 14);
826861
sw_snr_threshold = getDataBlockValue(din, (short) dbpp6, 16);
862+
spectrumHR_data_word_size = getDataBlockByte(din, (short) dbpp6, 19);
827863
spectrumHR_scale = getDataBlockValue1(din, (short) dbpp6, 20);
828864
spectrumHR_addoffset = getDataBlockValue1(din, (short) dbpp6, 24);
829865
spectrumHR_offset = (short) (dbpp6 + 28);
@@ -836,6 +872,7 @@ public Level2Record(RandomAccessFile din, int record, long message_offset31) thr
836872
zdrHR_gate_size = getDataBlockValue(din, (short) dbpp7, 12);
837873
zdrHR_rf_threshold = getDataBlockValue(din, (short) dbpp7, 14);
838874
zdrHR_snr_threshold = getDataBlockValue(din, (short) dbpp7, 16);
875+
zdrHR_data_word_size = getDataBlockByte(din, (short) dbpp7, 19);
839876
zdrHR_scale = getDataBlockValue1(din, (short) dbpp7, 20);
840877
zdrHR_addoffset = getDataBlockValue1(din, (short) dbpp7, 24);
841878
zdrHR_offset = (short) (dbpp7 + 28);
@@ -847,6 +884,7 @@ public Level2Record(RandomAccessFile din, int record, long message_offset31) thr
847884
phiHR_gate_size = getDataBlockValue(din, (short) dbpp8, 12);
848885
phiHR_rf_threshold = getDataBlockValue(din, (short) dbpp8, 14);
849886
phiHR_snr_threshold = getDataBlockValue(din, (short) dbpp8, 16);
887+
phiHR_data_word_size = getDataBlockByte(din, (short) dbpp8, 19);
850888
phiHR_scale = getDataBlockValue1(din, (short) dbpp8, 20);
851889
phiHR_addoffset = getDataBlockValue1(din, (short) dbpp8, 24);
852890
phiHR_offset = (short) (dbpp8 + 28);
@@ -858,6 +896,7 @@ public Level2Record(RandomAccessFile din, int record, long message_offset31) thr
858896
rhoHR_gate_size = getDataBlockValue(din, (short) dbpp9, 12);
859897
rhoHR_rf_threshold = getDataBlockValue(din, (short) dbpp9, 14);
860898
rhoHR_snr_threshold = getDataBlockValue(din, (short) dbpp9, 16);
899+
rhoHR_data_word_size = getDataBlockByte(din, (short) dbpp9, 19);
861900
rhoHR_scale = getDataBlockValue1(din, (short) dbpp9, 20);
862901
rhoHR_addoffset = getDataBlockValue1(din, (short) dbpp9, 24);
863902
rhoHR_offset = (short) (dbpp9 + 28);
@@ -1101,6 +1140,24 @@ private short getDataOffset(int datatype) {
11011140
return Short.MIN_VALUE;
11021141
}
11031142

1143+
byte getDataWordSize(int datatype) {
1144+
switch (datatype) {
1145+
case REFLECTIVITY_HIGH:
1146+
return reflectHR_data_word_size;
1147+
case VELOCITY_HIGH:
1148+
return velocityHR_data_word_size;
1149+
case SPECTRUM_WIDTH_HIGH:
1150+
return spectrumHR_data_word_size;
1151+
case DIFF_REFLECTIVITY_HIGH:
1152+
return zdrHR_data_word_size;
1153+
case DIFF_PHASE:
1154+
return phiHR_data_word_size;
1155+
case CORRELATION_COEFFICIENT:
1156+
return rhoHR_data_word_size;
1157+
}
1158+
return 8;
1159+
}
1160+
11041161
private short getDataBlockValue(RandomAccessFile raf, short offset, int skip) throws IOException {
11051162
long off = offset + message_offset + MESSAGE_HEADER_SIZE;
11061163
raf.seek(off);
@@ -1122,6 +1179,13 @@ private float getDataBlockValue1(RandomAccessFile raf, short offset, int skip) t
11221179
return raf.readFloat();
11231180
}
11241181

1182+
private byte getDataBlockByte(RandomAccessFile raf, short offset, int skip) throws IOException {
1183+
long off = offset + message_offset + MESSAGE_HEADER_SIZE;
1184+
raf.seek(off);
1185+
raf.skipBytes(skip);
1186+
return raf.readByte();
1187+
}
1188+
11251189
public java.util.Date getDate() {
11261190
return getDate(data_julian_date, data_msecs);
11271191
}
@@ -1146,7 +1210,9 @@ public void readData(RandomAccessFile raf, int datatype, Range gateRange, IndexI
11461210
}
11471211

11481212
int dataCount = getGateCount(datatype);
1149-
if (datatype == DIFF_PHASE) {
1213+
byte wordSize = getDataWordSize(datatype);
1214+
1215+
if (wordSize == 16) {
11501216
short[] data = new short[dataCount];
11511217
raf.readShort(data, 0, dataCount);
11521218

@@ -1156,7 +1222,7 @@ public void readData(RandomAccessFile raf, int datatype, Range gateRange, IndexI
11561222
else
11571223
ii.setShortNext(data[i]);
11581224
}
1159-
} else {
1225+
} else if (wordSize == 8) {
11601226
byte[] data = new byte[dataCount];
11611227
raf.readFully(data);
11621228
//short [] ds = convertunsignedByte2Short(data);
@@ -1166,7 +1232,10 @@ public void readData(RandomAccessFile raf, int datatype, Range gateRange, IndexI
11661232
else
11671233
ii.setByteNext(data[i]);
11681234
}
1169-
1235+
} else {
1236+
String message = String.format("Data word size of %d bits not understood for data type %d (%s).", wordSize,
1237+
datatype, getDatatypeName(datatype));
1238+
throw new IOException(message);
11701239
}
11711240
}
11721241

‎cdm/src/main/java/ucar/nc2/iosp/nexrad2/Nexrad2IOServiceProvider.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -288,7 +288,7 @@ public Variable makeVariable(NetcdfFile ncfile, int datatype, String shortName,
288288
dims.add( gateDim);
289289

290290
Variable v = new Variable(ncfile, null, null, shortName);
291-
if(datatype == DIFF_PHASE){
291+
if (firstRecord.getDataWordSize(datatype) == 16) {
292292
v.setDataType(DataType.SHORT);
293293
} else {
294294
v.setDataType(DataType.BYTE);

0 commit comments

Comments
 (0)