Skip to content

Commit 2993fef

Browse files
dopplershiftdopplershift
authored
Merge pull request #514 from lesserwhirls/level2
Use the data_word_size from the level 2 message
2 parents 200ddfc + b569339 commit 2993fef

File tree

3 files changed

+136
-14
lines changed

3 files changed

+136
-14
lines changed

‎cdm/radial/src/main/java/ucar/nc2/iosp/nexrad2/Level2Record.java

+81-13
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@
66

77
package ucar.nc2.iosp.nexrad2;
88

9+
import java.util.ArrayList;
10+
import java.util.List;
911
import ucar.unidata.io.RandomAccessFile;
1012
import ucar.ma2.IndexIterator;
1113
import ucar.ma2.Range;
@@ -179,9 +181,9 @@ public static String getDatatypeUnits(int datatype) {
179181
switch (datatype) {
180182
case REFLECTIVITY:
181183
case REFLECTIVITY_HIGH:
182-
case DIFF_REFLECTIVITY_HIGH:
183184
return "dBz";
184-
185+
case DIFF_REFLECTIVITY_HIGH:
186+
return "dB";
185187
case VELOCITY_HI:
186188
case VELOCITY_LOW:
187189
case SPECTRUM_WIDTH:
@@ -506,6 +508,10 @@ public static java.util.Date getDate(int julianDays, int msecs) {
506508
short phiHR_first_gate;
507509
short rhoHR_first_gate;
508510

511+
byte reflectHR_data_word_size, velocityHR_data_word_size, spectrumHR_data_word_size;
512+
byte zdrHR_data_word_size, phiHR_data_word_size, rhoHR_data_word_size;
513+
514+
private static List<String> missingProductWarned = new ArrayList<>();
509515

510516
public static Level2Record factory(RandomAccessFile din, int record, long message_offset31) throws IOException {
511517
long offset = record * RADAR_DATA_SIZE + FILE_HEADER_SIZE + message_offset31;
@@ -604,6 +610,10 @@ public Level2Record(RandomAccessFile din, int record, long message_offset31) thr
604610
int dbpp8 = 0;
605611
int dbpp9 = 0;
606612

613+
int missingPointerNumber = -999;
614+
int missingPointerValue = -999;
615+
String missingName = "";
616+
607617
if (dbp4 > 0) {
608618
String tname = getDataBlockStringValue(din, (short) dbp4, 1, 3);
609619
if (tname.startsWith("REF")) {
@@ -625,7 +635,9 @@ public Level2Record(RandomAccessFile din, int record, long message_offset31) thr
625635
hasHighResRHOData = true;
626636
dbpp9 = dbp4;
627637
} else {
628-
logger.warn("Missing radial product dbp4={} tname={}", dbp4, tname);
638+
missingName = tname;
639+
missingPointerNumber = 4;
640+
missingPointerValue = dbp4;
629641
}
630642

631643
}
@@ -651,7 +663,9 @@ public Level2Record(RandomAccessFile din, int record, long message_offset31) thr
651663
hasHighResRHOData = true;
652664
dbpp9 = dbp5;
653665
} else {
654-
logger.warn("Missing radial product dbp5={} tname={}", dbp5, tname);
666+
missingName = tname;
667+
missingPointerNumber = 5;
668+
missingPointerValue = dbp5;
655669
}
656670
}
657671
if (dbp6 > 0) {
@@ -676,7 +690,9 @@ public Level2Record(RandomAccessFile din, int record, long message_offset31) thr
676690
hasHighResRHOData = true;
677691
dbpp9 = dbp6;
678692
} else {
679-
logger.warn("Missing radial product dbp6={} tname={}", dbp6, tname);
693+
missingName = tname;
694+
missingPointerNumber = 6;
695+
missingPointerValue = dbp6;
680696
}
681697
}
682698

@@ -702,7 +718,9 @@ public Level2Record(RandomAccessFile din, int record, long message_offset31) thr
702718
hasHighResRHOData = true;
703719
dbpp9 = dbp7;
704720
} else {
705-
logger.warn("Missing radial product dbp7={} tname={}", dbp7, tname);
721+
missingName = tname;
722+
missingPointerNumber = 7;
723+
missingPointerValue = dbp7;
706724
}
707725
}
708726

@@ -728,7 +746,9 @@ public Level2Record(RandomAccessFile din, int record, long message_offset31) thr
728746
hasHighResRHOData = true;
729747
dbpp9 = dbp8;
730748
} else {
731-
logger.warn("Missing radial product dbp8={} tname={}", dbp8, tname);
749+
missingName = tname;
750+
missingPointerNumber = 8;
751+
missingPointerValue = dbp8;
732752
}
733753
}
734754

@@ -754,17 +774,31 @@ public Level2Record(RandomAccessFile din, int record, long message_offset31) thr
754774
hasHighResRHOData = true;
755775
dbpp9 = dbp9;
756776
} else {
757-
logger.warn("Missing radial product dbp9={} tname={}", dbp9, tname);
777+
missingName = tname;
778+
missingPointerNumber = 9;
779+
missingPointerValue = dbp9;
780+
}
781+
}
782+
783+
if (missingPointerNumber != -999) {
784+
// warn once per type per pointer location
785+
String missingKey = String.format("%s%d", missingName, missingPointerNumber);
786+
if (!missingProductWarned.contains(missingKey)) {
787+
String msg = String.format(
788+
"Unknown radial product dbp%d=%d tname=%s " + "(this is the only message you will see about this.)",
789+
missingPointerNumber, missingPointerValue, missingName);
790+
logger.warn(msg);
791+
missingProductWarned.add(missingKey);
758792
}
759793
}
760-
// hasHighResREFData = (dbp4 > 0);
761794

762795
if (hasHighResREFData) {
763796
reflectHR_gate_count = getDataBlockValue(din, (short) dbpp4, 8);
764797
reflectHR_first_gate = getDataBlockValue(din, (short) dbpp4, 10);
765798
reflectHR_gate_size = getDataBlockValue(din, (short) dbpp4, 12);
766799
ref_rf_threshold = getDataBlockValue(din, (short) dbpp4, 14);
767800
ref_snr_threshold = getDataBlockValue(din, (short) dbpp4, 16);
801+
reflectHR_data_word_size = getDataBlockByte(din, (short) dbpp4, 19);
768802
reflectHR_scale = getDataBlockValue1(din, (short) dbpp4, 20);
769803
reflectHR_addoffset = getDataBlockValue1(din, (short) dbpp4, 24);
770804
reflectHR_offset = (short) (dbpp4 + 28);
@@ -777,6 +811,7 @@ public Level2Record(RandomAccessFile din, int record, long message_offset31) thr
777811
velocityHR_gate_size = getDataBlockValue(din, (short) dbpp5, 12);
778812
vel_rf_threshold = getDataBlockValue(din, (short) dbpp5, 14);
779813
vel_snr_threshold = getDataBlockValue(din, (short) dbpp5, 16);
814+
velocityHR_data_word_size = getDataBlockByte(din, (short) dbpp5, 19);
780815
velocityHR_scale = getDataBlockValue1(din, (short) dbpp5, 20);
781816
velocityHR_addoffset = getDataBlockValue1(din, (short) dbpp5, 24);
782817
velocityHR_offset = (short) (dbpp5 + 28);
@@ -789,6 +824,7 @@ public Level2Record(RandomAccessFile din, int record, long message_offset31) thr
789824
spectrumHR_gate_size = getDataBlockValue(din, (short) dbpp6, 12);
790825
sw_rf_threshold = getDataBlockValue(din, (short) dbpp6, 14);
791826
sw_snr_threshold = getDataBlockValue(din, (short) dbpp6, 16);
827+
spectrumHR_data_word_size = getDataBlockByte(din, (short) dbpp6, 19);
792828
spectrumHR_scale = getDataBlockValue1(din, (short) dbpp6, 20);
793829
spectrumHR_addoffset = getDataBlockValue1(din, (short) dbpp6, 24);
794830
spectrumHR_offset = (short) (dbpp6 + 28);
@@ -801,6 +837,7 @@ public Level2Record(RandomAccessFile din, int record, long message_offset31) thr
801837
zdrHR_gate_size = getDataBlockValue(din, (short) dbpp7, 12);
802838
zdrHR_rf_threshold = getDataBlockValue(din, (short) dbpp7, 14);
803839
zdrHR_snr_threshold = getDataBlockValue(din, (short) dbpp7, 16);
840+
zdrHR_data_word_size = getDataBlockByte(din, (short) dbpp7, 19);
804841
zdrHR_scale = getDataBlockValue1(din, (short) dbpp7, 20);
805842
zdrHR_addoffset = getDataBlockValue1(din, (short) dbpp7, 24);
806843
zdrHR_offset = (short) (dbpp7 + 28);
@@ -812,6 +849,7 @@ public Level2Record(RandomAccessFile din, int record, long message_offset31) thr
812849
phiHR_gate_size = getDataBlockValue(din, (short) dbpp8, 12);
813850
phiHR_rf_threshold = getDataBlockValue(din, (short) dbpp8, 14);
814851
phiHR_snr_threshold = getDataBlockValue(din, (short) dbpp8, 16);
852+
phiHR_data_word_size = getDataBlockByte(din, (short) dbpp8, 19);
815853
phiHR_scale = getDataBlockValue1(din, (short) dbpp8, 20);
816854
phiHR_addoffset = getDataBlockValue1(din, (short) dbpp8, 24);
817855
phiHR_offset = (short) (dbpp8 + 28);
@@ -823,6 +861,7 @@ public Level2Record(RandomAccessFile din, int record, long message_offset31) thr
823861
rhoHR_gate_size = getDataBlockValue(din, (short) dbpp9, 12);
824862
rhoHR_rf_threshold = getDataBlockValue(din, (short) dbpp9, 14);
825863
rhoHR_snr_threshold = getDataBlockValue(din, (short) dbpp9, 16);
864+
rhoHR_data_word_size = getDataBlockByte(din, (short) dbpp9, 19);
826865
rhoHR_scale = getDataBlockValue1(din, (short) dbpp9, 20);
827866
rhoHR_addoffset = getDataBlockValue1(din, (short) dbpp9, 24);
828867
rhoHR_offset = (short) (dbpp9 + 28);
@@ -1069,6 +1108,24 @@ private short getDataOffset(int datatype) {
10691108
return Short.MIN_VALUE;
10701109
}
10711110

1111+
byte getDataWordSize(int datatype) {
1112+
switch (datatype) {
1113+
case REFLECTIVITY_HIGH:
1114+
return reflectHR_data_word_size;
1115+
case VELOCITY_HIGH:
1116+
return velocityHR_data_word_size;
1117+
case SPECTRUM_WIDTH_HIGH:
1118+
return spectrumHR_data_word_size;
1119+
case DIFF_REFLECTIVITY_HIGH:
1120+
return zdrHR_data_word_size;
1121+
case DIFF_PHASE:
1122+
return phiHR_data_word_size;
1123+
case CORRELATION_COEFFICIENT:
1124+
return rhoHR_data_word_size;
1125+
}
1126+
return 8;
1127+
}
1128+
10721129
private short getDataBlockValue(RandomAccessFile raf, short offset, int skip) throws IOException {
10731130
long off = offset + message_offset + MESSAGE_HEADER_SIZE;
10741131
raf.seek(off);
@@ -1090,6 +1147,13 @@ private float getDataBlockValue1(RandomAccessFile raf, short offset, int skip) t
10901147
return raf.readFloat();
10911148
}
10921149

1150+
private byte getDataBlockByte(RandomAccessFile raf, short offset, int skip) throws IOException {
1151+
long off = offset + message_offset + MESSAGE_HEADER_SIZE;
1152+
raf.seek(off);
1153+
raf.skipBytes(skip);
1154+
return raf.readByte();
1155+
}
1156+
10931157
public java.util.Date getDate() {
10941158
return getDate(data_julian_date, data_msecs);
10951159
}
@@ -1115,7 +1179,9 @@ public void readData(RandomAccessFile raf, int datatype, Range gateRange, IndexI
11151179
}
11161180

11171181
int dataCount = getGateCount(datatype);
1118-
if (datatype == DIFF_PHASE) {
1182+
byte wordSize = getDataWordSize(datatype);
1183+
1184+
if (wordSize == 16) {
11191185
short[] data = new short[dataCount];
11201186
raf.readShort(data, 0, dataCount);
11211187

@@ -1125,8 +1191,7 @@ public void readData(RandomAccessFile raf, int datatype, Range gateRange, IndexI
11251191
else
11261192
ii.setShortNext(data[gateIdx]);
11271193
}
1128-
} else {
1129-
1194+
} else if (wordSize == 8) {
11301195
byte[] data = new byte[dataCount];
11311196
raf.readFully(data);
11321197
// short [] ds = convertunsignedByte2Short(data);
@@ -1136,7 +1201,10 @@ public void readData(RandomAccessFile raf, int datatype, Range gateRange, IndexI
11361201
else
11371202
ii.setByteNext(data[gateIdx]);
11381203
}
1139-
1204+
} else {
1205+
String message = String.format("Data word size of %i bits not understood for data type %i (%s).", wordSize,
1206+
datatype, getDatatypeName(datatype));
1207+
throw new IOException(message);
11401208
}
11411209
}
11421210

‎cdm/radial/src/main/java/ucar/nc2/iosp/nexrad2/Nexrad2IOServiceProvider.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -283,7 +283,7 @@ public Variable makeVariable(NetcdfFile ncfile, int datatype, String shortName,
283283
dims.add(gateDim);
284284

285285
Variable v = new Variable(ncfile, null, null, shortName);
286-
if (datatype == DIFF_PHASE) {
286+
if (firstRecord.getDataWordSize(datatype) == 16) {
287287
v.setDataType(DataType.USHORT);
288288
} else {
289289
v.setDataType(DataType.UBYTE);

‎cdm/radial/src/test/java/ucar/nc2/iosp/nexrad2/Test16BitDataWidth.java

+54
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
package ucar.nc2.iosp.nexrad2;
2+
3+
import static com.google.common.truth.Truth.assertThat;
4+
5+
import java.io.IOException;
6+
import org.junit.Test;
7+
import org.junit.experimental.categories.Category;
8+
import ucar.ma2.Array;
9+
import ucar.ma2.InvalidRangeException;
10+
import ucar.ma2.MAMath;
11+
import ucar.ma2.MAMath.MinMax;
12+
import ucar.nc2.NetcdfFile;
13+
import ucar.nc2.NetcdfFiles;
14+
import ucar.nc2.Variable;
15+
import ucar.nc2.dataset.NetcdfDataset;
16+
import ucar.nc2.dataset.NetcdfDatasets;
17+
import ucar.unidata.util.test.TestDir;
18+
import ucar.unidata.util.test.category.NeedsCdmUnitTest;
19+
20+
@Category(NeedsCdmUnitTest.class)
21+
public class Test16BitDataWidth {
22+
23+
private static final String filename =
24+
TestDir.cdmUnitTestDir + "formats/nexrad/newLevel2/testfiles/Level2_KDDC_20201007_1914.ar2v";
25+
26+
@Test
27+
public void testNonPhiVar() throws IOException, InvalidRangeException {
28+
try (NetcdfFile ncf = NetcdfFiles.open(filename)) {
29+
// verified against metpy
30+
int expectedMin = 0;
31+
int expectedMax = 1058;
32+
Variable var = ncf.findVariable("DifferentialReflectivity_HI");
33+
Array data = var.read("0,:,:");
34+
MinMax minMax = MAMath.getMinMax(data);
35+
assertThat(minMax.min).isWithin(1e-6).of(expectedMin);
36+
assertThat(minMax.max).isWithin(1e-6).of(expectedMax);
37+
}
38+
}
39+
40+
@Test
41+
public void testNonPhiVarEnhanced() throws IOException, InvalidRangeException {
42+
try (NetcdfDataset ncf = NetcdfDatasets.openDataset(filename)) {
43+
// verified against metpy
44+
int expectedMin = -13;
45+
int expectedMax = 20;
46+
Variable var = ncf.findVariable("DifferentialReflectivity_HI");
47+
Array data = var.read("0,:,:");
48+
MinMax minMax = MAMath.getMinMax(data);
49+
assertThat(minMax.min).isWithin(1e-6).of(expectedMin);
50+
assertThat(minMax.max).isWithin(1e-6).of(expectedMax);
51+
}
52+
}
53+
54+
}

0 commit comments

Comments
 (0)