GRIB table updates

cdm-test/src/test/java/ucar/nc2/grib/TestGribCreationOptions.java

@@ -44,16 +44,16 @@ public void testTimeUnitOption() throws Exception {
      * olderThan="5 min"/>
      * 
      * <tdm rewrite="test" rescan="0 0/15 * * * ? *"/>
-     * <gribConfig>
+     * <gribConfig>TestRotatedPole
      * <option name="timeUnit" value="1 minute" />
      * </gribConfig>
      * </featureCollection>
      */
 
     String dataset = TestDir.cdmUnitTestDir + "gribCollections/hrrr/DewpointTempFromGsdHrrrrConus3surface.grib2";
     try (NetcdfDataset ds = NetcdfDatasets.openDataset(dataset)) {
-      Variable v = ds.findVariable("Dewpoint_temperature_height_above_ground");
-      Assert.assertNotNull("Dewpoint_temperature_height_above_ground", v);
+      Variable v = ds.findVariable("DPT_height_above_ground");
+      Assert.assertNotNull("DPT_height_above_ground", v);
       Dimension d = v.getDimension(0);
       Assert.assertEquals(57, d.getLength());
     }

cdm/s3/src/test/java/thredds/inventory/s3/TestMFileS3.java

@@ -105,7 +105,7 @@ public void bucketAndKeyGcs() throws IOException {
 
   @Test
   public void bucketAndKeyOsdc() throws IOException {
-    long lastModified = 1603308465000L;
+    long lastModified = 1611593614000L;
     checkWithBucketAndKey(OSDC_G16_S3_OBJECT_1, OSDC_G16_OBJECT_KEY_1, null, lastModified);
     checkWithBucketAndKey(OSDC_G16_S3_OBJECT_1 + DELIMITER_FRAGMENT, G16_NAME_1, "/", lastModified);
   }

grib/src/main/java/ucar/nc2/grib/grib2/table/FslHrrrLocalTables.java

@@ -1,12 +1,13 @@
 /*
- * Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata
+ * Copyright (c) 1998-2020 John Caron and University Corporation for Atmospheric Research/Unidata
  * See LICENSE for license information.
  */
 
 package ucar.nc2.grib.grib2.table;
 
 import com.google.common.collect.ImmutableList;
 import java.nio.charset.StandardCharsets;
+import org.checkerframework.checker.fenum.qual.SwingVerticalOrientation;
 import ucar.nc2.grib.GribTables;
 import ucar.nc2.grib.grib2.Grib2Parameter;
 import java.io.BufferedReader;
@@ -43,6 +44,11 @@ public ImmutableList<Parameter> getParameters() {
     return getLocalParameters();
   }
 
+  @Override
+  public GribTables.Parameter getParameter(int discipline, int category, int number) {
+    return localParams.get(makeParamId(discipline, category, number));
+  }
+
   @Override
   public GribTables.Parameter getParameterRaw(int discipline, int category, int number) {
     return localParams.get(makeParamId(discipline, category, number));
@@ -117,8 +123,60 @@ public void lookForProblems(Formatter f) {
     initLocalTable(f);
   }
 
+  private static Grib2Parameter parseVersion4(String[] flds, Formatter f) {
+    // V4:
+    // Record, MasterTable, LocalTable, Discipline, Category, Parameter, WGrib2Name, NCLName,
+    // FieldType, VerticalLevels, Units,
+    String recordNumber = flds[0].trim();
+    int masterTableNumber = Integer.parseInt(flds[1].trim());
+    int localTableNumber = Integer.parseInt(flds[2].trim());
+    int disciplineNumber = Integer.parseInt(flds[3].trim());
+    int categoryNumber = Integer.parseInt(flds[4].trim());
+    int parameterNumber = Integer.parseInt(flds[5].trim());
+
+    String WGrib2Name = flds[6].trim();
+    String NCLName = flds[7].trim();
+    String FieldType = flds[8].trim(); // closest we have to a description in version 4 of the FSL table
+    String VerticalLevels = flds[9].trim();
+    String Units = flds[10].trim();
+    if (f != null) {
+      f.format("%3s %3d %3d %3d %3d %3d %-10s %-25s %-30s %-100s %-20s%n", recordNumber, masterTableNumber,
+          localTableNumber, disciplineNumber, categoryNumber, parameterNumber, WGrib2Name, NCLName, FieldType,
+          VerticalLevels, Units);
+    }
+
+    String name = !WGrib2Name.equals("var") ? WGrib2Name : FieldType;
+    return new Grib2Parameter(disciplineNumber, categoryNumber, parameterNumber, name, Units, null, FieldType);
+  }
+
+  private static Grib2Parameter parsePre4(String[] flds, Formatter f) {
+    // v1, v2:
+    // RecordNumber, TableNumber, DisciplineNumber, CategoryNumber, ParameterNumber, WGrib2Name, NCLName,
+    // FieldType, Description, Units,
+    String recordNumber = flds[0].trim();
+    int tableNumber = Integer.parseInt(flds[1].trim());
+    int disciplineNumber = Integer.parseInt(flds[2].trim());
+    int categoryNumber = Integer.parseInt(flds[3].trim());
+    int parameterNumber = Integer.parseInt(flds[4].trim());
+
+    String WGrib2Name = flds[5].trim();
+    String NCLName = flds[6].trim();
+    String FieldType = flds[7].trim();
+    String Description = flds[8].trim();
+    String Units = flds[9].trim();
+    if (f != null) {
+      f.format("%3s %3d %3d %3d %3d %-10s %-25s %-30s %-100s %-20s%n", recordNumber, tableNumber, disciplineNumber,
+          categoryNumber, parameterNumber, WGrib2Name, NCLName, FieldType, Description, Units);
+    }
+
+    String name = !WGrib2Name.equals("var") ? WGrib2Name : FieldType;
+    return new Grib2Parameter(disciplineNumber, categoryNumber, parameterNumber, name, Units, null, Description);
+  }
+
   private Map<Integer, Grib2Parameter> readCsv(String resourcePath, Formatter f) {
     boolean header = true;
+    int version = resourcePath.endsWith("hrrr4.csv") ? 4 : -1;
+
     Map<Integer, Grib2Parameter> result = new HashMap<>(100);
 
     ClassLoader cl = getClass().getClassLoader();
@@ -146,39 +204,41 @@ private Map<Integer, Grib2Parameter> readCsv(String resourcePath, Formatter f) {
           if ((line.isEmpty()) || line.startsWith("#")) {
             continue;
           }
-          String[] flds = line.split(",");
 
-          // RecordNumber, TableNumber, DisciplineNumber, CategoryNumber, ParameterNumber, WGrib2Name, NCLName,
-          // FieldType, Description, Units,
-          String recordNumber = flds[0].trim();
-          int tableNumber = Integer.parseInt(flds[1].trim());
-          int disciplineNumber = Integer.parseInt(flds[2].trim());
-          int categoryNumber = Integer.parseInt(flds[3].trim());
-          int parameterNumber = Integer.parseInt(flds[4].trim());
-
-          String WGrib2Name = flds[5].trim();
-          String NCLName = flds[6].trim();
-          String FieldType = flds[7].trim();
-          String Description = flds[8].trim();
-          String Units = flds[9].trim();
-          if (f != null) {
-            f.format("%3s %3d %3d %3d %3d %-10s %-25s %-30s %-100s %-20s%n", recordNumber, tableNumber,
-                disciplineNumber, categoryNumber, parameterNumber, WGrib2Name, NCLName, FieldType, Description, Units);
+          String[] flds = line.split(",");
+          // starting with version 4, the HRRR parameter table includes multiple tables:
+          // HRRR 2-D Hourly
+          // HRRR 2-D Sub-hrly
+          // HRRR 3-D Native Level
+          // HRRR 3-D Isobaric Level
+          //
+          // To make updating simple, the tables are concatenated in a single file, which leads to
+          // the presence of multiple headers. This allows us to skip the internal headers of the csv "table".
+          // Also, some of the subtables include garbage, like the CONUS 3-D Native Level Fields table, which has
+          // things like:
+          // 1-14 repeats for remaining 49 native sigma levels...
+          // and this will allow us to skip those as well.
+          if ((version >= 4) && (flds.length != 11) || line.startsWith("Record")) {
+            continue;
           }
 
-          String name = !WGrib2Name.equals("var") ? WGrib2Name : FieldType;
-          Grib2Parameter s =
-              new Grib2Parameter(disciplineNumber, categoryNumber, parameterNumber, name, Units, null, Description);
-          // s.desc = Description;
-          result.put(makeParamId(disciplineNumber, categoryNumber, parameterNumber), s);
+          Grib2Parameter s = version == 4 ? parseVersion4(flds, f) : parsePre4(flds, f);
+
+          result.put(makeParamId(s.getDiscipline(), s.getCategory(), s.getNumber()), s);
           if (f != null) {
             f.format(" %s%n", s);
           }
-          if (categoryNumber > 191 || parameterNumber > 191) {
+          if (s.getCategory() > 191 || s.getNumber() > 191) {
             Grib2Parameter dup = names.get(s.getName());
             if (dup != null && f != null) {
               if (header) {
-                f.format("Problems in table %s %n", resourcePath);
+                if (version >= 4) {
+                  // Duplicate entries are only potentially an issue in version 4 and above.
+                  // Need to examine them to know for sure.
+                  f.format("Potential problem in table %s %n", resourcePath);
+                } else {
+                  f.format("Problems in table %s %n", resourcePath);
+                }
               }
               header = false;
               f.format(" DUPLICATE NAME %s and %s (%s)%n", s.getId(), dup.getId(), s.getName());