Merge pull request #1344 from Mikhail-Aoun/Add_VLen_Writing_feature

Add VLEN writing feature and test.

Author: lesserwhirls

cdm/src/test/data/vlen-test/SGB1-HKT-00-SRC_C_EUMT_20181109084333_G_O_20170103000340_20170103000359_O_N____.nc

@@ -33,8 +33,9 @@
 package ucar.nc2;
 
 import java.io.*;
-
+import java.util.HashMap;
 import ucar.ma2.Array;
+import ucar.ma2.InvalidRangeException;
 import ucar.nc2.constants.CDM;
 
 /**
@@ -106,4 +107,158 @@ static public boolean close( float d1, float d2) {
     else
       return Math.abs(d1-d2) < 1.0e-5;
   }
+
+  /**
+   * Clone metadata from the given netcdf to the given netcdf file writer.
+   * 
+   * @param fromNetcdf the cloned netcdf
+   * @param toWriter the netcdf file writer
+   */
+  public static void cloneMetadata(NetcdfFile fromNetcdf, NetcdfFileWriter toWriter) {
+
+    final Group rootFrom = fromNetcdf.getRootGroup();
+    final Group rootTo = toWriter.addGroup(null, null);
+
+    cloneMetadata(toWriter, rootFrom, rootTo);
+  }
+
+  /**
+   * Clone data from the given netcdf to the given netcdf file writer.
+   * 
+   * @param fromNetcdf the cloned netcdf
+   * @param toWriter the netcdf file writer
+   * @throws IOException
+   * @throws InvalidRangeException
+   */
+  public static void cloneData(NetcdfFile fromNetcdf, NetcdfFileWriter toWriter)
+      throws IOException, InvalidRangeException {
+
+    final Group rootFrom = fromNetcdf.getRootGroup();
+    final Group rootTo = toWriter.getNetcdfFile().getRootGroup();
+
+    cloneData(toWriter, rootFrom, rootTo);
+
+  }
+
+  /**
+   * Clone metadata from the given group to the given group of the netcdf file writer.
+   * 
+   * @param to the netcdf file writer
+   * @param groupFrom the cloned metadata group
+   * @param groupTo the written group
+   */
+  private static void cloneMetadata(final NetcdfFileWriter to, final Group groupFrom, final Group groupTo) {
+
+    // dimension
+    cloneMetadataDimension(to, groupFrom, groupTo);
+
+    // attribute
+    cloneMetadataAttribute(to, groupFrom, groupTo);
+
+    // variable
+    cloneMetadataVariable(to, groupFrom, groupTo);
+
+    // enum
+    cloneMetadataEnum(to, groupFrom, groupTo);
+
+    for (final Group gFrom : groupFrom.getGroups()) {
+      // create new Group
+      final Group g = to.addGroup(groupTo, gFrom.getShortName());
+      cloneMetadata(to, gFrom, g);
+
+    }
+  }
+
+  /**
+   * Clone metadata dimension.
+   *
+   * @param to the netcdf file writer
+   * @param groupFrom the cloned metadata dimension group
+   * @param groupTo the written group
+   */
+  private static void cloneMetadataDimension(final NetcdfFileWriter to, final Group groupFrom, final Group groupTo) {
+    for (final Dimension dimension : groupFrom.getDimensions()) {
+      if (!dimension.isVariableLength()) {
+        to.addDimension(groupTo, dimension.getShortName(), dimension.getLength(), dimension.isShared(),
+            dimension.isUnlimited(), dimension.isVariableLength());
+      }
+    }
+  }
+
+  /**
+   * Clone metadata attribute.
+   *
+   * @param to the netcdf file writer
+   * @param groupFrom the cloned metadata attribute group
+   * @param groupTo the written group
+   */
+  private static void cloneMetadataAttribute(final NetcdfFileWriter to, final Group groupFrom, final Group groupTo) {
+    for (final Attribute attribute : groupFrom.getAttributes()) {
+      final Attribute att = new Attribute(attribute.getShortName(), attribute);
+      to.addGroupAttribute(groupTo, att);
+    }
+  }
+
+  /**
+   * Clone metadata variable.
+   *
+   * @param to the netcdf file writer
+   * @param groupFrom the cloned metadata variable group
+   * @param groupTo the written group
+   */
+  private static void cloneMetadataVariable(final NetcdfFileWriter to, final Group groupFrom, final Group groupTo) {
+    for (final Variable variable : groupFrom.getVariables()) {
+
+      final String parsedDim = variable.getDimensionsString();
+      final Variable v = to.addVariable(groupTo, variable.getShortName(), variable.getDataType(), parsedDim);
+
+      for (final Attribute attribute : variable.getAttributes()) {
+        final Attribute att = new Attribute(attribute.getShortName(), attribute);
+        v.addAttribute(att);
+      }
+    }
+
+  }
+
+  /**
+   * Clone metadata enum.
+   *
+   * @param to the netcdf file writer
+   * @param groupFrom the cloned metadata enum group
+   * @param groupTo the written group
+   */
+  private static void cloneMetadataEnum(final NetcdfFileWriter to, final Group groupFrom, final Group groupTo) {
+    for (final EnumTypedef enumTypedef : groupFrom.getEnumTypedefs()) {
+      final EnumTypedef e =
+          new EnumTypedef(enumTypedef.getShortName(), new HashMap<>(enumTypedef.getMap()), enumTypedef.getBaseType());
+      to.addTypedef(groupTo, e);
+    }
+  }
+
+  /**
+   * Clone data.
+   * 
+   * @param to the netcdf file writer
+   * @param groupFrom the cloned metadata enum group
+   * @param groupTo the written group
+   * @throws IOException
+   * @throws InvalidRangeException
+   */
+  private static void cloneData(final NetcdfFileWriter to, final Group groupFrom, final Group groupTo)
+      throws IOException, InvalidRangeException {
+    if (null != groupTo) {
+      for (final Variable variableFrom : groupFrom.getVariables()) {
+
+        final Variable variableTo = groupTo.findVariable(variableFrom.getShortName());
+        if (null != variableTo) {
+          to.write(variableTo, variableFrom.read().copy());
+        }
+      }
+
+      for (final Group gFrom : groupFrom.getGroups()) {
+        final Group gTo = groupTo.findGroup(gFrom.getShortName());
+        cloneData(to, gFrom, gTo);
+      }
+    }
+  }
 }

cdm/src/test/java/ucar/nc2/TestUtils.java

@@ -36,7 +36,8 @@
 import org.junit.rules.TemporaryFolder;
 import org.junit.runners.MethodSorters;
 import ucar.ma2.*;
-
+import ucar.unidata.util.test.TestDir;
+import java.io.File;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
@@ -1040,4 +1041,31 @@ public void testRecordSizeBug() throws IOException, InvalidRangeException {
       ucar.unidata.util.test.CompareNetcdf.compareData(result1, result2);
     }
   }
+
+  /** Test the cloning of files with vlen in order to demonstrate vlen variable writting. */
+  @Test
+  public void testNC4CloningVlen() {
+
+    // GIVEN
+    final String filename = tempFolder.getRoot().getAbsolutePath() + "/test-vlen.nc";
+
+    // -- A nc with VLEN
+    final NetcdfFile ncfile = TestDir
+        .openFileLocal("vlen-test/SGB1-HKT-00-SRC_C_EUMT_20181109084333_G_O_20170103000340_20170103000359_O_N____.nc");
+
+    // WHEN
+    try (NetcdfFileWriter writer = NetcdfFileWriter.createNew(NetcdfFileWriter.Version.netcdf4, filename)) {
+      TestUtils.cloneMetadata(ncfile, writer);
+      writer.create();
+      TestUtils.cloneData(ncfile, writer);
+
+    } catch (IOException | InvalidRangeException e) {
+      Assert.fail(e.getMessage());
+    }
+
+    // THEN
+    File f = new File(filename);
+    Assert.assertTrue(f.exists());
+  }
+
 }