Simplify data embedding (#270)

This is an internal-only refactoring that changes the mechanics of how
the sdf/** file data is embedded into the library.

For one, it solves the "static initialization order fiasco" for the
embedded data. The data is only loaded into memory upon first use,
instead of as the library is being loaded.

It also simplifies the ruby embedding code, making it more concise.
I hope this is easier to maintain, but it also helps me when sharing
Drake with consumers that always build from source, but will not
install ruby as a compile-time prerequisite

The EmbeddedSdf codegen now emits the cc file with the data; the header
is stored in git. This provides an easier way to use a function to
retrieve the embedded strings as static locals.

The embedded SDF data is now combined into a single map.  Embedding
files should be boring, ala the WIP std::embed specification.  It
should not include application-layer logic encoding upgrade paths.

This will also make it easier to avoid the "static destruction fiasco"
in future commits, due to fewer functions to repair.

Signed-off-by: Jeremy Nimmer <jeremy.nimmer@tri.global>

Author: jwnimmer-tri

sdf/CMakeLists.txt

@@ -10,12 +10,12 @@ add_subdirectory(1.8)
 add_custom_target(schema)
 add_dependencies(schema schema1_8)
 
-# Generate the EmbeddedSdf.hh file, which contains all the supported SDF
+# Generate the EmbeddedSdf.cc file, which contains all the supported SDF
 # descriptions in a map of strings. The parser.cc file uses EmbeddedSdf.hh.
 execute_process(
   COMMAND ${RUBY} ${CMAKE_SOURCE_DIR}/sdf/embedSdf.rb
   WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}/sdf"
-  OUTPUT_FILE "${PROJECT_BINARY_DIR}/include/sdf/EmbeddedSdf.hh"
+  OUTPUT_FILE "${PROJECT_BINARY_DIR}/src/EmbeddedSdf.cc"
 )
 
 # Generate aggregated SDF description files for use by the sdformat.org 

sdf/embedSdf.rb

@@ -9,62 +9,37 @@
 supportedSdfConversions = ['1.8', '1.7', '1.6', '1.5', '1.4', '1.3']
 
 puts %q!
-#ifndef SDF_INTERNAL_EMBEDDEDSDF_HH_
-#define SDF_INTERNAL_EMBEDDEDSDF_HH_
+#include "EmbeddedSdf.hh"
 
-// An empty SDF string is returned if a query into the embeddedSdf map fails.
-static const std::string emptySdfString = "";
+namespace sdf {
+inline namespace SDF_VERSION_NAMESPACE {
 
-// A map of maps where the keys in the first/parent map are SDF version
-// strings, keys in the second/child map are SDF specification filenames and
-// values are the contents of the SDF specification files.
-static const std::map<std::string, std::map<std::string, std::string>> embeddedSdf = {
+const std::map<std::string, std::string> &GetEmbeddedSdf() {
+  static const std::map<std::string, std::string> result{
 !
 
-# Iterate over each version
-supportedSdfVersions.each do |version|
-  # Make sure the directory exists. Quietly fail so that we don't pollute
-  # the output, which gets included in EmbeddedSdf.hh
-  if Dir.exist?(version)
-    puts "{\"#{version}\", {"
-
-    # Iterate over each .sdf file in the version directory
-    Dir.glob("#{version}/*.sdf") do |file|
-
-      # Store the contents of the file in the child map
-      puts "{\"#{File.basename(file)}\", R\"__sdf_literal__("
-      infile = File.open(file)
-      puts infile.read
-      puts ")__sdf_literal__\"},"
-    end
-    puts "}},"
-  end
+# Stores the contents of the file in the map.
+def embed(pathname)
+  puts "{\"#{pathname}\", R\"__sdf_literal__("
+  infile = File.open(pathname)
+  puts infile.read
+  puts ")__sdf_literal__\"},"
 end
 
-puts "};"
-
-puts "static const std::map<std::string, std::pair<std::string, std::string>> conversionMap = {"
+# Embed the supported *.sdf files.
+supportedSdfVersions.each do |version|
+  Dir.glob("#{version}/*.sdf").sort.each { |file| embed(file) }
+end
 
-# Iterate over each version
+# Embed the supported *.convert files.
 supportedSdfConversions.each do |version|
-  # from-to
-  # Make sure the directory exists. Quietly fail so that we don't pollute
-  # the output, which gets included in EmbeddedSdf.hh
-  if Dir.exist?(version)
-
-    # Iterate over each .sdf file in the version directory
-    Dir.glob("#{version}/*.convert") do |file|
-
-      basename = File.basename(file, ".*").gsub(/_/, '.')
-      # Store the contents of the file in the child map
-      puts "{\"#{basename}\", {\"#{version}\", R\"__sdf_literal__("
-      infile = File.open(file)
-      puts infile.read
-      puts ")__sdf_literal__\"}},"
-    end
-  end
+  Dir.glob("#{version}/*.convert").sort.each { |file| embed(file) }
 end
 puts %q!
-};
-#endif
+  };
+  return result;
+}
+
+}
+}
 !

src/CMakeLists.txt

@@ -24,6 +24,7 @@ set (sources
   Converter.cc
   Cylinder.cc
   Element.cc
+  EmbeddedSdf.cc
   Error.cc
   Exception.cc
   Frame.cc
@@ -62,6 +63,7 @@ set (sources
   Visual.cc
   World.cc
 )
+include_directories(${CMAKE_CURRENT_SOURCE_DIR})
 
 if (USE_EXTERNAL_TINYXML)
   include_directories(${tinyxml_INCLUDE_DIRS})
@@ -157,7 +159,7 @@ if (NOT WIN32)
 endif()
 
 if (NOT WIN32)
-  set(SDF_BUILD_TESTS_EXTRA_EXE_SRCS Converter.cc)
+  set(SDF_BUILD_TESTS_EXTRA_EXE_SRCS Converter.cc EmbeddedSdf.cc)
   sdf_build_tests(Converter_TEST.cc)
 endif()
 

src/Converter.cc

@@ -29,12 +29,18 @@
 #include "sdf/Types.hh"
 
 #include "Converter.hh"
-
-// This include file is generated at configure time.
-#include "sdf/EmbeddedSdf.hh"
+#include "EmbeddedSdf.hh"
 
 using namespace sdf;
 
+namespace {
+bool EndsWith(const std::string& _a, const std::string& _b)
+{
+  return (_a.size() >= _b.size()) &&
+      (_a.compare(_a.size() - _b.size(), _b.size(), _b) == 0);
+}
+}
+
 /////////////////////////////////////////////////
 bool Converter::Convert(TiXmlDocument *_doc, const std::string &_toVersion,
                         bool _quiet)
@@ -73,42 +79,47 @@ bool Converter::Convert(TiXmlDocument *_doc, const std::string &_toVersion,
 
   elem->SetAttribute("version", _toVersion);
 
-  // The conversionMap in EmbeddedSdf.hh has keys that represent a version
-  // of SDF to convert from. The values in conversionmap are pairs, where
-  // the first element is the SDF version that the second element will
-  // convert to. For example, the following will convert from 1.4 to 1.5
-  // according to "conversion_xml":
-  //
-  // {"1.4", {"1.5", "conversion_xml"}}
-  std::map<std::string, std::pair<std::string, std::string> >::const_iterator
-    fromIter = conversionMap.find(origVersion);
-
-  std::string toVer = "";
+  // The conversion recipes within the embedded files database are named, e.g.,
+  // "1.8/1_7.convert" to upgrade from 1.7 to 1.8.
+  const std::map<std::string, std::string> &embedded = GetEmbeddedSdf();
 
-  // Starting with the original SDF version, perform all the conversions
-  // necessary in order to reach the _toVersion.
-  while (fromIter != conversionMap.end() && fromIter->first != _toVersion)
+  // Apply the conversions one at a time until we reach the desired _toVersion.
+  std::string curVersion = origVersion;
+  while (curVersion != _toVersion)
   {
-    // Get the SDF to version.
-    toVer = fromIter->second.first;
+    // Find the (at most one) file named, e.g., ".../1_7.convert".
+    std::string snakeVersion = curVersion;
+    std::replace(snakeVersion.begin(), snakeVersion.end(), '.', '_');
+    const std::string suffix = "/" + snakeVersion + ".convert";
+    const char* convertXml = nullptr;
+    for (const auto& [pathname, data] : embedded)
+    {
+      if (EndsWith(pathname, suffix))
+      {
+        curVersion = pathname.substr(0, pathname.size() - suffix.size());
+        convertXml = data.c_str();
+        break;
+      }
+    }
+    if (convertXml == nullptr)
+    {
+      break;
+    }
 
     // Parse and apply the conversion XML.
     TiXmlDocument xmlDoc;
-    xmlDoc.Parse(fromIter->second.second.c_str());
+    xmlDoc.Parse(convertXml);
     if (xmlDoc.Error())
     {
       sdferr << "Error parsing XML from string: "
              << xmlDoc.ErrorDesc() << '\n';
       return false;
     }
     ConvertImpl(elem, xmlDoc.FirstChildElement("convert"));
-
-    // Get the next conversion XML map element.
-    fromIter = conversionMap.find(toVer);
   }
 
   // Check that we actually converted to the desired final version.
-  if (toVer != _toVersion)
+  if (curVersion != _toVersion)
   {
     sdferr << "Unable to convert from SDF version " << origVersion
            << " to " << _toVersion << "\n";

src/EmbeddedSdf.hh

@@ -0,0 +1,40 @@
+/*
+ * Copyright 2020 Open Source Robotics Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+#ifndef SDF_EMBEDDEDSDF_HH_
+#define SDF_EMBEDDEDSDF_HH_
+
+#include <map>
+#include <string>
+
+#include "sdf/Types.hh"
+
+namespace sdf
+{
+  // Inline bracket to help doxygen filtering.
+  inline namespace SDF_VERSION_NAMESPACE {
+  //
+
+  /// \internal
+
+  /// A map where the keys are a source-relative pathnames within the "sdf"
+  /// directory such as "1.8/root.sdf", and the values are the contents of
+  /// that source file.
+  const std::map<std::string, std::string> &GetEmbeddedSdf();
+}
+}
+#endif

src/SDF.cc

@@ -31,9 +31,7 @@
 #include "sdf/SDFImpl.hh"
 #include "SDFImplPrivate.hh"
 #include "sdf/sdf_config.h"
-
-// This include file is generated at configure time.
-#include "sdf/EmbeddedSdf.hh"
+#include "EmbeddedSdf.hh"
 
 namespace sdf
 {
@@ -453,14 +451,18 @@ const std::string &SDF::EmbeddedSpec(
 {
   try
   {
-    return embeddedSdf.at(SDF::Version()).at(_filename);
+    const std::string pathname = SDF::Version() + "/" + _filename;
+    return GetEmbeddedSdf().at(pathname);
   }
   catch(const std::out_of_range &)
   {
     if (!_quiet)
       sdferr << "Unable to find SDF filename[" << _filename << "] with "
         << "version " << SDF::Version() << "\n";
   }
+
+  // An empty SDF string is returned if a query into the embeddedSdf map fails.
+  static const std::string emptySdfString;
   return emptySdfString;
 }
 }