[Gazebo 11] Allow gazebo to download models from Fuel in the sdf files (#2822)

Signed-off-by: ahcorde <ahcorde@gmail.com>
Signed-off-by: Louise Poubel <louise@openrobotics.org>

Co-authored-by: Louise Poubel <louise@openrobotics.org>

Author: ahcorde

gazebo/Server.cc

@@ -33,7 +33,15 @@
 #include <sdf/sdf.hh>
 
 #include <ignition/math/Rand.hh>
-#include "ignition/common/Profiler.hh"
+#include <ignition/common/Profiler.hh>
+#include <ignition/common/Filesystem.hh>
+#include <ignition/common/URI.hh>
+#ifdef _WIN32
+  // DELETE is defined in winnt.h and causes a problem with
+  // ignition::fuel_tools::REST::DELETE
+  #undef DELETE
+#endif
+#include <ignition/fuel_tools/Interface.hh>
 
 #include "gazebo/gazebo.hh"
 #include "gazebo/transport/transport.hh"
@@ -385,15 +393,6 @@ bool Server::GetInitialized() const
 bool Server::LoadFile(const std::string &_filename,
                       const std::string &_physics)
 {
-  // Quick test for a valid file
-  FILE *test = fopen(common::find_file(_filename).c_str(), "r");
-  if (!test)
-  {
-    gzerr << "Could not open file[" << _filename << "]\n";
-    return false;
-  }
-  fclose(test);
-
   // Load the world file
   sdf::SDFPtr sdf(new sdf::SDF);
   if (!sdf::init(sdf))
@@ -402,10 +401,53 @@ bool Server::LoadFile(const std::string &_filename,
     return false;
   }
 
-  if (!sdf::readFile(common::find_file(_filename), sdf))
+  std::string filename = _filename;
+  auto filenameUri = ignition::common::URI(filename);
+
+  if (filenameUri.Scheme() == "http" || filenameUri.Scheme() == "https")
   {
-    gzerr << "Unable to read sdf file[" << _filename << "]\n";
-    return false;
+    std::string downloadedDir = common::FuelModelDatabase::Instance()->WorldPath(filename);
+    // Find the first sdf file in the world path for now, the later intention is
+    // to load an optional world config file first and if that does not exist,
+    // continue to load the first sdf file found as done below
+    for (ignition::common::DirIter file(downloadedDir);
+         file != ignition::common::DirIter(); ++file)
+    {
+      std::string current(*file);
+      if (ignition::common::isFile(current))
+      {
+        std::string fileName = ignition::common::basename(current);
+        std::string::size_type fileExtensionIndex = fileName.rfind(".");
+        std::string fileExtension = fileName.substr(fileExtensionIndex + 1);
+
+        if (fileExtension == "sdf" || fileExtension == "world")
+        {
+          filename = current.c_str();
+          if (!sdf::readFile(filename, sdf))
+          {
+            gzerr << "Unable to read SDF from URL[" << filename << "]\n";
+            return false;
+          }
+        }
+      }
+    }
+  }
+  else
+  {
+    // Quick test for a valid file
+    FILE *test = fopen(common::find_file(filename).c_str(), "r");
+    if (!test)
+    {
+      gzerr << "Could not open file[" << filename << "]\n";
+      return false;
+    }
+    fclose(test);
+
+    if (!sdf::readFile(common::find_file(filename), sdf))
+    {
+      gzerr << "Unable to read sdf file[" << filename << "]\n";
+      return false;
+    }
   }
 
   return this->LoadImpl(sdf->Root(), _physics);

gazebo/common/CommonIface.cc

@@ -44,8 +44,16 @@
 #include "gazebo/common/Console.hh"
 #include "gazebo/common/CommonIface.hh"
 #include "gazebo/common/Exception.hh"
+#include "gazebo/common/FuelModelDatabase.hh"
 #include "gazebo/common/SystemPaths.hh"
 
+#ifdef _WIN32
+  // DELETE is defined in winnt.h and causes a problem with
+  // ignition::fuel_tools::REST::DELETE
+  #undef DELETE
+#endif
+#include "ignition/fuel_tools/Interface.hh"
+
 using namespace gazebo;
 
 #ifdef _WIN32
@@ -156,13 +164,23 @@ void common::add_search_path_suffix(const std::string &_suffix)
 /////////////////////////////////////////////////
 std::string common::find_file(const std::string &_file)
 {
-  return common::SystemPaths::Instance()->FindFile(_file, true);
+  std::string path = common::FuelModelDatabase::Instance()->ModelPath(_file);
+  if (path.empty())
+  {
+    path = common::SystemPaths::Instance()->FindFile(_file, true);
+  }
+  return path;
 }
 
 /////////////////////////////////////////////////
 std::string common::find_file(const std::string &_file, bool _searchLocalPath)
 {
-  return common::SystemPaths::Instance()->FindFile(_file, _searchLocalPath);
+  std::string path = common::FuelModelDatabase::Instance()->ModelPath(_file);
+  if (path.empty())
+  {
+    path = common::SystemPaths::Instance()->FindFile(_file, _searchLocalPath);
+  }
+  return path;
 }
 
 /////////////////////////////////////////////////

gazebo/common/FuelModelDatabase.cc

@@ -26,7 +26,14 @@
 
 #include <ignition/common/Console.hh>
 #include <ignition/common/Filesystem.hh>
+#include <ignition/common/URI.hh>
+#ifdef _WIN32
+  // DELETE is defined in winnt.h and causes a problem with
+  // ignition::fuel_tools::REST::DELETE
+  #undef DELETE
+#endif
 #include <ignition/fuel_tools/FuelClient.hh>
+#include <ignition/fuel_tools/WorldIdentifier.hh>
 #include <sdf/sdf.hh>
 
 #include "gazebo/gazebo_config.h"
@@ -208,31 +215,90 @@ std::string FuelModelDatabase::ModelPath(const std::string &_uri,
 
   if (fuelUri.Scheme() != "http" && fuelUri.Scheme() != "https")
   {
-    gzwarn << "URI not supported by Fuel [" << _uri << "]" << std::endl;
     return std::string();
   }
 
   std::string path;
+  std::string fileUrl;
+  ignition::fuel_tools::ModelIdentifier model;
+  using namespace ignition::fuel_tools;
 
-  if (!_forceDownload)
+  if ((this->dataPtr->fuelClient->ParseModelUrl(fuelUri, model) &&
+       !this->dataPtr->fuelClient->CachedModel(fuelUri, path))
+      || _forceDownload)
   {
-    if (this->dataPtr->fuelClient->CachedModel(fuelUri, path))
+    Result result_download =
+      this->dataPtr->fuelClient->DownloadModel(fuelUri, path);
+    if (result_download != Result(ResultType::FETCH) ||
+      result_download != Result(ResultType::FETCH_ALREADY_EXISTS))
+    {
+      gzerr << "Unable to download model[" << _uri << "]" << std::endl;
+      return std::string();
+    }
+    else
     {
-      return path;
+        gzmsg << "Downloaded model URL: " << std::endl
+              << "          " << _uri << std::endl
+              << "      to: " << std::endl
+              << "          " << path << std::endl;
+    }
+  }
+  if (path.empty()) {
+    if ((this->dataPtr->fuelClient->ParseModelFileUrl(fuelUri, model, fileUrl) &&
+        !this->dataPtr->fuelClient->CachedModelFile(fuelUri, path))
+      || _forceDownload)
+    {
+      auto modelUri = _uri.substr(0,
+          _uri.find("files", model.UniqueName().size())-1);
+      Result result_download =
+        this->dataPtr->fuelClient->DownloadModel(ignition::common::URI(modelUri), path);
+      if (result_download != Result(ResultType::FETCH) ||
+        result_download != Result(ResultType::FETCH_ALREADY_EXISTS))
+      {
+        gzerr << "Unable to download model[" << _uri << "]" << std::endl;
+        return std::string();
+      }
+      else
+      {
+          gzmsg << "Downloaded model URL: " << std::endl
+                << "          " << _uri << std::endl
+                << "      to: " << std::endl
+                << "          " << path << std::endl;
+          path += "/" + fileUrl;
+      }
     }
   }
 
-  if (!this->dataPtr->fuelClient->DownloadModel(fuelUri, path))
+  return path;
+}
+
+/////////////////////////////////////////////////
+std::string FuelModelDatabase::WorldPath(const std::string &_uri, const bool _forceDownload)
+{
+  auto fuelUri = ignition::common::URI(_uri);
+
+  if (fuelUri.Scheme() != "http" && fuelUri.Scheme() != "https")
   {
-    gzerr << "Unable to download model[" << _uri << "]" << std::endl;
     return std::string();
   }
-  else
+
+  std::string path;
+  std::string fileUrl;
+  ignition::fuel_tools::WorldIdentifier world;
+
+  if ((this->dataPtr->fuelClient->ParseWorldUrl(fuelUri, world) &&
+      !this->dataPtr->fuelClient->CachedWorld(fuelUri, path)) || _forceDownload)
   {
-      gzmsg << "Downloaded model URL: " << std::endl
-            << "          " << _uri << std::endl
-            << "      to: " << std::endl
-            << "          " << path << std::endl;
+    this->dataPtr->fuelClient->DownloadWorld(fuelUri, path);
+  }
+  // Download the world, if it's a world file URI
+  else if (this->dataPtr->fuelClient->ParseWorldFileUrl(fuelUri, world, fileUrl) &&
+      !this->dataPtr->fuelClient->CachedWorldFile(fuelUri, path))
+  {
+    auto worldUri = _uri.substr(0,
+        _uri.find("files", world.UniqueName().size())-1);
+    this->dataPtr->fuelClient->DownloadWorld(ignition::common::URI(worldUri), path);
+    path += "/" + fileUrl;
   }
 
   return path;
@@ -256,4 +322,3 @@ std::string FuelModelDatabase::CachedFilePath(const std::string &_uri)
 
   return path;
 }
-

gazebo/common/FuelModelDatabase.hh

@@ -21,6 +21,11 @@
 #include <memory>
 #include <string>
 #include <vector>
+#ifdef _WIN32
+  // DELETE is defined in winnt.h and causes a problem with
+  // ignition::fuel_tools::REST::DELETE
+  #undef DELETE
+#endif
 #include <ignition/fuel_tools/ClientConfig.hh>
 #include <ignition/fuel_tools/ModelIdentifier.hh>
 
@@ -103,6 +108,16 @@ namespace gazebo
       public: std::string ModelPath(const std::string &_uri,
         const bool _forceDownload = false);
 
+      /// \brief Get the local path to a world.
+      ///
+      /// Get the path to a world based on a URI. If the world is on
+      /// a remote server, then the model fetched and installed locally.
+      /// \param[in] _uri the model uri
+      /// \param[in] _forceDownload True to skip searching local cache.
+      /// \return Local path to a world directory
+      public: std::string WorldPath(const std::string &_uri,
+        const bool _forceDownload = false);
+
       /// \brief Get the full local path to a cached file based on its URI.
       /// \param[in] _uri The file's URI
       /// \return Local path to the file

gazebo/gui/InsertModelWidget.cc

@@ -48,21 +48,6 @@ using namespace gui;
 
 static bool gInsertModelWidgetDeleted = false;
 
-/////////////////////////////////////////////////
-// TODO: Remove this once Fuel support is fully functional
-bool usingFuel()
-{
-  auto useFuel = std::getenv("USE_IGNITION_FUEL");
-  if (!useFuel || *useFuel == '\0')
-    return false;
-
-  std::string useFuelStr(useFuel);
-  std::transform(useFuelStr.begin(), useFuelStr.end(),
-                 useFuelStr.begin(), ::tolower);
-
-  return useFuelStr != "false" && useFuelStr != "0";
-}
-
 /////////////////////////////////////////////////
 InsertModelWidget::InsertModelWidget(QWidget *_parent)
 : QWidget(_parent), dataPtr(new InsertModelWidgetPrivate)
@@ -523,9 +508,6 @@ bool InsertModelWidget::IsPathAccessible(const boost::filesystem::path &_path)
 /////////////////////////////////////////////////
 void InsertModelWidget::InitializeFuelServers()
 {
-  if (!usingFuel())
-    return;
-
   // Get the list of Ignition Fuel servers.
   auto servers = common::FuelModelDatabase::Instance()->Servers();
 
@@ -550,9 +532,6 @@ void InsertModelWidget::InitializeFuelServers()
 /////////////////////////////////////////////////
 void InsertModelWidget::PopulateFuelServers()
 {
-  if (!usingFuel())
-    return;
-
   // Get the list of Ignition Fuel servers.
   auto servers = common::FuelModelDatabase::Instance()->Servers();
 

gazebo/rendering/RenderEngine.cc

@@ -465,7 +465,7 @@ void RenderEngine::LoadPlugins()
 /////////////////////////////////////////////////
 void RenderEngine::AddResourcePath(const std::string &_uri)
 {
-  if (_uri == "__default__" || _uri.empty())
+  if (_uri.find("__default__") != std::string::npos || _uri.empty())
     return;
 
   std::string path = common::find_file_path(_uri);