From 0c4337d202b27da5a49f001d5f33950532351a3d Mon Sep 17 00:00:00 2001 From: Louise Poubel Date: Thu, 25 Jun 2020 20:14:30 -0700 Subject: [PATCH 01/14] Accept relative paths in SDF files Signed-off-by: Louise Poubel --- gazebo/common/CMakeLists.txt | 4 ++ gazebo/common/CommonIface.cc | 79 ++++++++++++++++++++++++++++ gazebo/common/CommonIface.hh | 21 ++++++++ gazebo/common/CommonIface_TEST.cc | 85 +++++++++++++++++++++++++++++++ gazebo/gui/ModelMaker.cc | 2 + gazebo/gui/model/ModelCreator.cc | 3 ++ gazebo/msgs/msgs.cc | 3 +- gazebo/physics/MeshShape.cc | 10 ++-- gazebo/rendering/Visual.cc | 16 +++--- test/integration/factory.cc | 36 +++++++++++++ 10 files changed, 248 insertions(+), 11 deletions(-) diff --git a/gazebo/common/CMakeLists.txt b/gazebo/common/CMakeLists.txt index 680bbc0bcd..2b92a8bbd9 100644 --- a/gazebo/common/CMakeLists.txt +++ b/gazebo/common/CMakeLists.txt @@ -265,6 +265,10 @@ if (UNIX AND NOT APPLE) target_link_libraries(gazebo_common PRIVATE rt) endif() +if (NOT APPLE) + target_link_libraries(gazebo_common PRIVATE stdc++fs) +endif() + if (HAVE_GTS) target_link_libraries(gazebo_common PRIVATE ${gts_LIBRARIES}) endif() diff --git a/gazebo/common/CommonIface.cc b/gazebo/common/CommonIface.cc index b04fde8167..088203994a 100644 --- a/gazebo/common/CommonIface.cc +++ b/gazebo/common/CommonIface.cc @@ -31,6 +31,11 @@ #include #endif +#ifndef __APPLE__ +#include +#endif +#include + #include #include @@ -435,3 +440,77 @@ bool common::copyDir(const boost::filesystem::path &_source, } return true; } + +////////////////////////////////////////////////// +// Copied from ignition/gazebo/Util.hh +std::string common::asFullPath(const std::string &_uri, + const std::string &_filePath) +{ + // No path, return unmodified + if (_filePath.empty()) + { + return _uri; + } + +#ifdef __APPLE__ + const std::string absPrefix = "/"; + // Not a relative path, return unmodified + if (_uri.find("://") != std::string::npos || + _uri.compare(0, absPrefix.size(), absPrefix) == 0) + { + return _uri; + } +#else + // Not a relative path, return unmodified + if (_uri.find("://") != std::string::npos || + !std::filesystem::path(_uri).is_relative()) + { + return _uri; + } +#endif + + // When SDF is loaded from a string instead of a file + if ("data-string" == _filePath) + { + gzwarn << "Can't resolve full path for relative path [" + << _uri << "]. Loaded from a data-string." << std::endl; + return _uri; + } + + // Remove file name from path + auto path = ignition::common::parentPath(_filePath); + auto uri = _uri; + + // If path is URI, use "/" separator for all platforms + if (path.find("://") != std::string::npos) + { + std::replace(uri.begin(), uri.end(), '\\', '/'); + return path + "/" + uri; + } + + // In case relative path doesn't match platform +#ifdef _WIN32 + std::replace(uri.begin(), uri.end(), '/', '\\'); +#else + std::replace(uri.begin(), uri.end(), '\\', '/'); +#endif + + // Use platform-specific separator + return ignition::common::joinPaths(path, uri); +} + +////////////////////////////////////////////////// +void common::convertToFullPaths(const sdf::ElementPtr &_elem) +{ + for (auto child = _elem->GetFirstElement(); child != nullptr; + child = child->GetNextElement()) + { + common::convertToFullPaths(child); + } + + if (_elem->GetName() == "uri") + { + _elem->Set(common::asFullPath(_elem->Get(), + _elem->FilePath())); + } +} diff --git a/gazebo/common/CommonIface.hh b/gazebo/common/CommonIface.hh index 6dea3305b6..b890e05da3 100644 --- a/gazebo/common/CommonIface.hh +++ b/gazebo/common/CommonIface.hh @@ -31,6 +31,8 @@ #include #include +#include + #include "gazebo/util/system.hh" namespace gazebo @@ -181,6 +183,25 @@ namespace gazebo GZ_COMMON_VISIBLE std::string unique_file_path(const std::string &_pathAndName, const std::string &_extension); + + /// \brief Combine a URI and a file path into a full path. + /// If the URI is already a full path or contains a scheme, it won't be + /// modified. + /// If the URI is a relative path, the file path will be prepended. + /// \param[in] _uri URI, which can have a scheme, or be full or relative + /// paths. + /// \param[in] _filePath The path to a file in disk. + /// \return The full path URI. + GZ_COMMON_VISIBLE + std::string asFullPath(const std::string &_uri, + const std::string &_filePath); + + /// \brief Convert all the URIs nested inside the given element to + /// full paths based on the SDF element's file path. + /// \sa asFullPath + /// \param[in, out] _elem Element that will have its paths converted. + GZ_COMMON_VISIBLE + void convertToFullPaths(const sdf::ElementPtr &_elem); /// \} } diff --git a/gazebo/common/CommonIface_TEST.cc b/gazebo/common/CommonIface_TEST.cc index 503cafb9e3..fda1464f66 100644 --- a/gazebo/common/CommonIface_TEST.cc +++ b/gazebo/common/CommonIface_TEST.cc @@ -253,6 +253,91 @@ TEST_F(CommonIface_TEST, directoryOps) EXPECT_FALSE(common::exists(destFilePath.string())); } +///////////////////////////////////////////////// +TEST_F(CommonIface_TEST, asFullPath) +{ + const std::string relativeUriUnix{"meshes/collision.dae"}; + const std::string relativeUriWindows{"meshes\\collision.dae"}; + const std::string absoluteUriUnix{"/path/to/collision.dae"}; + const std::string absoluteUriWindows{"C:\\path\\to\\collision.dae"}; + const std::string schemeUri{"https://website.com/collision.dae"}; + + // Empty path + { + const std::string path{""}; + + EXPECT_EQ(relativeUriUnix, common::asFullPath(relativeUriUnix, path)); + EXPECT_EQ(relativeUriWindows, common::asFullPath(relativeUriWindows, path)); + EXPECT_EQ(absoluteUriUnix, common::asFullPath(absoluteUriUnix, path)); + EXPECT_EQ(absoluteUriWindows, common::asFullPath(absoluteUriWindows, path)); + EXPECT_EQ(schemeUri, common::asFullPath(schemeUri, path)); + } + + // Data string + { + const std::string path{"data-string"}; + + EXPECT_EQ(relativeUriUnix, common::asFullPath(relativeUriUnix, path)); + EXPECT_EQ(relativeUriWindows, common::asFullPath(relativeUriWindows, path)); + EXPECT_EQ(absoluteUriUnix, common::asFullPath(absoluteUriUnix, path)); + EXPECT_EQ(absoluteUriWindows, common::asFullPath(absoluteUriWindows, path)); + EXPECT_EQ(schemeUri, common::asFullPath(schemeUri, path)); + } + +#ifdef _WIN32 + { + // Absolute Windows path + const std::string path{"C:\\abs\\path\\file"}; + + // Directory + EXPECT_EQ("C:\\abs\\path\\meshes\\collision.dae", + common::asFullPath(relativeUriUnix, path)); + EXPECT_EQ("C:\\abs\\path\\meshes\\collision.dae", + common::asFullPath(relativeUriWindows, path)); + // TODO(anyone) Support absolute Unix-style URIs on Windows + EXPECT_EQ(absoluteUriWindows, common::asFullPath(absoluteUriWindows, path)); + EXPECT_EQ(schemeUri, common::asFullPath(schemeUri, path)); + + // File + auto filePath = common::joinPaths(path, "file.sdf"); + + EXPECT_EQ("C:\\abs\\path\\file\\meshes\\collision.dae", + common::asFullPath(relativeUriUnix, filePath)); + EXPECT_EQ("C:\\abs\\path\\file\\meshes\\collision.dae", + common::asFullPath(relativeUriWindows, filePath)); + // TODO(anyone) Support absolute Unix-style URIs on Windows + EXPECT_EQ(absoluteUriWindows, common::asFullPath(absoluteUriWindows, + filePath)); + EXPECT_EQ(schemeUri, common::asFullPath(schemeUri, filePath)); + } +#else + { + // Absolute Unix path + const std::string path{"/abs/path/file"}; + + // Directory + EXPECT_EQ("/abs/path/meshes/collision.dae", + common::asFullPath(relativeUriUnix, path)); + EXPECT_EQ("/abs/path/meshes/collision.dae", + common::asFullPath(relativeUriWindows, path)); + EXPECT_EQ(absoluteUriUnix, common::asFullPath(absoluteUriUnix, path)); + // TODO(anyone) Support absolute Windows paths on Unix + EXPECT_EQ(schemeUri, common::asFullPath(schemeUri, path)); + + // File + auto filePath = common::joinPaths(path, "file.sdf"); + + EXPECT_EQ("/abs/path/file/meshes/collision.dae", + common::asFullPath(relativeUriUnix, filePath)); + EXPECT_EQ("/abs/path/file/meshes/collision.dae", + common::asFullPath(relativeUriWindows, filePath)); + EXPECT_EQ(absoluteUriUnix, common::asFullPath(absoluteUriUnix, filePath)); + // TODO(anyone) Support absolute Windows paths on Unix + EXPECT_EQ(schemeUri, common::asFullPath(schemeUri, filePath)); + } +#endif +} + ///////////////////////////////////////////////// int main(int argc, char **argv) { diff --git a/gazebo/gui/ModelMaker.cc b/gazebo/gui/ModelMaker.cc index 9835e93d11..96e9f5c27a 100644 --- a/gazebo/gui/ModelMaker.cc +++ b/gazebo/gui/ModelMaker.cc @@ -18,6 +18,7 @@ #include "gazebo/msgs/msgs.hh" +#include "gazebo/common/CommonIface.hh" #include "gazebo/common/Console.hh" #include "gazebo/common/Exception.hh" #include "gazebo/common/SdfFrameSemantics.hh" @@ -121,6 +122,7 @@ bool ModelMaker::InitFromFile(const std::string &_filename) common::convertPosesToSdf16( this->dataPtr->modelSDF->Root()->GetElement("model")); } + common::convertToFullPaths(this->dataPtr->modelSDF->Root()); return this->Init(); } diff --git a/gazebo/gui/model/ModelCreator.cc b/gazebo/gui/model/ModelCreator.cc index 755f960b41..8986cf42e8 100644 --- a/gazebo/gui/model/ModelCreator.cc +++ b/gazebo/gui/model/ModelCreator.cc @@ -21,6 +21,7 @@ #include +#include "gazebo/common/CommonIface.hh" #include "gazebo/common/Exception.hh" #include "gazebo/common/KeyEvent.hh" #include "gazebo/common/MouseEvent.hh" @@ -1853,6 +1854,8 @@ void ModelCreator::CreateTheEntity() modelElem->GetAttribute("name")->Set(modelElemName); } + common::convertToFullPaths(modelElem); + msg.set_sdf(this->dataPtr->modelSDF->ToString()); msgs::Set(msg.mutable_pose(), this->dataPtr->modelPose); this->dataPtr->makerPub->Publish(msg); diff --git a/gazebo/msgs/msgs.cc b/gazebo/msgs/msgs.cc index 5d33e5913b..74ebcab175 100644 --- a/gazebo/msgs/msgs.cc +++ b/gazebo/msgs/msgs.cc @@ -873,7 +873,8 @@ namespace gazebo msgs::Set(result.mutable_scale(), _sdf->Get("scale")); - result.set_filename(_sdf->Get("uri")); + result.set_filename(common::asFullPath(_sdf->Get("uri"), + _sdf->FilePath())); if (_sdf->HasElement("submesh")) { diff --git a/gazebo/physics/MeshShape.cc b/gazebo/physics/MeshShape.cc index 6f7ecc49d4..8229b7e43e 100644 --- a/gazebo/physics/MeshShape.cc +++ b/gazebo/physics/MeshShape.cc @@ -46,18 +46,22 @@ MeshShape::~MeshShape() ////////////////////////////////////////////////// void MeshShape::Init() { - std::string meshStr = this->sdf->Get("uri"); + auto uri = common::asFullPath(this->sdf->Get("uri"), + this->sdf->FilePath()); + this->sdf->GetElement("uri")->Set(uri); + + std::string meshStr = uri; common::MeshManager *meshManager = common::MeshManager::Instance(); this->mesh = meshManager->GetMesh(meshStr); if (!this->mesh) { - meshStr = common::find_file(this->sdf->Get("uri")); + meshStr = common::find_file(uri); if (meshStr == "__default__" || meshStr.empty()) { - gzerr << "No mesh specified\n"; + gzerr << "Failed to find mesh file [" << uri << "]" << std::endl; return; } diff --git a/gazebo/rendering/Visual.cc b/gazebo/rendering/Visual.cc index bcf8dc02cf..a5ac33b64f 100644 --- a/gazebo/rendering/Visual.cc +++ b/gazebo/rendering/Visual.cc @@ -469,7 +469,8 @@ void Visual::Load() // Add all the URI paths to the render engine while (uriElem) { - std::string matUri = uriElem->Get(); + auto matUri = common::asFullPath(uriElem->Get(), + scriptElem->FilePath()); if (!matUri.empty()) RenderEngine::Instance()->AddResourcePath(matUri); uriElem = uriElem->GetNextElement("uri"); @@ -2926,17 +2927,18 @@ std::string Visual::GetMeshName() const } else if (geomElem->HasElement("mesh")) { - sdf::ElementPtr tmpElem = geomElem->GetElement("mesh"); - std::string filename; + sdf::ElementPtr meshElem = geomElem->GetElement("mesh"); - std::string uri = tmpElem->Get("uri"); - if (uri.empty()) + if (!meshElem->HasElement("uri")) { - gzerr << " element missing for geometry element:\n"; + gzerr << " element missing for geometry element\n"; return std::string(); } - filename = common::find_file(uri); + auto uri = common::asFullPath(meshElem->Get("uri"), + meshElem->FilePath()); + + auto filename = common::find_file(uri); if (filename == "__default__" || filename.empty()) gzerr << "No mesh specified\n"; diff --git a/test/integration/factory.cc b/test/integration/factory.cc index 46c1e5980a..8a7be91b3d 100644 --- a/test/integration/factory.cc +++ b/test/integration/factory.cc @@ -1146,6 +1146,42 @@ TEST_F(FactoryTest, FilenameModelDatabase) ASSERT_NE(nullptr, world->ModelByName("cococan")); } +////////////////////////////////////////////////// +TEST_F(FactoryTest, FilenameModelDatabaseRelativePaths) +{ + this->Load("worlds/empty.world", true); + + // Test database + common::SystemPaths::Instance()->AddModelPaths( + PROJECT_SOURCE_PATH "/test/models/testdb"); + + // World + auto world = physics::get_world("default"); + ASSERT_NE(nullptr, world); + + // Publish factory msg + msgs::Factory msg; + msg.set_sdf_filename("model://cococan_relative_paths"); + + auto pub = this->node->Advertise("~/factory"); + pub->Publish(msg); + + // Wait for it to be spawned + int sleep = 0; + int maxSleep = 50; + while (!world->ModelByName("cococan") && sleep++ < maxSleep) + { + common::Time::MSleep(100); + } + + // Check model was spawned + auto model = world->ModelByName("cococan"); + ASSERT_NE(nullptr, model); + + auto link = model->LinkByName("link"); + ASSERT_NE(nullptr, link); +} + ////////////////////////////////////////////////// #ifdef HAVE_IGNITION_FUEL_TOOLS TEST_F(FactoryTest, FilenameFuelURL) From 6e2bb541f67333c9c876c330f6226a68e1d77aaa Mon Sep 17 00:00:00 2001 From: Louise Poubel Date: Wed, 8 Jul 2020 18:42:35 -0700 Subject: [PATCH 02/14] convertToFullPaths unit test Signed-off-by: Louise Poubel --- gazebo/common/CommonIface_TEST.cc | 88 ++++++++++++++++++++++++++++++- 1 file changed, 86 insertions(+), 2 deletions(-) diff --git a/gazebo/common/CommonIface_TEST.cc b/gazebo/common/CommonIface_TEST.cc index fda1464f66..fa58018af2 100644 --- a/gazebo/common/CommonIface_TEST.cc +++ b/gazebo/common/CommonIface_TEST.cc @@ -22,6 +22,9 @@ #include #include +#include +#include + #include "gazebo/common/CommonIface.hh" #include "gazebo/common/SystemPaths.hh" #include "test/util.hh" @@ -299,7 +302,7 @@ TEST_F(CommonIface_TEST, asFullPath) EXPECT_EQ(schemeUri, common::asFullPath(schemeUri, path)); // File - auto filePath = common::joinPaths(path, "file.sdf"); + auto filePath = ignition::common::joinPaths(path, "file.sdf"); EXPECT_EQ("C:\\abs\\path\\file\\meshes\\collision.dae", common::asFullPath(relativeUriUnix, filePath)); @@ -325,7 +328,7 @@ TEST_F(CommonIface_TEST, asFullPath) EXPECT_EQ(schemeUri, common::asFullPath(schemeUri, path)); // File - auto filePath = common::joinPaths(path, "file.sdf"); + auto filePath = ignition::common::joinPaths(path, "file.sdf"); EXPECT_EQ("/abs/path/file/meshes/collision.dae", common::asFullPath(relativeUriUnix, filePath)); @@ -338,6 +341,87 @@ TEST_F(CommonIface_TEST, asFullPath) #endif } +///////////////////////////////////////////////// +TEST_F(CommonIface_TEST, fullPathsMesh) +{ + std::string filePath{"/tmp/model.sdf"}; + std::string relativePath{"meshes/collision.dae"}; + + std::stringstream ss; + ss << "" + << "" + << "" + << " " + << " " + << " " + << " " << relativePath << "" + << " " + << " " + << " " + << " " + << " " + << " " << relativePath << "" + << " " + << " " + << "" + << "" + << ""; + + auto modelElem = std::make_shared(); + sdf::initFile("model.sdf", modelElem); + sdf::readString(ss.str(), modelElem); + + // Before conversion + ASSERT_NE(nullptr, modelElem); + + auto linkElem = modelElem->GetElement("link"); + ASSERT_NE(nullptr, linkElem); + + for (auto elemStr : {"collision", "visual"}) + { + auto elem = linkElem->GetElement(elemStr); + ASSERT_NE(nullptr, elem); + + auto geometryElem = elem->GetElement("geometry"); + ASSERT_NE(nullptr, geometryElem); + + auto meshElem = geometryElem->GetElement("mesh"); + ASSERT_NE(nullptr, meshElem); + + auto uriElem = meshElem->GetElement("uri"); + ASSERT_NE(nullptr, uriElem); + + EXPECT_EQ(relativePath, uriElem->Get()); + uriElem->SetFilePath(filePath); + } + + common::convertToFullPaths(modelElem); + + // After conversion + ASSERT_NE(nullptr, modelElem); + + linkElem = modelElem->GetElement("link"); + ASSERT_NE(nullptr, linkElem); + + for (auto elemStr : {"collision", "visual"}) + { + auto elem = linkElem->GetElement(elemStr); + ASSERT_NE(nullptr, elem); + + auto geometryElem = elem->GetElement("geometry"); + ASSERT_NE(nullptr, geometryElem); + + auto meshElem = geometryElem->GetElement("mesh"); + ASSERT_NE(nullptr, meshElem); + + auto uriElem = meshElem->GetElement("uri"); + ASSERT_NE(nullptr, uriElem); + + EXPECT_EQ(ignition::common::joinPaths("/tmp", relativePath), + uriElem->Get()); + } +} + ///////////////////////////////////////////////// int main(int argc, char **argv) { From d96cfabe36ac1eb7fcbefdcced0b4f80873a8d5f Mon Sep 17 00:00:00 2001 From: ahcorde Date: Wed, 12 Aug 2020 12:21:19 +0200 Subject: [PATCH 03/14] Allow gazebo to download models from Fuel in the sdf files Signed-off-by: ahcorde --- gazebo/common/CommonIface.cc | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/gazebo/common/CommonIface.cc b/gazebo/common/CommonIface.cc index b04fde8167..eb9b7b8152 100644 --- a/gazebo/common/CommonIface.cc +++ b/gazebo/common/CommonIface.cc @@ -42,6 +42,8 @@ #include "gazebo/common/Exception.hh" #include "gazebo/common/SystemPaths.hh" +#include "ignition/fuel_tools/Interface.hh" + using namespace gazebo; #ifdef _WIN32 @@ -152,7 +154,21 @@ void common::add_search_path_suffix(const std::string &_suffix) ///////////////////////////////////////////////// std::string common::find_file(const std::string &_file) { - return common::SystemPaths::Instance()->FindFile(_file, true); + std::string path = common::SystemPaths::Instance()->FindFile(_file, true); + if (path.empty()) + { + gzwarn << "Trying to find the model in Fuel [" << _file << "]" << std::endl; + path = ignition::fuel_tools::fetchResource(_file); + if (path.empty()) + { + gzwarn << "Not able to download or find the model [" << _file << "]" << std::endl; + } + else + { + gzwarn << "Model ready to use [" << _file << "]" << std::endl; + } + } + return path; } ///////////////////////////////////////////////// From d3ab7192a43bce20a2e6b5440f26f913e63180a0 Mon Sep 17 00:00:00 2001 From: ahcorde Date: Thu, 13 Aug 2020 10:24:38 +0200 Subject: [PATCH 04/14] Load worlds using URL from command line Signed-off-by: ahcorde --- gazebo/Server.cc | 35 +++++++++++++++++++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/gazebo/Server.cc b/gazebo/Server.cc index 9152a7ec8b..a4e0de7283 100644 --- a/gazebo/Server.cc +++ b/gazebo/Server.cc @@ -62,6 +62,10 @@ #include "gazebo/Master.hh" #include "gazebo/Server.hh" +#include "ignition/common/Filesystem.hh" +#include "ignition/common/URI.hh" +#include "ignition/fuel_tools/Interface.hh" + namespace po = boost::program_options; using namespace gazebo; @@ -402,9 +406,36 @@ bool Server::LoadFile(const std::string &_filename, return false; } - if (!sdf::readFile(common::find_file(_filename), sdf)) + std::string filename = _filename; + auto filenameUri = ignition::common::URI(filename); + + if (filenameUri.Scheme() == "http" || filenameUri.Scheme() == "https") { - gzerr << "Unable to read sdf file[" << _filename << "]\n"; + std::string downloadedDir = ignition::fuel_tools::fetchResource(filename); + // Find the first sdf file in the world path for now, the later intention is + // to load an optional world config file first and if that does not exist, + // continue to load the first sdf file found as done below + for (ignition::common::DirIter file(downloadedDir); + file != ignition::common::DirIter(); ++file) + { + std::string current(*file); + if (ignition::common::isFile(current)) + { + std::string fileName = ignition::common::basename(current); + std::string::size_type fileExtensionIndex = fileName.rfind("."); + std::string fileExtension = fileName.substr(fileExtensionIndex + 1); + + if (fileExtension == "sdf" || fileExtension == "world") + { + filename = current.c_str(); + } + } + } + } + + if (!sdf::readFile(common::find_file(filename), sdf)) + { + gzerr << "Unable to read sdf file[" << filename << "]\n"; return false; } From 38fe3c4f0e058e614f062aaa444057acc3b7d383 Mon Sep 17 00:00:00 2001 From: Louise Poubel Date: Thu, 13 Aug 2020 19:12:36 -0700 Subject: [PATCH 05/14] adapt to ModelMaker updates Signed-off-by: Louise Poubel --- gazebo/common/CommonIface.cc | 29 +++++++++++++++++++++++++--- gazebo/common/CommonIface.hh | 13 ++++++++++++- gazebo/common/CommonIface_TEST.cc | 32 ++++++++++++++++++++++++++++++- gazebo/gui/ModelMaker.cc | 3 +++ 4 files changed, 72 insertions(+), 5 deletions(-) diff --git a/gazebo/common/CommonIface.cc b/gazebo/common/CommonIface.cc index 088203994a..0958e30c8b 100644 --- a/gazebo/common/CommonIface.cc +++ b/gazebo/common/CommonIface.cc @@ -39,6 +39,8 @@ #include #include +#include + #include #include @@ -500,17 +502,38 @@ std::string common::asFullPath(const std::string &_uri, } ////////////////////////////////////////////////// -void common::convertToFullPaths(const sdf::ElementPtr &_elem) +void common::convertToFullPaths(const sdf::ElementPtr &_elem, + const std::string &_filePath) { for (auto child = _elem->GetFirstElement(); child != nullptr; child = child->GetNextElement()) { - common::convertToFullPaths(child); + common::convertToFullPaths(child, _filePath); } if (_elem->GetName() == "uri") { + auto filePath = _filePath.empty() ? _elem->FilePath() : _filePath; _elem->Set(common::asFullPath(_elem->Get(), - _elem->FilePath())); + filePath)); } } + +////////////////////////////////////////////////// +void common::convertToFullPaths(std::string &_sdfString, + const std::string &_filePath) +{ + sdf::SDFPtr sdf = std::make_shared(); + sdf::initFile("root.sdf", sdf); + + if (!sdf::readString(_sdfString, sdf)) + { + return; + } + + sdf->Root()->SetFilePath(_filePath); + + convertToFullPaths(sdf->Root(), _filePath); + + _sdfString = sdf->Root()->ToString(""); +} diff --git a/gazebo/common/CommonIface.hh b/gazebo/common/CommonIface.hh index b890e05da3..d7b5a86b1f 100644 --- a/gazebo/common/CommonIface.hh +++ b/gazebo/common/CommonIface.hh @@ -200,8 +200,19 @@ namespace gazebo /// full paths based on the SDF element's file path. /// \sa asFullPath /// \param[in, out] _elem Element that will have its paths converted. + /// \param[in] _filePath Optional filepath to override `_elem->FilePath()`. GZ_COMMON_VISIBLE - void convertToFullPaths(const sdf::ElementPtr &_elem); + void convertToFullPaths(const sdf::ElementPtr &_elem, + const std::string &_filePath = {}); + + /// \brief Convert all the URIs nested inside the given SDF string to + /// full paths based on the SDF element's file path. + /// \sa asFullPath + /// \param[in, out] _sdfString SDF file in string format + /// \param[in] _filePath Path to SDF file + GZ_COMMON_VISIBLE + void convertToFullPaths(std::string &_sdfString, + const std::string &_filePath); /// \} } diff --git a/gazebo/common/CommonIface_TEST.cc b/gazebo/common/CommonIface_TEST.cc index fa58018af2..d8a863c93d 100644 --- a/gazebo/common/CommonIface_TEST.cc +++ b/gazebo/common/CommonIface_TEST.cc @@ -395,9 +395,11 @@ TEST_F(CommonIface_TEST, fullPathsMesh) uriElem->SetFilePath(filePath); } + auto sdfString = modelElem->ToString(""); + + // SDF element conversion common::convertToFullPaths(modelElem); - // After conversion ASSERT_NE(nullptr, modelElem); linkElem = modelElem->GetElement("link"); @@ -420,6 +422,34 @@ TEST_F(CommonIface_TEST, fullPathsMesh) EXPECT_EQ(ignition::common::joinPaths("/tmp", relativePath), uriElem->Get()); } + + + // SDF string conversion + sdfString = "\n" + sdfString + + "\n"; + common::convertToFullPaths(sdfString, modelElem->FilePath()); + sdf::readString(sdfString, modelElem); + + linkElem = modelElem->GetElement("link"); + ASSERT_NE(nullptr, linkElem); + + for (auto elemStr : {"collision", "visual"}) + { + auto elem = linkElem->GetElement(elemStr); + ASSERT_NE(nullptr, elem); + + auto geometryElem = elem->GetElement("geometry"); + ASSERT_NE(nullptr, geometryElem); + + auto meshElem = geometryElem->GetElement("mesh"); + ASSERT_NE(nullptr, meshElem); + + auto uriElem = meshElem->GetElement("uri"); + ASSERT_NE(nullptr, uriElem); + + EXPECT_EQ(ignition::common::joinPaths("/tmp", relativePath), + uriElem->Get()); + } } ///////////////////////////////////////////////// diff --git a/gazebo/gui/ModelMaker.cc b/gazebo/gui/ModelMaker.cc index aa38a967e5..0bd1b6a6de 100644 --- a/gazebo/gui/ModelMaker.cc +++ b/gazebo/gui/ModelMaker.cc @@ -127,6 +127,9 @@ bool ModelMaker::InitFromFile(const std::string &_filename) std::string(std::istreambuf_iterator(inputFile), std::istreambuf_iterator()); + common::convertToFullPaths(this->dataPtr->modelSDFString, + this->dataPtr->modelSDF->FilePath()); + if (this->dataPtr->modelSDF->Root()->HasElement("model")) { common::convertPosesToSdf16( From 737037fb3b302a20b64d596f120b350aed84e0c6 Mon Sep 17 00:00:00 2001 From: Louise Poubel Date: Fri, 14 Aug 2020 15:50:04 -0700 Subject: [PATCH 06/14] actor, factory, tests Signed-off-by: Louise Poubel --- gazebo/common/CommonIface.cc | 7 +- gazebo/common/CommonIface_TEST.cc | 48 +++++++ gazebo/physics/World.cc | 6 +- test/integration/factory.cc | 49 ++++++- .../materials/textures/checker.png | Bin 0 -> 17270 bytes .../testdb/relative_paths/meshes/test.dae | 126 ++++++++++++++++++ .../models/testdb/relative_paths/model.config | 15 +++ test/models/testdb/relative_paths/model.sdf | 22 +++ 8 files changed, 266 insertions(+), 7 deletions(-) create mode 100644 test/models/testdb/relative_paths/materials/textures/checker.png create mode 100644 test/models/testdb/relative_paths/meshes/test.dae create mode 100644 test/models/testdb/relative_paths/model.config create mode 100644 test/models/testdb/relative_paths/model.sdf diff --git a/gazebo/common/CommonIface.cc b/gazebo/common/CommonIface.cc index 0958e30c8b..7e6b567c0e 100644 --- a/gazebo/common/CommonIface.cc +++ b/gazebo/common/CommonIface.cc @@ -511,7 +511,12 @@ void common::convertToFullPaths(const sdf::ElementPtr &_elem, common::convertToFullPaths(child, _filePath); } - if (_elem->GetName() == "uri") + // * All + // * All which are children of or + if (_elem->GetName() == "uri" || + ((_elem->GetParent()->GetName() == "animation" || + _elem->GetParent()->GetName() == "skin") && + _elem->GetName() == "filename")) { auto filePath = _filePath.empty() ? _elem->FilePath() : _filePath; _elem->Set(common::asFullPath(_elem->Get(), diff --git a/gazebo/common/CommonIface_TEST.cc b/gazebo/common/CommonIface_TEST.cc index d8a863c93d..a20d1b04b8 100644 --- a/gazebo/common/CommonIface_TEST.cc +++ b/gazebo/common/CommonIface_TEST.cc @@ -452,6 +452,54 @@ TEST_F(CommonIface_TEST, fullPathsMesh) } } +///////////////////////////////////////////////// +TEST_F(CommonIface_TEST, fullPathsActor) +{ + std::string filePath{"/tmp/actor.sdf"}; + std::string relativePath{"meshes/walk.dae"}; + + std::stringstream ss; + ss << "" + << "" + << " " + << " " << relativePath << "" + << " " + << " " + << " " << relativePath << "" + << " " + << "" + << ""; + + auto actorElem = std::make_shared(); + sdf::initFile("actor.sdf", actorElem); + sdf::readString(ss.str(), actorElem); + + // Before conversion + ASSERT_NE(nullptr, actorElem); + + auto skinElem = actorElem->GetElement("skin"); + ASSERT_NE(nullptr, skinElem); + EXPECT_EQ(relativePath, skinElem->Get()); + + auto animationElem = actorElem->GetElement("animation"); + ASSERT_NE(nullptr, animationElem); + EXPECT_EQ(relativePath, animationElem->Get()); + + // SDF element conversion + common::convertToFullPaths(actorElem); + ASSERT_NE(nullptr, actorElem); + + skinElem = actorElem->GetElement("skin"); + ASSERT_NE(nullptr, skinElem); + EXPECT_EQ(ignition::common::joinPaths("/tmp", relativePath), + skinElem->Get()); + + animationElem = actorElem->GetElement("animation"); + ASSERT_NE(nullptr, animationElem); + EXPECT_EQ(ignition::common::joinPaths("/tmp", relativePath), + animationElem->Get()); +} + ///////////////////////////////////////////////// int main(int argc, char **argv) { diff --git a/gazebo/physics/World.cc b/gazebo/physics/World.cc index 6873d644f5..821cf88116 100644 --- a/gazebo/physics/World.cc +++ b/gazebo/physics/World.cc @@ -179,6 +179,7 @@ void World::Load(sdf::ElementPtr _sdf) { this->dataPtr->loaded = false; this->dataPtr->sdf = _sdf; + common::convertToFullPaths(this->dataPtr->sdf); // Create a DOM object to compute the resolved initial pose (with frame // semantics) @@ -2091,6 +2092,7 @@ void World::ProcessFactoryMsgs() for (auto const &factoryMsg : factoryMsgsCopy) { + this->dataPtr->factorySDF->Clear(); if (factoryMsg.has_sdf() && !factoryMsg.sdf().empty()) @@ -2122,9 +2124,11 @@ void World::ProcessFactoryMsgs() if (!sdf::readFile(filename, this->dataPtr->factorySDF)) { - gzerr << "Unable to read sdf file.\n"; + gzerr << "Unable to read sdf file [" << filename << "]\n"; continue; } + + common::convertToFullPaths(this->dataPtr->factorySDF->Root()); } else if (factoryMsg.has_clone_model_name()) { diff --git a/test/integration/factory.cc b/test/integration/factory.cc index 8a7be91b3d..e3767c58ea 100644 --- a/test/integration/factory.cc +++ b/test/integration/factory.cc @@ -18,6 +18,7 @@ #include "gazebo/transport/TransportTypes.hh" #include "gazebo/transport/Node.hh" +#include "gazebo/physics/MeshShape.hh" #include "gazebo/rendering/RenderEngine.hh" #include "gazebo/rendering/Camera.hh" #include "gazebo/sensors/SensorsIface.hh" @@ -1149,7 +1150,8 @@ TEST_F(FactoryTest, FilenameModelDatabase) ////////////////////////////////////////////////// TEST_F(FactoryTest, FilenameModelDatabaseRelativePaths) { - this->Load("worlds/empty.world", true); + // World with a rendering sensor + this->Load("worlds/camera.world", true); // Test database common::SystemPaths::Instance()->AddModelPaths( @@ -1161,7 +1163,7 @@ TEST_F(FactoryTest, FilenameModelDatabaseRelativePaths) // Publish factory msg msgs::Factory msg; - msg.set_sdf_filename("model://cococan_relative_paths"); + msg.set_sdf_filename("model://relative_paths"); auto pub = this->node->Advertise("~/factory"); pub->Publish(msg); @@ -1169,17 +1171,54 @@ TEST_F(FactoryTest, FilenameModelDatabaseRelativePaths) // Wait for it to be spawned int sleep = 0; int maxSleep = 50; - while (!world->ModelByName("cococan") && sleep++ < maxSleep) + while (!world->ModelByName("relative_paths") && sleep++ < maxSleep) { common::Time::MSleep(100); } // Check model was spawned - auto model = world->ModelByName("cococan"); + auto model = world->ModelByName("relative_paths"); ASSERT_NE(nullptr, model); - auto link = model->LinkByName("link"); + auto link = model->GetLink("link"); ASSERT_NE(nullptr, link); + + auto collision = link->GetCollision("collision"); + ASSERT_NE(nullptr, collision); + + auto shape = collision->GetShape(); + ASSERT_NE(nullptr, shape); + + auto meshShape = boost::static_pointer_cast(shape); + ASSERT_NE(nullptr, meshShape); + EXPECT_EQ(PROJECT_SOURCE_PATH + "/test/models/testdb/relative_paths/meshes/test.dae", + meshShape->GetMeshURI()); + + // Make sure the render engine is available + if (rendering::RenderEngine::Instance()->GetRenderPathType() == + rendering::RenderEngine::NONE) + { + FAIL() << "No rendering engine"; + } + + auto scene = rendering::get_scene(); + ASSERT_NE(nullptr, scene); + + auto modelVis = scene->GetVisual("relative_paths"); + ASSERT_NE(nullptr, modelVis); + + auto linkVis = modelVis->GetChild(0); + ASSERT_NE(nullptr, linkVis); + EXPECT_EQ("relative_paths::link", linkVis->Name()); + + auto visualVis = linkVis->GetChild(0); + ASSERT_NE(nullptr, visualVis); + EXPECT_EQ("relative_paths::link::visual", visualVis->Name()); + + EXPECT_EQ(PROJECT_SOURCE_PATH + "/test/models/testdb/relative_paths/meshes/test.dae", + visualVis->GetMeshName()); } ////////////////////////////////////////////////// diff --git a/test/models/testdb/relative_paths/materials/textures/checker.png b/test/models/testdb/relative_paths/materials/textures/checker.png new file mode 100644 index 0000000000000000000000000000000000000000..2e37021462ade27c8be2e75e24f2f13a249413fc GIT binary patch literal 17270 zcmeI4Ur19?9LLY}*8a3*OQUI0S5_?OMS8G3RC8@(TcAitCiI}zay`sNyIB#ntAnIB zq3EFoy+shD{eh$>r9TG6=%t)1@kRtf%SM4N#;%>S+OD)Y_lcB~y2r>D6?gNDA=vN)ll~J?jsk@sR{Rv-tgpf6r?9h;FaSJ`EmHbT} z?R!C}GH(mM8~Q{~RZE^jk}Dhv1$rdnIvMDc0!PKu$EBd?^*6P&^{w4Vh)DcyXX}|% z`t6u$wBGn+_)+dlI$d|!YMJ?1C-u(9ZXbJQw!cu1ER3nKyf#-cwSz=07!xWsh;4&cBDEQL$K-iZw(In7IZS@&c#?MOz47dmK} z!gbvNP``jeJ*FTCOd#+J_yzg%^;+d_Si6fnrsZQLKX7nvWYB84_0?Vle}ljM1OArK zXD1uH%CuJ5ttrMBvHZZ!C?DO%L2ikYZPCz=pyzhoR&H?tI4}ax1J?mKfCD3dd6Mg3 z8E~kNhkA;=02r4NuH#b;ZOU|ok}zkl=Er|-i9EoTNWnKy0_Rf%IDi8qfQ*{!035)9 z5m?#^K!v*?2yrGZ-v~tn^*l*b%Di2ZZH#RHrsaZJ2C*jM%M!+yE3g!mNj{8ik~=M{ zyE3Y4$GOAL&BoTVwFXqtxCes+I4}aJqH!I7L+LrZ3Po0_c`{qu7e?a+8Wxd_aTf#! za9{+Gm2w?`12`}OzkjX+Ef$OuQaeTg9FWp*9e~5%;E+^@CI~4@viAb;`=hZ994LVm xu{>o`_sgs4pOu~JoB8a1>&3-)Qy3R3;TD6fG5opxb=OTcrhZSe`_8_O{vWZs)z1I` literal 0 HcmV?d00001 diff --git a/test/models/testdb/relative_paths/meshes/test.dae b/test/models/testdb/relative_paths/meshes/test.dae new file mode 100644 index 0000000000..e85746a85d --- /dev/null +++ b/test/models/testdb/relative_paths/meshes/test.dae @@ -0,0 +1,126 @@ + + + + + Blender User + Blender 2.78.0 + + 2017-04-04T15:56:25 + 2017-04-04T15:56:25 + + Z_UP + + + + checker.png + + + + + + + + checker_png + + + + + checker_png-surface + + + + + + 1 1 1 1 + + + 1 1 1 1 + + + + + + 0 0 0 1 + + + 50 + + + 1 + + + + + + + + + + + + + + + + -1 -1 0 1 -1 0 -1 1 0 1 1 0 + + + + + + + + + + 0 0 1 + + + + + + + + + + 0.9999001 9.998e-5 9.998e-5 0.9999001 1.0004e-4 9.998e-5 0.9999001 9.998e-5 0.9999001 0.9999001 9.998e-5 0.9999001 + + + + + + + + + + + + + + + 3 3 +

1 0 0 2 0 1 0 0 2 1 0 3 3 0 4 2 0 5

+
+
+
+
+ + + + + 1 0 0 0 0 1 0 0 0 0 1 0 0 0 0 1 + + + + + + + + + + + + + + + +
\ No newline at end of file diff --git a/test/models/testdb/relative_paths/model.config b/test/models/testdb/relative_paths/model.config new file mode 100644 index 0000000000..464640c45f --- /dev/null +++ b/test/models/testdb/relative_paths/model.config @@ -0,0 +1,15 @@ + + + + relative_paths + 1.0 + model.sdf + + Testo + testo@osrfoundation.org + + + + Model using relative URI paths + + diff --git a/test/models/testdb/relative_paths/model.sdf b/test/models/testdb/relative_paths/model.sdf new file mode 100644 index 0000000000..03ab23a5f8 --- /dev/null +++ b/test/models/testdb/relative_paths/model.sdf @@ -0,0 +1,22 @@ + + + + 0 0 0.5 0 0 0 + + + + + meshes/test.dae + + + + + + + meshes/test.dae + + + + + + From f5dbf5f5489d3ddc93601264c68e76cea06cae85 Mon Sep 17 00:00:00 2001 From: ahcorde Date: Fri, 28 Aug 2020 13:41:32 +0200 Subject: [PATCH 07/14] Added feedback Signed-off-by: ahcorde --- gazebo/Server.cc | 9 +++-- gazebo/common/CommonIface.cc | 13 ++----- gazebo/gui/InsertModelWidget.cc | 21 ----------- test/integration/factory.cc | 62 +++++++++++++++++++++++++++++++++ worlds/fuel_models.world | 39 +++++++++++++++++++++ 5 files changed, 107 insertions(+), 37 deletions(-) create mode 100644 worlds/fuel_models.world diff --git a/gazebo/Server.cc b/gazebo/Server.cc index a4e0de7283..611df672af 100644 --- a/gazebo/Server.cc +++ b/gazebo/Server.cc @@ -33,7 +33,10 @@ #include #include -#include "ignition/common/Profiler.hh" +#include +#include +#include +#include #include "gazebo/gazebo.hh" #include "gazebo/transport/transport.hh" @@ -62,10 +65,6 @@ #include "gazebo/Master.hh" #include "gazebo/Server.hh" -#include "ignition/common/Filesystem.hh" -#include "ignition/common/URI.hh" -#include "ignition/fuel_tools/Interface.hh" - namespace po = boost::program_options; using namespace gazebo; diff --git a/gazebo/common/CommonIface.cc b/gazebo/common/CommonIface.cc index 3dfff1bcf0..f9e0974c4e 100644 --- a/gazebo/common/CommonIface.cc +++ b/gazebo/common/CommonIface.cc @@ -161,19 +161,10 @@ void common::add_search_path_suffix(const std::string &_suffix) ///////////////////////////////////////////////// std::string common::find_file(const std::string &_file) { - std::string path = common::SystemPaths::Instance()->FindFile(_file, true); + std::string path = ignition::fuel_tools::fetchResource(_file); if (path.empty()) { - gzwarn << "Trying to find the model in Fuel [" << _file << "]" << std::endl; - path = ignition::fuel_tools::fetchResource(_file); - if (path.empty()) - { - gzwarn << "Not able to download or find the model [" << _file << "]" << std::endl; - } - else - { - gzwarn << "Model ready to use [" << _file << "]" << std::endl; - } + path = common::SystemPaths::Instance()->FindFile(_file, true); } return path; } diff --git a/gazebo/gui/InsertModelWidget.cc b/gazebo/gui/InsertModelWidget.cc index 80e794252c..e5f1780700 100644 --- a/gazebo/gui/InsertModelWidget.cc +++ b/gazebo/gui/InsertModelWidget.cc @@ -48,21 +48,6 @@ using namespace gui; static bool gInsertModelWidgetDeleted = false; -///////////////////////////////////////////////// -// TODO: Remove this once Fuel support is fully functional -bool usingFuel() -{ - auto useFuel = std::getenv("USE_IGNITION_FUEL"); - if (!useFuel || *useFuel == '\0') - return false; - - std::string useFuelStr(useFuel); - std::transform(useFuelStr.begin(), useFuelStr.end(), - useFuelStr.begin(), ::tolower); - - return useFuelStr != "false" && useFuelStr != "0"; -} - ///////////////////////////////////////////////// InsertModelWidget::InsertModelWidget(QWidget *_parent) : QWidget(_parent), dataPtr(new InsertModelWidgetPrivate) @@ -523,9 +508,6 @@ bool InsertModelWidget::IsPathAccessible(const boost::filesystem::path &_path) ///////////////////////////////////////////////// void InsertModelWidget::InitializeFuelServers() { - if (!usingFuel()) - return; - // Get the list of Ignition Fuel servers. auto servers = common::FuelModelDatabase::Instance()->Servers(); @@ -550,9 +532,6 @@ void InsertModelWidget::InitializeFuelServers() ///////////////////////////////////////////////// void InsertModelWidget::PopulateFuelServers() { - if (!usingFuel()) - return; - // Get the list of Ignition Fuel servers. auto servers = common::FuelModelDatabase::Instance()->Servers(); diff --git a/test/integration/factory.cc b/test/integration/factory.cc index e3767c58ea..f3c1afee3e 100644 --- a/test/integration/factory.cc +++ b/test/integration/factory.cc @@ -1249,6 +1249,68 @@ TEST_F(FactoryTest, FilenameFuelURL) // Check model was spawned ASSERT_NE(nullptr, world->ModelByName("test_box")); } + +TEST_F(FactoryTest, worldWithFuelModels) +{ + this->Load("worlds/fuel_models.world", true); + + // World + auto world = physics::get_world("default"); + ASSERT_NE(nullptr, world); + + // Wait for it to be spawned + int sleep = 0; + int maxSleep = 50; + while (!world->ModelByName("Radio") && sleep++ < maxSleep) + { + common::Time::MSleep(100); + } + + // Check model was spawned + ASSERT_NE(nullptr, world->ModelByName("Radio")); +} + +TEST_F(FactoryTest, FuelURIAsWorldArgument) +{ + this->Load( + "https://staging-fuel.ignitionrobotics.org/1.0/chapulina/worlds/Shapes", + true); + + // World + auto world = physics::get_world("default"); + ASSERT_NE(nullptr, world); + + // Wait for it to be spawned + int sleep = 0; + int maxSleep = 50; + while (!world->ModelByName("box") && sleep++ < maxSleep) + { + common::Time::MSleep(100); + } + + // Check model was spawned + ASSERT_NE(nullptr, world->ModelByName("box")); + + sleep = 0; + maxSleep = 50; + while (!world->ModelByName("cylinder") && sleep++ < maxSleep) + { + common::Time::MSleep(100); + } + + // Check model was spawned + ASSERT_NE(nullptr, world->ModelByName("cylinder")); + + sleep = 0; + maxSleep = 50; + while (!world->ModelByName("sphere") && sleep++ < maxSleep) + { + common::Time::MSleep(100); + } + + // Check model was spawned + ASSERT_NE(nullptr, world->ModelByName("sphere")); +} #endif ////////////////////////////////////////////////// diff --git a/worlds/fuel_models.world b/worlds/fuel_models.world new file mode 100644 index 0000000000..77062c3bce --- /dev/null +++ b/worlds/fuel_models.world @@ -0,0 +1,39 @@ + + + + + + model://sun + + + + model://ground_plane + + + + 0 1 3 0.0 0.0 1.57 + https://fuel.ignitionrobotics.org/1.0/openrobotics/models/Construction Cone + + + + 3 -1.5 0 0 0 0 + true + + + + + https://fuel.ignitionrobotics.org/1.0/openrobotics/models/Radio/4/files/meshes/Radio.dae + + + + + + + https://fuel.ignitionrobotics.org/1.0/openrobotics/models/Radio/4/files/meshes/Radio.dae + + + + + + + From 350ab80cdbd9028a0b32cced3b6d9a4f907bd826 Mon Sep 17 00:00:00 2001 From: ahcorde Date: Mon, 31 Aug 2020 16:51:11 +0200 Subject: [PATCH 08/14] Used common::FuelModelDatabase to get the modelpath Signed-off-by: ahcorde --- gazebo/common/CommonIface.cc | 3 +- gazebo/common/FuelModelDatabase.cc | 58 ++++++++++++++++++++++-------- 2 files changed, 45 insertions(+), 16 deletions(-) diff --git a/gazebo/common/CommonIface.cc b/gazebo/common/CommonIface.cc index f9e0974c4e..050884ad21 100644 --- a/gazebo/common/CommonIface.cc +++ b/gazebo/common/CommonIface.cc @@ -47,6 +47,7 @@ #include "gazebo/common/Console.hh" #include "gazebo/common/CommonIface.hh" #include "gazebo/common/Exception.hh" +#include "gazebo/common/FuelModelDatabase.hh" #include "gazebo/common/SystemPaths.hh" #include "ignition/fuel_tools/Interface.hh" @@ -161,7 +162,7 @@ void common::add_search_path_suffix(const std::string &_suffix) ///////////////////////////////////////////////// std::string common::find_file(const std::string &_file) { - std::string path = ignition::fuel_tools::fetchResource(_file); + std::string path = common::FuelModelDatabase::Instance()->ModelPath(_file); if (path.empty()) { path = common::SystemPaths::Instance()->FindFile(_file, true); diff --git a/gazebo/common/FuelModelDatabase.cc b/gazebo/common/FuelModelDatabase.cc index 2d57c378b6..b6cd9484c3 100644 --- a/gazebo/common/FuelModelDatabase.cc +++ b/gazebo/common/FuelModelDatabase.cc @@ -213,26 +213,54 @@ std::string FuelModelDatabase::ModelPath(const std::string &_uri, } std::string path; + std::string fileUrl; + ignition::fuel_tools::ModelIdentifier model; + using namespace ignition::fuel_tools; - if (!_forceDownload) + if ((this->dataPtr->fuelClient->ParseModelUrl(fuelUri, model) && + !this->dataPtr->fuelClient->CachedModel(fuelUri, path)) + || _forceDownload) { - if (this->dataPtr->fuelClient->CachedModel(fuelUri, path)) + Result result_download = + this->dataPtr->fuelClient->DownloadModel(fuelUri, path); + if (result_download != Result(ResultType::FETCH) || + result_download != Result(ResultType::FETCH_ALREADY_EXISTS)) { - return path; + gzerr << "Unable to download model[" << _uri << "]" << std::endl; + return std::string(); + } + else + { + gzmsg << "Downloaded model URL: " << std::endl + << " " << _uri << std::endl + << " to: " << std::endl + << " " << path << std::endl; } } - - if (!this->dataPtr->fuelClient->DownloadModel(fuelUri, path)) - { - gzerr << "Unable to download model[" << _uri << "]" << std::endl; - return std::string(); - } - else - { - gzmsg << "Downloaded model URL: " << std::endl - << " " << _uri << std::endl - << " to: " << std::endl - << " " << path << std::endl; + if (path.empty()) { + if ((this->dataPtr->fuelClient->ParseModelFileUrl(fuelUri, model, fileUrl) && + !this->dataPtr->fuelClient->CachedModelFile(fuelUri, path)) + || _forceDownload) + { + auto modelUri = _uri.substr(0, + _uri.find("files", model.UniqueName().size())-1); + Result result_download = + this->dataPtr->fuelClient->DownloadModel(ignition::common::URI(modelUri), path); + if (result_download != Result(ResultType::FETCH) || + result_download != Result(ResultType::FETCH_ALREADY_EXISTS)) + { + gzerr << "Unable to download model[" << _uri << "]" << std::endl; + return std::string(); + } + else + { + gzmsg << "Downloaded model URL: " << std::endl + << " " << _uri << std::endl + << " to: " << std::endl + << " " << path << std::endl; + path += "/" + fileUrl; + } + } } return path; From 34026fd25d2d63b4c2dd83d9ef196a887e91d417 Mon Sep 17 00:00:00 2001 From: ahcorde Date: Thu, 3 Sep 2020 16:53:39 +0200 Subject: [PATCH 09/14] Added feedback Signed-off-by: ahcorde --- gazebo/Server.cc | 2 +- gazebo/common/FuelModelDatabase.cc | 22 +++++++++++++++++++++- test/integration/factory.cc | 11 ++++++----- 3 files changed, 28 insertions(+), 7 deletions(-) diff --git a/gazebo/Server.cc b/gazebo/Server.cc index 611df672af..7013083a18 100644 --- a/gazebo/Server.cc +++ b/gazebo/Server.cc @@ -410,7 +410,7 @@ bool Server::LoadFile(const std::string &_filename, if (filenameUri.Scheme() == "http" || filenameUri.Scheme() == "https") { - std::string downloadedDir = ignition::fuel_tools::fetchResource(filename); + std::string downloadedDir = common::FuelModelDatabase::Instance()->ModelPath(filename); // Find the first sdf file in the world path for now, the later intention is // to load an optional world config file first and if that does not exist, // continue to load the first sdf file found as done below diff --git a/gazebo/common/FuelModelDatabase.cc b/gazebo/common/FuelModelDatabase.cc index b6cd9484c3..b75df10b23 100644 --- a/gazebo/common/FuelModelDatabase.cc +++ b/gazebo/common/FuelModelDatabase.cc @@ -26,7 +26,9 @@ #include #include +#include #include +#include #include #include "gazebo/gazebo_config.h" @@ -208,13 +210,13 @@ std::string FuelModelDatabase::ModelPath(const std::string &_uri, if (fuelUri.Scheme() != "http" && fuelUri.Scheme() != "https") { - gzwarn << "URI not supported by Fuel [" << _uri << "]" << std::endl; return std::string(); } std::string path; std::string fileUrl; ignition::fuel_tools::ModelIdentifier model; + ignition::fuel_tools::WorldIdentifier world; using namespace ignition::fuel_tools; if ((this->dataPtr->fuelClient->ParseModelUrl(fuelUri, model) && @@ -263,6 +265,24 @@ std::string FuelModelDatabase::ModelPath(const std::string &_uri, } } + // ig path is still empty then try to download a world + if (path.empty()) { + if (this->dataPtr->fuelClient->ParseWorldUrl(fuelUri, world) && + !this->dataPtr->fuelClient->CachedWorld(fuelUri, path)) + { + this->dataPtr->fuelClient->DownloadWorld(fuelUri, path); + } + // Download the world, if it's a world file URI + else if (this->dataPtr->fuelClient->ParseWorldFileUrl(fuelUri, world, fileUrl) && + !this->dataPtr->fuelClient->CachedWorldFile(fuelUri, path)) + { + auto worldUri = _uri.substr(0, + _uri.find("files", world.UniqueName().size())-1); + this->dataPtr->fuelClient->DownloadWorld(ignition::common::URI(worldUri), path); + path += "/" + fileUrl; + } + } + return path; } diff --git a/test/integration/factory.cc b/test/integration/factory.cc index f3c1afee3e..1b305e790e 100644 --- a/test/integration/factory.cc +++ b/test/integration/factory.cc @@ -1222,7 +1222,6 @@ TEST_F(FactoryTest, FilenameModelDatabaseRelativePaths) } ////////////////////////////////////////////////// -#ifdef HAVE_IGNITION_FUEL_TOOLS TEST_F(FactoryTest, FilenameFuelURL) { this->Load("worlds/empty.world", true); @@ -1233,7 +1232,7 @@ TEST_F(FactoryTest, FilenameFuelURL) msgs::Factory msg; msg.set_sdf_filename( - "https://api.ignitionfuel.org/1.0/chapulina/models/Test box"); + "https://fuel.ignitionrobotics.org/1.0/chapulina/models/Test box"); auto pub = this->node->Advertise("~/factory"); pub->Publish(msg); @@ -1250,7 +1249,7 @@ TEST_F(FactoryTest, FilenameFuelURL) ASSERT_NE(nullptr, world->ModelByName("test_box")); } -TEST_F(FactoryTest, worldWithFuelModels) +TEST_F(FactoryTest, WorldWithFuelModels) { this->Load("worlds/fuel_models.world", true); @@ -1270,10 +1269,13 @@ TEST_F(FactoryTest, worldWithFuelModels) ASSERT_NE(nullptr, world->ModelByName("Radio")); } +////////////////////////////////////////////////// TEST_F(FactoryTest, FuelURIAsWorldArgument) { + // ServerFixture::LoadArgs will split whitespaces, so we use a world name + // without spaces this->Load( - "https://staging-fuel.ignitionrobotics.org/1.0/chapulina/worlds/Shapes", + "https://fuel.ignitionrobotics.org/1.0/OpenRobotics/worlds/Test_shapes", true); // World @@ -1311,7 +1313,6 @@ TEST_F(FactoryTest, FuelURIAsWorldArgument) // Check model was spawned ASSERT_NE(nullptr, world->ModelByName("sphere")); } -#endif ////////////////////////////////////////////////// TEST_P(FactoryTest, InvalidMeshInsertion) From ff1138368bf53d4fd2484fb569e1f135454831be Mon Sep 17 00:00:00 2001 From: ahcorde Date: Fri, 4 Sep 2020 10:31:36 +0200 Subject: [PATCH 10/14] Added WorldPath method to FuelModelDatabase Signed-off-by: ahcorde --- gazebo/common/FuelModelDatabase.cc | 47 +++++++++++++++++++----------- gazebo/common/FuelModelDatabase.hh | 10 +++++++ 2 files changed, 40 insertions(+), 17 deletions(-) diff --git a/gazebo/common/FuelModelDatabase.cc b/gazebo/common/FuelModelDatabase.cc index b75df10b23..09cad7bdfd 100644 --- a/gazebo/common/FuelModelDatabase.cc +++ b/gazebo/common/FuelModelDatabase.cc @@ -216,7 +216,6 @@ std::string FuelModelDatabase::ModelPath(const std::string &_uri, std::string path; std::string fileUrl; ignition::fuel_tools::ModelIdentifier model; - ignition::fuel_tools::WorldIdentifier world; using namespace ignition::fuel_tools; if ((this->dataPtr->fuelClient->ParseModelUrl(fuelUri, model) && @@ -265,22 +264,36 @@ std::string FuelModelDatabase::ModelPath(const std::string &_uri, } } - // ig path is still empty then try to download a world - if (path.empty()) { - if (this->dataPtr->fuelClient->ParseWorldUrl(fuelUri, world) && - !this->dataPtr->fuelClient->CachedWorld(fuelUri, path)) - { - this->dataPtr->fuelClient->DownloadWorld(fuelUri, path); - } - // Download the world, if it's a world file URI - else if (this->dataPtr->fuelClient->ParseWorldFileUrl(fuelUri, world, fileUrl) && - !this->dataPtr->fuelClient->CachedWorldFile(fuelUri, path)) - { - auto worldUri = _uri.substr(0, - _uri.find("files", world.UniqueName().size())-1); - this->dataPtr->fuelClient->DownloadWorld(ignition::common::URI(worldUri), path); - path += "/" + fileUrl; - } + return path; +} + +///////////////////////////////////////////////// +std::string FuelModelDatabase::WorldPath(const std::string &_uri, const bool _forceDownload) +{ + auto fuelUri = ignition::common::URI(_uri); + + if (fuelUri.Scheme() != "http" && fuelUri.Scheme() != "https") + { + return std::string(); + } + + std::string path; + std::string fileUrl; + ignition::fuel_tools::WorldIdentifier world; + + if ((this->dataPtr->fuelClient->ParseWorldUrl(fuelUri, world) && + !this->dataPtr->fuelClient->CachedWorld(fuelUri, path)) || _forceDownload) + { + this->dataPtr->fuelClient->DownloadWorld(fuelUri, path); + } + // Download the world, if it's a world file URI + else if (this->dataPtr->fuelClient->ParseWorldFileUrl(fuelUri, world, fileUrl) && + !this->dataPtr->fuelClient->CachedWorldFile(fuelUri, path)) + { + auto worldUri = _uri.substr(0, + _uri.find("files", world.UniqueName().size())-1); + this->dataPtr->fuelClient->DownloadWorld(ignition::common::URI(worldUri), path); + path += "/" + fileUrl; } return path; diff --git a/gazebo/common/FuelModelDatabase.hh b/gazebo/common/FuelModelDatabase.hh index de281036a0..ce7fbe7289 100644 --- a/gazebo/common/FuelModelDatabase.hh +++ b/gazebo/common/FuelModelDatabase.hh @@ -103,6 +103,16 @@ namespace gazebo public: std::string ModelPath(const std::string &_uri, const bool _forceDownload = false); + /// \brief Get the local path to a world. + /// + /// Get the path to a world based on a URI. If the world is on + /// a remote server, then the model fetched and installed locally. + /// \param[in] _uri the model uri + /// \param[in] _forceDownload True to skip searching local cache. + /// \return Local path to a world directory + public: std::string WorldPath(const std::string &_uri, + const bool _forceDownload = false); + /// \brief Get the full local path to a cached file based on its URI. /// \param[in] _uri The file's URI /// \return Local path to the file From 9bad120e6260130351f7f5f86fe334db84b4dad5 Mon Sep 17 00:00:00 2001 From: ahcorde Date: Fri, 4 Sep 2020 10:32:29 +0200 Subject: [PATCH 11/14] Updated logic to load worlds from URLs Signed-off-by: ahcorde --- gazebo/Server.cc | 35 +++++++++++++++++++------------- gazebo/rendering/RenderEngine.cc | 2 +- 2 files changed, 22 insertions(+), 15 deletions(-) diff --git a/gazebo/Server.cc b/gazebo/Server.cc index 7013083a18..f998cb23df 100644 --- a/gazebo/Server.cc +++ b/gazebo/Server.cc @@ -388,15 +388,6 @@ bool Server::GetInitialized() const bool Server::LoadFile(const std::string &_filename, const std::string &_physics) { - // Quick test for a valid file - FILE *test = fopen(common::find_file(_filename).c_str(), "r"); - if (!test) - { - gzerr << "Could not open file[" << _filename << "]\n"; - return false; - } - fclose(test); - // Load the world file sdf::SDFPtr sdf(new sdf::SDF); if (!sdf::init(sdf)) @@ -410,7 +401,7 @@ bool Server::LoadFile(const std::string &_filename, if (filenameUri.Scheme() == "http" || filenameUri.Scheme() == "https") { - std::string downloadedDir = common::FuelModelDatabase::Instance()->ModelPath(filename); + std::string downloadedDir = common::FuelModelDatabase::Instance()->WorldPath(filename); // Find the first sdf file in the world path for now, the later intention is // to load an optional world config file first and if that does not exist, // continue to load the first sdf file found as done below @@ -427,15 +418,31 @@ bool Server::LoadFile(const std::string &_filename, if (fileExtension == "sdf" || fileExtension == "world") { filename = current.c_str(); + if (!sdf::readFile(filename, sdf)) + { + gzerr << "Unable to read SDF from URL[" << filename << "]\n"; + return false; + } } } } } - - if (!sdf::readFile(common::find_file(filename), sdf)) + else { - gzerr << "Unable to read sdf file[" << filename << "]\n"; - return false; + // Quick test for a valid file + FILE *test = fopen(common::find_file(filename).c_str(), "r"); + if (!test) + { + gzerr << "Could not open file[" << filename << "]\n"; + return false; + } + fclose(test); + + if (!sdf::readFile(common::find_file(filename), sdf)) + { + gzerr << "Unable to read sdf file[" << filename << "]\n"; + return false; + } } return this->LoadImpl(sdf->Root(), _physics); diff --git a/gazebo/rendering/RenderEngine.cc b/gazebo/rendering/RenderEngine.cc index cc8a1d0a59..de56615ba4 100644 --- a/gazebo/rendering/RenderEngine.cc +++ b/gazebo/rendering/RenderEngine.cc @@ -465,7 +465,7 @@ void RenderEngine::LoadPlugins() ///////////////////////////////////////////////// void RenderEngine::AddResourcePath(const std::string &_uri) { - if (_uri == "__default__" || _uri.empty()) + if (_uri.find("__default__") != std::string::npos || _uri.empty()) return; std::string path = common::find_file_path(_uri); From 176b2df56aaff948475f5d66091e79ac6030e92d Mon Sep 17 00:00:00 2001 From: ahcorde Date: Fri, 4 Sep 2020 10:38:00 +0200 Subject: [PATCH 12/14] find_file functions behave the same way Signed-off-by: ahcorde --- gazebo/common/CommonIface.cc | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/gazebo/common/CommonIface.cc b/gazebo/common/CommonIface.cc index 7446322cab..525262e80d 100644 --- a/gazebo/common/CommonIface.cc +++ b/gazebo/common/CommonIface.cc @@ -173,7 +173,12 @@ std::string common::find_file(const std::string &_file) ///////////////////////////////////////////////// std::string common::find_file(const std::string &_file, bool _searchLocalPath) { - return common::SystemPaths::Instance()->FindFile(_file, _searchLocalPath); + std::string path = common::FuelModelDatabase::Instance()->ModelPath(_file); + if (path.empty()) + { + path = common::SystemPaths::Instance()->FindFile(_file, _searchLocalPath); + } + return path; } ///////////////////////////////////////////////// From 1528320b37463a68fe64eb28925fc44abd8c6c54 Mon Sep 17 00:00:00 2001 From: ahcorde Date: Mon, 21 Sep 2020 20:37:33 +0200 Subject: [PATCH 13/14] Fixed windows build Signed-off-by: ahcorde --- gazebo/common/FuelModelDatabase.cc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/gazebo/common/FuelModelDatabase.cc b/gazebo/common/FuelModelDatabase.cc index 09cad7bdfd..3ba2b04fb8 100644 --- a/gazebo/common/FuelModelDatabase.cc +++ b/gazebo/common/FuelModelDatabase.cc @@ -27,6 +27,11 @@ #include #include #include +#ifdef _WIN32 + // DELETE is defined in winnt.h and causes a problem with + // ignition::fuel_tools::REST::DELETE + #undef DELETE +#endif #include #include #include @@ -317,4 +322,3 @@ std::string FuelModelDatabase::CachedFilePath(const std::string &_uri) return path; } - From be691e498f3207ad2aa7c93d07810146e2e935c9 Mon Sep 17 00:00:00 2001 From: ahcorde Date: Mon, 21 Sep 2020 22:18:20 +0200 Subject: [PATCH 14/14] Fixed windows build Signed-off-by: ahcorde --- gazebo/Server.cc | 5 +++++ gazebo/common/CommonIface.cc | 5 +++++ gazebo/common/FuelModelDatabase.hh | 5 +++++ 3 files changed, 15 insertions(+) diff --git a/gazebo/Server.cc b/gazebo/Server.cc index f998cb23df..2cb9cd4518 100644 --- a/gazebo/Server.cc +++ b/gazebo/Server.cc @@ -36,6 +36,11 @@ #include #include #include +#ifdef _WIN32 + // DELETE is defined in winnt.h and causes a problem with + // ignition::fuel_tools::REST::DELETE + #undef DELETE +#endif #include #include "gazebo/gazebo.hh" diff --git a/gazebo/common/CommonIface.cc b/gazebo/common/CommonIface.cc index 59d1d0d0ad..271197013b 100644 --- a/gazebo/common/CommonIface.cc +++ b/gazebo/common/CommonIface.cc @@ -47,6 +47,11 @@ #include "gazebo/common/FuelModelDatabase.hh" #include "gazebo/common/SystemPaths.hh" +#ifdef _WIN32 + // DELETE is defined in winnt.h and causes a problem with + // ignition::fuel_tools::REST::DELETE + #undef DELETE +#endif #include "ignition/fuel_tools/Interface.hh" using namespace gazebo; diff --git a/gazebo/common/FuelModelDatabase.hh b/gazebo/common/FuelModelDatabase.hh index ce7fbe7289..81daafbefc 100644 --- a/gazebo/common/FuelModelDatabase.hh +++ b/gazebo/common/FuelModelDatabase.hh @@ -21,6 +21,11 @@ #include #include #include +#ifdef _WIN32 + // DELETE is defined in winnt.h and causes a problem with + // ignition::fuel_tools::REST::DELETE + #undef DELETE +#endif #include #include