From ac4345d63076a41e58916c2f1eed6b436e59ff34 Mon Sep 17 00:00:00 2001 From: Simon Haegler Date: Sun, 4 Aug 2019 23:35:53 +0200 Subject: [PATCH] #98: adding new palladio_fs component to avoid RPK unpacking Some linux distros need a workaround: ensure that LD_LIBRARY_PATH is set (add some dummy value if needed), then the houdini launcher script will actually prepend the current dsolib subdir (without dsolib on the runpath, palladio_fs will not correctly load). --- src/CMakeLists.txt | 3 +- src/codec/encoder/HoudiniEncoder.cpp | 8 +- src/palladio/CMakeLists.txt | 10 --- src/palladio/PalladioMain.cpp | 4 +- src/palladio/ResolveMapCache.cpp | 6 +- src/palladio/Utils.cpp | 8 +- src/palladio/Utils.h | 1 + src/palladio_fs/CMakeLists.txt | 57 ++++++++++++ src/palladio_fs/RulePackageFS.cpp | 127 +++++++++++++++++++++++++++ src/palladio_fs/RulePackageFS.h | 39 ++++++++ src/palladio_fs/main.cpp | 25 ++++++ 11 files changed, 267 insertions(+), 21 deletions(-) create mode 100644 src/palladio_fs/CMakeLists.txt create mode 100644 src/palladio_fs/RulePackageFS.cpp create mode 100644 src/palladio_fs/RulePackageFS.h create mode 100644 src/palladio_fs/main.cpp diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 9dc912d5..1019f0e5 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -94,7 +94,8 @@ endif() add_subdirectory(codec) add_subdirectory(palladio) -add_dependencies(palladio palladio_codec) +add_subdirectory(palladio_fs) +add_dependencies(palladio palladio_codec palladio_fs) add_subdirectory(test EXCLUDE_FROM_ALL) diff --git a/src/codec/encoder/HoudiniEncoder.cpp b/src/codec/encoder/HoudiniEncoder.cpp index 9c1f9c6b..d87809d6 100644 --- a/src/codec/encoder/HoudiniEncoder.cpp +++ b/src/codec/encoder/HoudiniEncoder.cpp @@ -226,7 +226,7 @@ void convertMaterialToAttributeMap(prtx::PRTUtils::AttributeMapBuilderPtr& aBuil case prtx::Material::PT_TEXTURE: { const auto& t = prtxAttr.getTexture(key); - const std::wstring p = uriToPath(t); + const std::wstring p = t->getURI()->wstring(); aBuilder->setString(key.c_str(), p.c_str()); break; } @@ -235,7 +235,8 @@ void convertMaterialToAttributeMap(prtx::PRTUtils::AttributeMapBuilderPtr& aBuil const auto& ta = prtxAttr.getTextureArray(key); prtx::WStringVector pa(ta.size()); - std::transform(ta.begin(), ta.end(), pa.begin(), uriToPath); + std::transform(ta.begin(), ta.end(), pa.begin(), + [](const prtx::TexturePtr& t) { return t->getURI()->wstring(); }); std::vector ppa = toPtrVec(pa); aBuilder->setStringArray(key.c_str(), ppa.data(), ppa.size()); @@ -316,7 +317,8 @@ void forwardGenericAttributes(HoudiniCallbacks* hc, size_t initialShapeIndex, co case prtx::Attributable::PT_BOOL_ARRAY: { const prtx::BoolVector& v = shape->getBoolArray(keyStr); const std::unique_ptr vPtrs(new bool[v.size()]); - for (size_t i = 0; i < v.size(); i++) vPtrs[i] = prtx::toPrimitive(v[i]); + for (size_t i = 0; i < v.size(); i++) + vPtrs[i] = prtx::toPrimitive(v[i]); hc->attrBoolArray(initialShapeIndex, shape->getID(), key, vPtrs.get(), v.size(), 1); break; } diff --git a/src/palladio/CMakeLists.txt b/src/palladio/CMakeLists.txt index 278dcdcb..fc26d6cc 100644 --- a/src/palladio/CMakeLists.txt +++ b/src/palladio/CMakeLists.txt @@ -48,13 +48,6 @@ if(PLD_WINDOWS) # nothing, inheriting compiler flags from houdini elseif(PLD_LINUX) - target_compile_options(${PROJECT_NAME} PRIVATE - -std=c++14 -D_GLIBCXX_USE_CXX11_ABI=0 - -mtune=generic -m64 -fPIC -mmmx -msse -msse2 -msse3 - -Wall -W -Wno-parentheses -Wno-sign-compare -Wno-reorder - -Wno-uninitialized -Wunused -Wno-unused-parameter - -fno-strict-aliasing -Wno-unused-local-typedefs) - if (${CMAKE_BUILD_TYPE} STREQUAL "Release") target_compile_options(${PROJECT_NAME} PRIVATE -O3 -flto) target_compile_definitions(${PROJECT_NAME} PRIVATE -DNDEBUG) @@ -73,9 +66,6 @@ elseif(PLD_LINUX) BUILD_WITH_INSTALL_RPATH TRUE) elseif(PLD_MACOS) - target_compile_options(${PROJECT_NAME} PRIVATE - -std=c++11) # TODO - if(${CMAKE_BUILD_TYPE} STREQUAL "RelWithDebInfo") target_compile_options(${PROJECT_NAME} PRIVATE -O3 -g) target_compile_definitions(${PROJECT_NAME} PRIVATE -DNDEBUG) diff --git a/src/palladio/PalladioMain.cpp b/src/palladio/PalladioMain.cpp index d7ca3586..4854bff2 100644 --- a/src/palladio/PalladioMain.cpp +++ b/src/palladio/PalladioMain.cpp @@ -16,17 +16,15 @@ #include "PalladioMain.h" +#include "LogHandler.h" #include "NodeParameter.h" #include "PRTContext.h" #include "SOPAssign.h" #include "SOPGenerate.h" -#include "LogHandler.h" #include "OP/OP_OperatorTable.h" #include "UT/UT_Exit.h" -//#undef major // fixes warning regarding duplicate macros in and -//#undef minor #include "UT/UT_DSOVersion.h" namespace { diff --git a/src/palladio/ResolveMapCache.cpp b/src/palladio/ResolveMapCache.cpp index e1bec1fe..f156ceaa 100644 --- a/src/palladio/ResolveMapCache.cpp +++ b/src/palladio/ResolveMapCache.cpp @@ -22,6 +22,8 @@ namespace { +constexpr bool UNPACK_RULE_PACKAGES = false; + const ResolveMapSPtr RESOLVE_MAP_NONE; const ResolveMapCache::LookupResult LOOKUP_FAILURE = {RESOLVE_MAP_NONE, ResolveMapCache::CacheStatus::MISS}; const std::chrono::system_clock::time_point INVALID_TIMESTAMP; @@ -155,8 +157,8 @@ ResolveMapCache::LookupResult ResolveMapCache::get(const PLD_BOOST_NS::filesyste prt::Status status = prt::STATUS_UNSPECIFIED_ERROR; LOG_DBG << "createResolveMap from " << rpkURI; - rmce.mResolveMap.reset(prt::createResolveMap(rpkURI.c_str(), mRPKUnpackPath.wstring().c_str(), &status), - PRTDestroyer()); + const wchar_t* extractionPathPtr = UNPACK_RULE_PACKAGES ? mRPKUnpackPath.wstring().c_str() : nullptr; + rmce.mResolveMap.reset(prt::createResolveMap(rpkURI.c_str(), extractionPathPtr, &status), PRTDestroyer()); if (status != prt::STATUS_OK) return LOOKUP_FAILURE; diff --git a/src/palladio/Utils.cpp b/src/palladio/Utils.cpp index 081773ff..4e49dd7a 100644 --- a/src/palladio/Utils.cpp +++ b/src/palladio/Utils.cpp @@ -185,17 +185,21 @@ std::string toUTF8FromOSNarrow(const std::string& osString) { return std::string(temp.data()); } -std::wstring toFileURI(const PLD_BOOST_NS::filesystem::path& p) { +std::wstring toFileURI(const std::string& p) { #ifdef _WIN32 static const std::wstring schema = L"file:/"; #else static const std::wstring schema = L"file:"; #endif - std::string utf8Path = toUTF8FromOSNarrow(p.generic_string()); + std::string utf8Path = toUTF8FromOSNarrow(p); std::wstring pecString = percentEncode(utf8Path); return schema + pecString; } +std::wstring toFileURI(const PLD_BOOST_NS::filesystem::path& p) { + return toFileURI(p.generic_string()); +} + std::wstring percentEncode(const std::string& utf8String) { std::vector temp(2 * utf8String.size()); size_t size = temp.size(); diff --git a/src/palladio/Utils.h b/src/palladio/Utils.h index e62db205..25d9b267 100644 --- a/src/palladio/Utils.h +++ b/src/palladio/Utils.h @@ -78,6 +78,7 @@ std::wstring toUTF16FromOSNarrow(const std::string& osString); std::string toUTF8FromOSNarrow(const std::string& osString); PLD_TEST_EXPORTS_API std::wstring toFileURI(const PLD_BOOST_NS::filesystem::path& p); +std::wstring toFileURI(const std::string& p); PLD_TEST_EXPORTS_API std::wstring percentEncode(const std::string& utf8String); inline void replace_all_not_of(std::wstring& s, const std::wstring& allowedChars) { diff --git a/src/palladio_fs/CMakeLists.txt b/src/palladio_fs/CMakeLists.txt new file mode 100644 index 00000000..0b7581bc --- /dev/null +++ b/src/palladio_fs/CMakeLists.txt @@ -0,0 +1,57 @@ +cmake_minimum_required(VERSION 3.13) + + +### project definition + +project(palladio_fs CXX) + + +### target definition + +add_library(${PROJECT_NAME} SHARED + main.cpp + RulePackageFS.cpp + ../palladio/Utils.cpp + ../palladio/LogHandler.cpp) + +target_include_directories(${PROJECT_NAME} PRIVATE + $ + ${CMAKE_CURRENT_SOURCE_DIR}/..) + +### compiler settings + +add_toolchain_definition(${PROJECT_NAME}) +pld_add_version_definitions(${PROJECT_NAME}) + +if(PLD_WINDOWS) + # nothing, inheriting compiler flags from houdini + +elseif(PLD_LINUX) + if (${CMAKE_BUILD_TYPE} STREQUAL "Release") + target_compile_options(${PROJECT_NAME} PRIVATE -O3 -flto) + target_compile_definitions(${PROJECT_NAME} PRIVATE -DNDEBUG) + elseif(${CMAKE_BUILD_TYPE} STREQUAL "RelWithDebInfo") + target_compile_options(${PROJECT_NAME} PRIVATE -O3 -ggdb -pg) + target_compile_definitions(${PROJECT_NAME} PRIVATE -DNDEBUG) + elseif(${CMAKE_BUILD_TYPE} STREQUAL "Debug") + target_compile_options(${PROJECT_NAME} PRIVATE -O0 -ggdb -pg) + target_compile_definitions(${PROJECT_NAME} PRIVATE -DDEBUG) + endif() + + set_target_properties(${PROJECT_NAME} PROPERTIES + INSTALL_RPATH "\$ORIGIN/../../packages/palladio" + INSTALL_RPATH_USE_LINK_PATH FALSE + SKIP_RPATH FALSE + BUILD_WITH_INSTALL_RPATH TRUE) +endif() + + +### dependencies + +pld_add_dependency_prt(${PROJECT_NAME}) +pld_add_dependency_houdini(${PROJECT_NAME}) + + +### setup install target + +install(TARGETS ${PROJECT_NAME} RUNTIME DESTINATION ${HOUDINI_RELATIVE_DSO_PATH}/fs LIBRARY DESTINATION ${HOUDINI_RELATIVE_DSO_PATH}/fs) diff --git a/src/palladio_fs/RulePackageFS.cpp b/src/palladio_fs/RulePackageFS.cpp new file mode 100644 index 00000000..c938a422 --- /dev/null +++ b/src/palladio_fs/RulePackageFS.cpp @@ -0,0 +1,127 @@ +#include "RulePackageFS.h" + +#include "prtx/DataBackend.h" // !!! use of PRTX requires palladio_fs to be built with the same compiler as PRT + +#include "FS/FS_ReaderStream.h" + +#include +#include +#include + +namespace { + +const std::string SCHEMA_RPK = "rpk:"; + +bool isRulePackageURI(const char* p) { + if (p == nullptr) + return false; + + // needs to start with rpk: schema + if (std::strncmp(p, SCHEMA_RPK.c_str(), SCHEMA_RPK.length()) != 0) + return false; + + // needs to contain '!' separator + if (std::strchr(p, '!') == nullptr) + return false; + + return true; +} + +// The base URI is the "inner most" URI as defined by prtx::URI, i.e. the actual file +std::string getBaseURI(const char* p) { + const char* lastSchemaSep = std::strrchr(p, ':'); + const char* firstInnerSep = std::strchr(p, '!'); + if (lastSchemaSep < firstInnerSep) { + return std::string(lastSchemaSep + 1, firstInnerSep); + } + else + return {}; +} + +prtx::BinaryVectorPtr resolveRulePackageFile(const char* source, prt::Cache* cache) { + assert(source != nullptr); + const std::wstring uri = toUTF16FromOSNarrow(source); + + try { + std::wstring warnings; + const prtx::BinaryVectorPtr data = prtx::DataBackend::resolveBinaryData(cache, uri, nullptr, &warnings); + if (!warnings.empty()) + std::wcerr << L"resolveBinaryData warnings: " << warnings << std::endl; + return data; + } + catch (std::exception& e) { + std::cerr << "caught exception: " << e.what() << std::endl; + } + + return {}; +} + +} // namespace + +RulePackageReader::RulePackageReader(prt::Cache* cache) : mCache(cache) { + UTaddAbsolutePathPrefix(SCHEMA_RPK.c_str()); +} + +FS_ReaderStream* RulePackageReader::createStream(const char* source, const UT_Options*) { + if (isRulePackageURI(source)) { + const prtx::BinaryVectorPtr buf = resolveRulePackageFile(source, mCache); + if (!buf) + return nullptr; + return new FS_ReaderStream((const char*)buf->data(), buf->size(), 0); // freed by Houdini + } + return nullptr; +} + +RulePackageInfoHelper::RulePackageInfoHelper(prt::Cache* cache) : mCache(cache) {} + +bool RulePackageInfoHelper::canHandle(const char* source) { + return isRulePackageURI(source); +} + +bool RulePackageInfoHelper::hasAccess(const char* source, int mode) { + std::string src(source); + if (isRulePackageURI(source)) + src = getBaseURI(source); + FS_Info info(src.c_str()); + return info.hasAccess(mode); +} + +bool RulePackageInfoHelper::getIsDirectory(const char* source) { + if (isRulePackageURI(source)) + return false; + FS_Info info(source); + return info.getIsDirectory(); +} + +int RulePackageInfoHelper::getModTime(const char* source) { + std::string src(source); + if (isRulePackageURI(source)) + src = getBaseURI(source); + FS_Info info(src.c_str()); + return info.getModTime(); +} + +int64 RulePackageInfoHelper::getSize(const char* source) { + if (isRulePackageURI(source)) { + const prtx::BinaryVectorPtr buf = resolveRulePackageFile(source, mCache); + return buf->size(); + } + FS_Info info(source); + return info.getFileDataSize(); +} + +UT_String RulePackageInfoHelper::getExtension(const char* source) { + if (isRulePackageURI(source)) { + const char* lastSchemaSep = std::strrchr(source, '.'); + return UT_String(lastSchemaSep); + } + FS_Info info(source); + return info.getExtension(); +} + +bool RulePackageInfoHelper::getContents(const char* source, UT_StringArray& contents, UT_StringArray* dirs) { + if (isRulePackageURI(source)) + return false; // we only support individual texture files atm + FS_Info info(source); + return info.getContents(contents, dirs); +} diff --git a/src/palladio_fs/RulePackageFS.h b/src/palladio_fs/RulePackageFS.h new file mode 100644 index 00000000..8107e0e2 --- /dev/null +++ b/src/palladio_fs/RulePackageFS.h @@ -0,0 +1,39 @@ +#pragma once + +#include "FS/FS_Info.h" +#include "FS/FS_Reader.h" + +#include "palladio/Utils.h" + +/** + * support for nested file RPK URIs: + * rpk:file:/path/to/file.rpk!/nested/file.cgb + */ + +class RulePackageReader : public FS_ReaderHelper { +public: + RulePackageReader(prt::Cache* cache); + virtual ~RulePackageReader() = default; + + FS_ReaderStream* createStream(const char* source, const UT_Options* options) override; + +private: + prt::Cache* mCache; +}; + +class RulePackageInfoHelper : public FS_InfoHelper { +public: + RulePackageInfoHelper(prt::Cache* cache); + virtual ~RulePackageInfoHelper() = default; + + bool canHandle(const char* source) override; + bool hasAccess(const char* source, int mode) override; + bool getIsDirectory(const char* source) override; + int getModTime(const char* source) override; + int64 getSize(const char* source) override; + UT_String getExtension(const char* source) override; + bool getContents(const char* source, UT_StringArray& contents, UT_StringArray* dirs) override; + +private: + prt::Cache* mCache; +}; diff --git a/src/palladio_fs/main.cpp b/src/palladio_fs/main.cpp new file mode 100644 index 00000000..01373cac --- /dev/null +++ b/src/palladio_fs/main.cpp @@ -0,0 +1,25 @@ +#include "RulePackageFS.h" + +#include "FS/FS_Utils.h" // required for installFSHelpers to work below +#include "UT/UT_DSOVersion.h" // required for valid Houdini DSO + +#include +#include + +namespace { + +CacheObjectUPtr prtCache; // TODO: prevent from growing too much + +std::unique_ptr rpkReader; +std::unique_ptr rpkInfoHelper; + +} // namespace + +void installFSHelpers() { + prtCache.reset(prt::CacheObject::create(prt::CacheObject::CACHE_TYPE_NONREDUNDANT)); + + rpkReader = std::make_unique(prtCache.get()); + rpkInfoHelper = std::make_unique(prtCache.get()); + + std::clog << "Palladio: Registered custom FS reader for Rule Packages.\n"; +}