Merge pull request #164 from Esri/issue98-avoid-RPK-unpacking

#98: adding new palladio_fs component to avoid RPK unpacking

src/CMakeLists.txt

@@ -94,7 +94,8 @@ endif()
 
 add_subdirectory(codec)
 add_subdirectory(palladio)
-add_dependencies(palladio palladio_codec)
+add_subdirectory(palladio_fs)
+add_dependencies(palladio palladio_codec palladio_fs)
 
 add_subdirectory(test EXCLUDE_FROM_ALL)
 

src/codec/encoder/HoudiniEncoder.cpp

@@ -226,7 +226,7 @@ void convertMaterialToAttributeMap(prtx::PRTUtils::AttributeMapBuilderPtr& aBuil
 
 			case prtx::Material::PT_TEXTURE: {
 				const auto& t = prtxAttr.getTexture(key);
-				const std::wstring p = uriToPath(t);
+				const std::wstring p = t->getURI()->wstring();
 				aBuilder->setString(key.c_str(), p.c_str());
 				break;
 			}
@@ -235,7 +235,8 @@ void convertMaterialToAttributeMap(prtx::PRTUtils::AttributeMapBuilderPtr& aBuil
 				const auto& ta = prtxAttr.getTextureArray(key);
 
 				prtx::WStringVector pa(ta.size());
-				std::transform(ta.begin(), ta.end(), pa.begin(), uriToPath);
+				std::transform(ta.begin(), ta.end(), pa.begin(),
+				               [](const prtx::TexturePtr& t) { return t->getURI()->wstring(); });
 
 				std::vector<const wchar_t*> ppa = toPtrVec(pa);
 				aBuilder->setStringArray(key.c_str(), ppa.data(), ppa.size());
@@ -316,7 +317,8 @@ void forwardGenericAttributes(HoudiniCallbacks* hc, size_t initialShapeIndex, co
 			           case prtx::Attributable::PT_BOOL_ARRAY: {
 				           const prtx::BoolVector& v = shape->getBoolArray(keyStr);
 				           const std::unique_ptr<bool[]> vPtrs(new bool[v.size()]);
-				           for (size_t i = 0; i < v.size(); i++) vPtrs[i] = prtx::toPrimitive(v[i]);
+				           for (size_t i = 0; i < v.size(); i++)
+					           vPtrs[i] = prtx::toPrimitive(v[i]);
 				           hc->attrBoolArray(initialShapeIndex, shape->getID(), key, vPtrs.get(), v.size(), 1);
 				           break;
 			           }

src/palladio/CMakeLists.txt

@@ -48,13 +48,6 @@ if(PLD_WINDOWS)
 	# nothing, inheriting compiler flags from houdini
 
 elseif(PLD_LINUX)
-	target_compile_options(${PROJECT_NAME} PRIVATE
-			-std=c++14 -D_GLIBCXX_USE_CXX11_ABI=0
-			-mtune=generic -m64 -fPIC -mmmx -msse -msse2 -msse3
-			-Wall -W -Wno-parentheses -Wno-sign-compare -Wno-reorder
-			-Wno-uninitialized -Wunused -Wno-unused-parameter
-			-fno-strict-aliasing -Wno-unused-local-typedefs)
-
 	if (${CMAKE_BUILD_TYPE} STREQUAL "Release")
 		target_compile_options(${PROJECT_NAME} PRIVATE -O3 -flto)
 		target_compile_definitions(${PROJECT_NAME} PRIVATE -DNDEBUG)
@@ -73,9 +66,6 @@ elseif(PLD_LINUX)
 			BUILD_WITH_INSTALL_RPATH TRUE)
 
 elseif(PLD_MACOS)
-	target_compile_options(${PROJECT_NAME} PRIVATE
-			-std=c++11) # TODO
-
 	if(${CMAKE_BUILD_TYPE} STREQUAL "RelWithDebInfo")
 		target_compile_options(${PROJECT_NAME} PRIVATE -O3 -g)
 		target_compile_definitions(${PROJECT_NAME} PRIVATE -DNDEBUG)

src/palladio/PalladioMain.cpp

@@ -16,17 +16,15 @@
 
 #include "PalladioMain.h"
 
+#include "LogHandler.h"
 #include "NodeParameter.h"
 #include "PRTContext.h"
 #include "SOPAssign.h"
 #include "SOPGenerate.h"
-#include "LogHandler.h"
 
 #include "OP/OP_OperatorTable.h"
 #include "UT/UT_Exit.h"
 
-//#undef major // fixes warning regarding duplicate macros in <sys/types.h> and <sys/sysmacros.h>
-//#undef minor
 #include "UT/UT_DSOVersion.h"
 
 namespace {

src/palladio/ResolveMapCache.cpp

@@ -22,6 +22,8 @@
 
 namespace {
 
+constexpr bool UNPACK_RULE_PACKAGES = false;
+
 const ResolveMapSPtr RESOLVE_MAP_NONE;
 const ResolveMapCache::LookupResult LOOKUP_FAILURE = {RESOLVE_MAP_NONE, ResolveMapCache::CacheStatus::MISS};
 const std::chrono::system_clock::time_point INVALID_TIMESTAMP;
@@ -155,8 +157,8 @@ ResolveMapCache::LookupResult ResolveMapCache::get(const PLD_BOOST_NS::filesyste
 
 		prt::Status status = prt::STATUS_UNSPECIFIED_ERROR;
 		LOG_DBG << "createResolveMap from " << rpkURI;
-		rmce.mResolveMap.reset(prt::createResolveMap(rpkURI.c_str(), mRPKUnpackPath.wstring().c_str(), &status),
-		                       PRTDestroyer());
+		const wchar_t* extractionPathPtr = UNPACK_RULE_PACKAGES ? mRPKUnpackPath.wstring().c_str() : nullptr;
+		rmce.mResolveMap.reset(prt::createResolveMap(rpkURI.c_str(), extractionPathPtr, &status), PRTDestroyer());
 		if (status != prt::STATUS_OK)
 			return LOOKUP_FAILURE;
 

src/palladio/Utils.cpp

@@ -185,17 +185,21 @@ std::string toUTF8FromOSNarrow(const std::string& osString) {
 	return std::string(temp.data());
 }
 
-std::wstring toFileURI(const PLD_BOOST_NS::filesystem::path& p) {
+std::wstring toFileURI(const std::string& p) {
 #ifdef _WIN32
 	static const std::wstring schema = L"file:/";
 #else
 	static const std::wstring schema = L"file:";
 #endif
-	std::string utf8Path = toUTF8FromOSNarrow(p.generic_string());
+	std::string utf8Path = toUTF8FromOSNarrow(p);
 	std::wstring pecString = percentEncode(utf8Path);
 	return schema + pecString;
 }
 
+std::wstring toFileURI(const PLD_BOOST_NS::filesystem::path& p) {
+	return toFileURI(p.generic_string());
+}
+
 std::wstring percentEncode(const std::string& utf8String) {
 	std::vector<char> temp(2 * utf8String.size());
 	size_t size = temp.size();

src/palladio/Utils.h

@@ -78,6 +78,7 @@ std::wstring toUTF16FromOSNarrow(const std::string& osString);
 std::string toUTF8FromOSNarrow(const std::string& osString);
 
 PLD_TEST_EXPORTS_API std::wstring toFileURI(const PLD_BOOST_NS::filesystem::path& p);
+std::wstring toFileURI(const std::string& p);
 PLD_TEST_EXPORTS_API std::wstring percentEncode(const std::string& utf8String);
 
 inline void replace_all_not_of(std::wstring& s, const std::wstring& allowedChars) {

src/palladio_fs/CMakeLists.txt

@@ -0,0 +1,57 @@
+cmake_minimum_required(VERSION 3.13)
+
+
+### project definition
+
+project(palladio_fs CXX)
+
+
+### target definition
+
+add_library(${PROJECT_NAME} SHARED
+	main.cpp
+	RulePackageFS.cpp
+	../palladio/Utils.cpp
+	../palladio/LogHandler.cpp)
+
+target_include_directories(${PROJECT_NAME} PRIVATE
+		$<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
+		${CMAKE_CURRENT_SOURCE_DIR}/..)
+
+### compiler settings
+
+add_toolchain_definition(${PROJECT_NAME})
+pld_add_version_definitions(${PROJECT_NAME})
+
+if(PLD_WINDOWS)
+	# nothing, inheriting compiler flags from houdini
+
+elseif(PLD_LINUX)
+	if (${CMAKE_BUILD_TYPE} STREQUAL "Release")
+		target_compile_options(${PROJECT_NAME} PRIVATE -O3 -flto)
+		target_compile_definitions(${PROJECT_NAME} PRIVATE -DNDEBUG)
+	elseif(${CMAKE_BUILD_TYPE} STREQUAL "RelWithDebInfo")
+		target_compile_options(${PROJECT_NAME} PRIVATE -O3 -ggdb -pg)
+		target_compile_definitions(${PROJECT_NAME} PRIVATE -DNDEBUG)
+	elseif(${CMAKE_BUILD_TYPE} STREQUAL "Debug")
+		target_compile_options(${PROJECT_NAME} PRIVATE -O0 -ggdb -pg)
+		target_compile_definitions(${PROJECT_NAME} PRIVATE -DDEBUG)
+	endif()
+
+	set_target_properties(${PROJECT_NAME} PROPERTIES
+			INSTALL_RPATH "\$ORIGIN/../../packages/palladio"
+			INSTALL_RPATH_USE_LINK_PATH FALSE
+			SKIP_RPATH FALSE
+			BUILD_WITH_INSTALL_RPATH TRUE)
+endif()
+
+
+### dependencies
+
+pld_add_dependency_prt(${PROJECT_NAME})
+pld_add_dependency_houdini(${PROJECT_NAME})
+
+
+### setup install target
+
+install(TARGETS ${PROJECT_NAME} RUNTIME DESTINATION ${HOUDINI_RELATIVE_DSO_PATH}/fs LIBRARY DESTINATION ${HOUDINI_RELATIVE_DSO_PATH}/fs)

src/palladio_fs/RulePackageFS.cpp

@@ -0,0 +1,127 @@
+#include "RulePackageFS.h"
+
+#include "prtx/DataBackend.h" // !!! use of PRTX requires palladio_fs to be built with the same compiler as PRT
+
+#include "FS/FS_ReaderStream.h"
+
+#include <iostream>
+#include <map>
+#include <mutex>
+
+namespace {
+
+const std::string SCHEMA_RPK = "rpk:";
+
+bool isRulePackageURI(const char* p) {
+	if (p == nullptr)
+		return false;
+
+	// needs to start with rpk: schema
+	if (std::strncmp(p, SCHEMA_RPK.c_str(), SCHEMA_RPK.length()) != 0)
+		return false;
+
+	// needs to contain '!' separator
+	if (std::strchr(p, '!') == nullptr)
+		return false;
+
+	return true;
+}
+
+// The base URI is the "inner most" URI as defined by prtx::URI, i.e. the actual file
+std::string getBaseURI(const char* p) {
+	const char* lastSchemaSep = std::strrchr(p, ':');
+	const char* firstInnerSep = std::strchr(p, '!');
+	if (lastSchemaSep < firstInnerSep) {
+		return std::string(lastSchemaSep + 1, firstInnerSep);
+	}
+	else
+		return {};
+}
+
+prtx::BinaryVectorPtr resolveRulePackageFile(const char* source, prt::Cache* cache) {
+	assert(source != nullptr);
+	const std::wstring uri = toUTF16FromOSNarrow(source);
+
+	try {
+		std::wstring warnings;
+		const prtx::BinaryVectorPtr data = prtx::DataBackend::resolveBinaryData(cache, uri, nullptr, &warnings);
+		if (!warnings.empty())
+			std::wcerr << L"resolveBinaryData warnings: " << warnings << std::endl;
+		return data;
+	}
+	catch (std::exception& e) {
+		std::cerr << "caught exception: " << e.what() << std::endl;
+	}
+
+	return {};
+}
+
+} // namespace
+
+RulePackageReader::RulePackageReader(prt::Cache* cache) : mCache(cache) {
+	UTaddAbsolutePathPrefix(SCHEMA_RPK.c_str());
+}
+
+FS_ReaderStream* RulePackageReader::createStream(const char* source, const UT_Options*) {
+	if (isRulePackageURI(source)) {
+		const prtx::BinaryVectorPtr buf = resolveRulePackageFile(source, mCache);
+		if (!buf)
+			return nullptr;
+		return new FS_ReaderStream((const char*)buf->data(), buf->size(), 0); // freed by Houdini
+	}
+	return nullptr;
+}
+
+RulePackageInfoHelper::RulePackageInfoHelper(prt::Cache* cache) : mCache(cache) {}
+
+bool RulePackageInfoHelper::canHandle(const char* source) {
+	return isRulePackageURI(source);
+}
+
+bool RulePackageInfoHelper::hasAccess(const char* source, int mode) {
+	std::string src(source);
+	if (isRulePackageURI(source))
+		src = getBaseURI(source);
+	FS_Info info(src.c_str());
+	return info.hasAccess(mode);
+}
+
+bool RulePackageInfoHelper::getIsDirectory(const char* source) {
+	if (isRulePackageURI(source))
+		return false;
+	FS_Info info(source);
+	return info.getIsDirectory();
+}
+
+int RulePackageInfoHelper::getModTime(const char* source) {
+	std::string src(source);
+	if (isRulePackageURI(source))
+		src = getBaseURI(source);
+	FS_Info info(src.c_str());
+	return info.getModTime();
+}
+
+int64 RulePackageInfoHelper::getSize(const char* source) {
+	if (isRulePackageURI(source)) {
+		const prtx::BinaryVectorPtr buf = resolveRulePackageFile(source, mCache);
+		return buf->size();
+	}
+	FS_Info info(source);
+	return info.getFileDataSize();
+}
+
+UT_String RulePackageInfoHelper::getExtension(const char* source) {
+	if (isRulePackageURI(source)) {
+		const char* lastSchemaSep = std::strrchr(source, '.');
+		return UT_String(lastSchemaSep);
+	}
+	FS_Info info(source);
+	return info.getExtension();
+}
+
+bool RulePackageInfoHelper::getContents(const char* source, UT_StringArray& contents, UT_StringArray* dirs) {
+	if (isRulePackageURI(source))
+		return false; // we only support individual texture files atm
+	FS_Info info(source);
+	return info.getContents(contents, dirs);
+}

src/palladio_fs/RulePackageFS.h

@@ -0,0 +1,39 @@
+#pragma once
+
+#include "FS/FS_Info.h"
+#include "FS/FS_Reader.h"
+
+#include "palladio/Utils.h"
+
+/**
+ * support for nested file RPK URIs:
+ * rpk:file:/path/to/file.rpk!/nested/file.cgb
+ */
+
+class RulePackageReader : public FS_ReaderHelper {
+public:
+	RulePackageReader(prt::Cache* cache);
+	virtual ~RulePackageReader() = default;
+
+	FS_ReaderStream* createStream(const char* source, const UT_Options* options) override;
+
+private:
+	prt::Cache* mCache;
+};
+
+class RulePackageInfoHelper : public FS_InfoHelper {
+public:
+	RulePackageInfoHelper(prt::Cache* cache);
+	virtual ~RulePackageInfoHelper() = default;
+
+	bool canHandle(const char* source) override;
+	bool hasAccess(const char* source, int mode) override;
+	bool getIsDirectory(const char* source) override;
+	int getModTime(const char* source) override;
+	int64 getSize(const char* source) override;
+	UT_String getExtension(const char* source) override;
+	bool getContents(const char* source, UT_StringArray& contents, UT_StringArray* dirs) override;
+
+private:
+	prt::Cache* mCache;
+};

src/palladio_fs/main.cpp

@@ -0,0 +1,25 @@
+#include "RulePackageFS.h"
+
+#include "FS/FS_Utils.h" // required for installFSHelpers to work below
+#include "UT/UT_DSOVersion.h" // required for valid Houdini DSO
+
+#include <iostream>
+#include <memory>
+
+namespace {
+
+CacheObjectUPtr prtCache; // TODO: prevent from growing too much
+
+std::unique_ptr<RulePackageReader> rpkReader;
+std::unique_ptr<RulePackageInfoHelper> rpkInfoHelper;
+
+} // namespace
+
+void installFSHelpers() {
+	prtCache.reset(prt::CacheObject::create(prt::CacheObject::CACHE_TYPE_NONREDUNDANT));
+
+	rpkReader = std::make_unique<RulePackageReader>(prtCache.get());
+	rpkInfoHelper = std::make_unique<RulePackageInfoHelper>(prtCache.get());
+
+	std::clog << "Palladio: Registered custom FS reader for Rule Packages.\n";
+}