vpkpp: add support for Volition VPP version 3

CMakeLists.txt

@@ -37,6 +37,7 @@ option(SOURCEPP_BUILD_WITH_OPENCL     "Build with support for GPU compute"
 option(SOURCEPP_BUILD_WITH_TBB        "Build with support for std::execution"         OFF)
 option(SOURCEPP_BUILD_WITH_THREADS    "Build with support for threading"               ON)
 option(SOURCEPP_BUILD_TESTS           "Build tests for supported libraries"           OFF)
+option(SOURCEPP_BUILD_TESTS_EXTRA     "Build extra tests that can't be run in CI"     OFF)
 option(SOURCEPP_BUILD_WIN7_COMPAT     "Build with Windows 7 compatibility"            OFF)
 
 option(SOURCEPP_LINK_STATIC_MSVC_RUNTIME "Link to static MSVC runtime library" OFF)
@@ -217,6 +218,9 @@ if(SOURCEPP_BUILD_TESTS)
     add_executable(${SOURCEPP_TEST_NAME} ${${SOURCEPP_TEST_NAME}_SOURCES})
     target_link_libraries(${SOURCEPP_TEST_NAME} PUBLIC ${${SOURCEPP_TEST_NAME}_DEPS})
     target_compile_definitions(${SOURCEPP_TEST_NAME} PUBLIC ASSET_ROOT="${CMAKE_CURRENT_SOURCE_DIR}/test/res/")
+    if(SOURCEPP_BUILD_TESTS_EXTRA)
+        target_compile_definitions(${SOURCEPP_TEST_NAME} PUBLIC SOURCEPP_BUILD_TESTS_EXTRA)
+    endif()
     include(GoogleTest)
     gtest_discover_tests(${SOURCEPP_TEST_NAME})
 endif()
@@ -273,6 +277,6 @@ endif()
 # Print options
 print_options(OPTIONS
         USE_BSPPP USE_DMXPP USE_FSPP USE_GAMEPP USE_KVPP USE_MDLPP USE_SNDPP USE_STEAMPP USE_TOOLPP USE_VCRYPTPP USE_VPKPP USE_VTFPP
-        BUILD_BENCHMARKS BUILD_C_WRAPPERS BUILD_CSHARP_WRAPPERS BUILD_PYTHON_WRAPPERS BUILD_WITH_OPENCL BUILD_WITH_TBB BUILD_WITH_THREADS BUILD_TESTS BUILD_WIN7_COMPAT
+        BUILD_BENCHMARKS BUILD_C_WRAPPERS BUILD_CSHARP_WRAPPERS BUILD_PYTHON_WRAPPERS BUILD_WITH_OPENCL BUILD_WITH_TBB BUILD_WITH_THREADS BUILD_TESTS BUILD_TESTS_EXTRA BUILD_WIN7_COMPAT
         LINK_STATIC_MSVC_RUNTIME
         VPKPP_SUPPORT_VPK_V54)

include/vpkpp/format/VPP.h

@@ -8,9 +8,17 @@ constexpr uint32_t VPP_SIGNATURE_LIL = 0x51890ACE;
 constexpr uint32_t VPP_SIGNATURE_BIG = 0xCE0A8951;
 constexpr uint32_t VPP_ALIGNMENT = 2048;
 constexpr std::string_view VPP_EXTENSION = ".vpp";
+constexpr std::string_view VPP_EXTENSION_PC = ".vpp_pc";
+constexpr std::string_view VPP_EXTENSION_XBOX2 = ".vpp_xbox2";
 
 class VPP : public PackFileReadOnly {
 public:
+	enum Flags : uint32_t {
+		FLAG_NONE       = 0,
+		FLAG_COMPRESSED = 1 << 0,
+		FLAG_CONDENSED  = 1 << 1
+	};
+
 	/// Open a VPP file
 	[[nodiscard]] static std::unique_ptr<PackFile> open(const std::string& path, const EntryCallback& callback = nullptr);
 
@@ -27,8 +35,15 @@ class VPP : public PackFileReadOnly {
 protected:
 	using PackFileReadOnly::PackFileReadOnly;
 
+	Flags flags = FLAG_NONE;
+	uint32_t entryBaseOffset = 0;
+	std::vector<std::byte> uncondensedData;
+
 private:
 	VPKPP_REGISTER_PACKFILE_OPEN(VPP_EXTENSION, &VPP::open);
+	VPKPP_REGISTER_PACKFILE_OPEN(VPP_EXTENSION_PC, &VPP::open);
+	VPKPP_REGISTER_PACKFILE_OPEN(VPP_EXTENSION_XBOX2, &VPP::open);
 };
+SOURCEPP_BITFLAGS_ENUM(VPP::Flags)
 
 } // namespace vpkpp

src/vpkpp/format/VPP.cpp

@@ -4,6 +4,8 @@
 
 #include <FileStream.h>
 
+#include <miniz.h>
+
 using namespace sourcepp;
 using namespace vpkpp;
 
@@ -41,9 +43,13 @@ std::unique_ptr<PackFile> VPP::open(const std::string& path, const EntryCallback
 		static constexpr uint32_t headerSize = sizeof(uint32_t) * 4;
 		reader.seek_in(headerSize + sourcepp::math::paddingForAlignment(VPP_ALIGNMENT, headerSize));
 
-		// Get first file offset
+		// Get base file offset
 		const uint32_t fileTableSize = (60 + sizeof(uint32_t)) * entryCount;
-		uint32_t entryOffset = reader.tell_in() + fileTableSize + sourcepp::math::paddingForAlignment(VPP_ALIGNMENT, fileTableSize);
+		vpp->entryBaseOffset = headerSize    + sourcepp::math::paddingForAlignment(VPP_ALIGNMENT, headerSize)
+		                     + fileTableSize + sourcepp::math::paddingForAlignment(VPP_ALIGNMENT, fileTableSize);
+
+		// Get first file offset
+		uint32_t entryOffset = 0;
 
 		// Read file entries
 		for (uint32_t i = 0; i < entryCount; i++) {
@@ -79,9 +85,13 @@ std::unique_ptr<PackFile> VPP::open(const std::string& path, const EntryCallback
 		static constexpr uint32_t headerSize = sizeof(uint32_t) * 4;
 		reader.seek_in(headerSize + sourcepp::math::paddingForAlignment(VPP_ALIGNMENT, headerSize));
 
-		// Get first file offset
+		// Get base file offset
 		const uint32_t fileTableSize = (24 + sizeof(uint32_t) * 2) * entryCount;
-		uint32_t entryOffset = reader.tell_in() + fileTableSize + sourcepp::math::paddingForAlignment(VPP_ALIGNMENT, fileTableSize);
+		vpp->entryBaseOffset = headerSize    + sourcepp::math::paddingForAlignment(VPP_ALIGNMENT, headerSize)
+		                     + fileTableSize + sourcepp::math::paddingForAlignment(VPP_ALIGNMENT, fileTableSize);
+
+		// Get first file offset
+		uint32_t entryOffset = 0;
 
 		// Read file entries
 		for (uint32_t i = 0; i < entryCount; i++) {
@@ -111,6 +121,95 @@ std::unique_ptr<PackFile> VPP::open(const std::string& path, const EntryCallback
 				callback(entryPath, entry);
 			}
 		}
+	} else if (version == 3) {
+		// Skip unused header data
+		reader.skip_in(64 + 256 + sizeof(uint32_t));
+
+		// Get package flags
+		reader >> vpp->flags;
+
+		// Remove compressed flag if we're also condensed
+		if (vpp->flags & FLAG_CONDENSED) {
+			vpp->flags &= ~FLAG_COMPRESSED;
+		}
+
+		// ??
+		reader.skip_in<uint32_t>();
+
+		// Get number of entries
+		const auto entryCount = reader.read<uint32_t>();
+
+		// Verify file size
+		if (reader.read<uint32_t>() != std::filesystem::file_size(path)) {
+			return nullptr;
+		}
+
+		// Get sizes
+		const auto entryDirectorySize = reader.read<uint32_t>();
+		const auto entryNamesSize = reader.read<uint32_t>();
+
+		// Check if we have compression
+		const auto entryDataSizeUncompressed = reader.read<uint32_t>();
+		const auto entryDataSizeCompressed = reader.read<uint32_t>();
+
+		// Set base data offset
+		vpp->entryBaseOffset = VPP_ALIGNMENT
+		                     + entryDirectorySize + sourcepp::math::paddingForAlignment(VPP_ALIGNMENT, entryDirectorySize)
+		                     + entryNamesSize     + sourcepp::math::paddingForAlignment(VPP_ALIGNMENT, entryNamesSize);
+
+		// Seek to file directory (alignment boundary)
+		reader.seek_in(VPP_ALIGNMENT);
+
+		// Read file entries
+		for (uint32_t i = 0; i < entryCount; i++) {
+			Entry entry = createNewEntry();
+
+			// Get file name offset
+			const auto entryNameOffset = reader.read<uint32_t>();
+
+			// ??
+			reader.skip_in<uint32_t>();
+
+			// Get file offset
+			entry.offset = reader.read<uint32_t>();
+
+			// ??
+			reader.skip_in<uint32_t>();
+
+			// Get file size
+			entry.length = reader.read<uint32_t>();
+			entry.compressedLength = reader.read<uint32_t>();
+
+			// Not compressed
+			if (entry.compressedLength == 0xFFFFFFFF || !(vpp->flags & FLAG_COMPRESSED)) {
+				entry.compressedLength = 0;
+			}
+
+			// ??
+			reader.skip_in<uint32_t>();
+
+			// Get file name
+			const auto lastPos = reader.tell_in();
+			reader.seek_in(VPP_ALIGNMENT + entryDirectorySize + sourcepp::math::paddingForAlignment(VPP_ALIGNMENT, entryDirectorySize) + entryNameOffset);
+			const auto entryPath = vpp->cleanEntryPath(reader.read_string(entryNamesSize - entryNameOffset));
+			reader.seek_in_u(lastPos);
+
+			// Put it in
+			vpp->entries.emplace(entryPath, entry);
+
+			if (callback) {
+				callback(entryPath, entry);
+			}
+		}
+
+		// Uncondense data
+		if (vpp->flags & FLAG_CONDENSED) {
+			reader.seek_in(vpp->entryBaseOffset);
+			vpp->uncondensedData.resize(entryDataSizeUncompressed);
+			auto compressedData = reader.read_bytes(entryDataSizeCompressed);
+			mz_ulong uncompressedLength = entryDataSizeUncompressed;
+			mz_uncompress(reinterpret_cast<unsigned char*>(vpp->uncondensedData.data()), &uncompressedLength, reinterpret_cast<const unsigned char*>(compressedData.data()), entryDataSizeCompressed);
+		}
 	} else {
 		return nullptr;
 	}
@@ -128,13 +227,35 @@ std::optional<std::vector<std::byte>> VPP::readEntry(const std::string& path_) c
 		return readUnbakedEntry(*entry);
 	}
 
-	// It's baked into the file on disk
-	FileStream stream{this->fullFilePath};
-	if (!stream) {
-		return std::nullopt;
+	if (this->flags & FLAG_CONDENSED) {
+		// Condensed entry
+		BufferStreamReadOnly stream{this->uncondensedData.data(), this->uncondensedData.size()};
+		stream.seek_u(entry->offset);
+		return stream.read_bytes(entry->length);
+	} else if (this->flags & FLAG_COMPRESSED) {
+		// Compressed entry
+		if (!entry->compressedLength) {
+			return std::nullopt;
+		}
+		FileStream stream{this->fullFilePath};
+		if (!stream) {
+			return std::nullopt;
+		}
+		stream.seek_in_u(this->entryBaseOffset + entry->offset);
+		auto compressedData = stream.read_bytes(entry->compressedLength);
+		mz_ulong uncompressedLength = entry->length;
+		std::vector<std::byte> uncompressedData(uncompressedLength);
+		mz_uncompress(reinterpret_cast<unsigned char*>(uncompressedData.data()), &uncompressedLength, reinterpret_cast<const unsigned char*>(compressedData.data()), entry->compressedLength);
+		return uncompressedData;
+	} else {
+		// Uncompressed entry
+		FileStream stream{this->fullFilePath};
+		if (!stream) {
+			return std::nullopt;
+		}
+		stream.seek_in_u(this->entryBaseOffset + entry->offset);
+		return stream.read_bytes(entry->length);
 	}
-	stream.seek_in_u(entry->offset);
-	return stream.read_bytes(entry->length);
 }
 
 Attribute VPP::getSupportedEntryAttributes() const {

test/fspp.cpp

@@ -1,5 +1,4 @@
-// These tests should be run manually outside CI
-#if 0
+#ifdef SOURCEPP_BUILD_TESTS_EXTRA
 
 #include <gtest/gtest.h>
 

test/gamepp.cpp

@@ -1,4 +1,4 @@
-#if 0
+#ifdef SOURCEPP_BUILD_TESTS_EXTRA
 
 #include <gtest/gtest.h>
 
@@ -8,7 +8,11 @@ using namespace gamepp;
 
 TEST(gamepp, commandsBlocking) {
 	auto game = GameInstance::find();
-	ASSERT_TRUE(game);
+	if (!game) {
+		// We won't fail here because this is expected to fail unless its specifically being tested
+		std::cerr << "Failed to find game!" << std::endl;
+		return SUCCEED();
+	}
 
 	// For Portal 2 - it can *almost* play through the whole map
 	(*game)

test/steampp.cpp

@@ -1,5 +1,4 @@
-// These tests should be run manually outside CI
-#if 0
+#ifdef SOURCEPP_BUILD_TESTS_EXTRA
 
 #include <gtest/gtest.h>
 
@@ -12,22 +11,24 @@ TEST(steampp, list_installed_apps) {
 	Steam steam;
 	ASSERT_TRUE(steam);
 
-	std::cout << "Steam install directory: " << steam.getInstallDir() << std::endl;
+	std::cout << "Steam install directory: " << steam.getInstallDir();
 
 	for (auto appID : steam.getInstalledApps()) {
-		std::cout << steam.getAppName(appID) << " (" << appID << "): " << steam.getAppInstallDir(appID) << std::endl;
+		std::cout << '\n' << steam.getAppName(appID) << " (" << appID << "): " << steam.getAppInstallDir(appID);
 	}
+	std::cout << std::endl;
 }
 
 TEST(steampp, search_for_apps_using_engine) {
 	Steam steam;
 	ASSERT_TRUE(steam);
 
 	for (auto appID : steam.getInstalledApps()) {
-		if (steam.isAppUsingSourceEngine(appID) || steam.isAppUsingSource2Engine(appID)) {
-			std::cout << steam.getAppName(appID) << " (" << appID << "): " << steam.getAppInstallDir(appID) << std::endl;
+		if (steam.isAppUsingGoldSrcEngine(appID) || steam.isAppUsingSourceEngine(appID) || steam.isAppUsingSource2Engine(appID)) {
+			std::cout << steam.getAppName(appID) << " (" << appID << "): " << steam.getAppInstallDir(appID) << '\n';
 		}
 	}
+	std::cout << std::endl;
 }
 
 #endif

test/vpkpp.cpp

@@ -5,20 +5,48 @@
 using namespace sourcepp;
 using namespace vpkpp;
 
+#ifdef SOURCEPP_BUILD_TESTS_EXTRA
+	#define VPKPP_PRINT_ALL_PATHS(packfile) packfile->runForAllEntries([](const std::string& path, const Entry&) {std::cout << path << std::endl;})
+#else
+	#define VPKPP_PRINT_ALL_PATHS(packfile) static_cast<void>(packfile)
+#endif
+
 TEST(vpkpp, hog_read) {
 	const auto hog = PackFile::open(ASSET_ROOT "vpkpp/hog/chaos.hog");
 	ASSERT_TRUE(hog);
+	VPKPP_PRINT_ALL_PATHS(hog);
 	EXPECT_EQ(hog->getEntryCount(), 5);
+	EXPECT_TRUE(hog->hasEntry("chaos1.rdl"));
 }
 
 TEST(vpkpp, vpp_v1_read) {
 	const auto vpp = PackFile::open(ASSET_ROOT "vpkpp/vpp/v1.vpp");
 	ASSERT_TRUE(vpp);
+	VPKPP_PRINT_ALL_PATHS(vpp);
 	EXPECT_EQ(vpp->getEntryCount(), 29);
+	EXPECT_TRUE(vpp->hasEntry("maps.txt"));
 }
 
 TEST(vpkpp, vpp_v2_read) {
 	const auto vpp = PackFile::open(ASSET_ROOT "vpkpp/vpp/v2.vpp");
 	ASSERT_TRUE(vpp);
+	VPKPP_PRINT_ALL_PATHS(vpp);
 	EXPECT_EQ(vpp->getEntryCount(), 32);
+	EXPECT_TRUE(vpp->hasEntry("credits.tbl"));
+}
+
+TEST(vpkpp, vpp_v3_lil_read) {
+	const auto vpp = PackFile::open(ASSET_ROOT "vpkpp/vpp/v3.vpp_pc");
+	ASSERT_TRUE(vpp);
+	VPKPP_PRINT_ALL_PATHS(vpp);
+	EXPECT_EQ(vpp->getEntryCount(), 128);
+	EXPECT_TRUE(vpp->hasEntry("address.rfgvpx.str2_pc"));
+}
+
+TEST(vpkpp, vpp_v3_big_read) {
+	const auto vpp = PackFile::open(ASSET_ROOT "vpkpp/vpp/v3.vpp_xbox2");
+	ASSERT_TRUE(vpp);
+	VPKPP_PRINT_ALL_PATHS(vpp);
+	EXPECT_EQ(vpp->getEntryCount(), 20);
+	EXPECT_TRUE(vpp->hasEntry("pretty.txt"));
 }

test/vtfpp.cpp

@@ -155,7 +155,7 @@ TEST(vtfpp, read_ttx_no_ttz) {
 	EXPECT_EQ(vtf.getThumbnailHeight(), 4);
 }
 
-#if 0
+#ifdef SOURCEPP_BUILD_TESTS_EXTRA
 
 #define TEST_WRITE_FMT(Format, Flags) \
 		TEST(vtfpp, write_fmt_##Format) { \