feat(vpkpp): add common zip compression formats

README.md

@@ -125,6 +125,7 @@ Several modern C++20 libraries for sanely parsing Valve formats, rolled into one
         <li>GMA v1-3 (Garry's Mod)</li>
         <li><a href="https://quakewiki.org/wiki/.pak">PAK</a> (Quake, WON Half-Life)</li>
         <li><a href="https://docs.godotengine.org/en/stable/tutorials/export/exporting_pcks.html">PCK</a> v1-2 (Godot Engine)</li>
+        <li><a href="">PK3</a> (Quake III)</li>
         <li><a href="https://developer.valvesoftware.com/wiki/VPK">VPK</a> v1-2</li>
         <li>VPK (Vampire: The Masquerade - Bloodlines)</li>
         <li>ZIP</li>

THIRDPARTY_LEGAL_NOTICES.txt

@@ -79,6 +79,21 @@ OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
+---------------    liblzma     ---------------
+
+Permission to use, copy, modify, and/or distribute this
+software for any purpose with or without fee is hereby granted.
+
+THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL
+WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL
+THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR
+CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
+NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+
 ---------------     miniz      ---------------
 
 Copyright 2013-2014 RAD Game Tools and Valve Software
@@ -172,3 +187,61 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 THE SOFTWARE.
+
+
+---------------      zlib      ---------------
+
+(C) 1995-2024 Jean-loup Gailly and Mark Adler
+
+This software is provided 'as-is', without any express or implied
+warranty.  In no event will the authors be held liable for any damages
+arising from the use of this software.
+
+Permission is granted to anyone to use this software for any purpose,
+including commercial applications, and to alter it and redistribute it
+freely, subject to the following restrictions:
+
+1. The origin of this software must not be misrepresented; you must not
+   claim that you wrote the original software. If you use this software
+   in a product, an acknowledgment in the product documentation would be
+   appreciated but is not required.
+2. Altered source versions must be plainly marked as such, and must not be
+   misrepresented as being the original software.
+3. This notice may not be removed or altered from any source distribution.
+
+Jean-loup Gailly        Mark Adler
+jloup@gzip.org          madler@alumni.caltech.edu
+
+
+---------------      zstd      ---------------
+
+BSD License
+
+For Zstandard software
+
+Copyright (c) Meta Platforms, Inc. and affiliates. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+ * Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+ * Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+ * Neither the name Facebook, nor Meta, nor the names of its contributors may
+   be used to endorse or promote products derived from this software without
+   specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

ext/_ext.cmake

@@ -40,10 +40,10 @@ endif()
 # minizip-ng
 if(NOT TARGET MINIZIP::minizip)
     set(MZ_COMPAT           OFF CACHE INTERNAL "")
-    set(MZ_ZLIB             OFF CACHE INTERNAL "")
-    set(MZ_BZIP2            OFF CACHE INTERNAL "")
-    set(MZ_LZMA             OFF CACHE INTERNAL "")
-    set(MZ_ZSTD             OFF CACHE INTERNAL "")
+    set(MZ_ZLIB             ON  CACHE INTERNAL "")
+    set(MZ_BZIP2            ON  CACHE INTERNAL "")
+    set(MZ_LZMA             ON  CACHE INTERNAL "")
+    set(MZ_ZSTD             ON  CACHE INTERNAL "")
     set(MZ_LIBCOMP          OFF CACHE INTERNAL "")
     set(MZ_PKCRYPT          OFF CACHE INTERNAL "")
     set(MZ_WZAES            OFF CACHE INTERNAL "")

include/vpkpp/Options.h

@@ -4,7 +4,25 @@
 
 namespace vpkpp {
 
+enum class EntryCompressionType : int8_t {
+	NO_OVERRIDE = -1,
+
+	// Matches MZ_COMPRESS_METHOD_<type> defines
+	NO_COMPRESS =  0,
+	DEFLATE     =  8,
+	BZIP2       = 12,
+	LZMA        = 14,
+	ZSTD        = 93,
+	XZ          = 95,
+};
+
 struct BakeOptions {
+	/// BSP/ZIP - Override compression type of all stored entries
+	EntryCompressionType zip_compressionTypeOverride = EntryCompressionType::NO_OVERRIDE;
+
+	/// BSP/ZIP - Compression strength (use -1 for the compression type's default)
+	int8_t zip_compressionStrength = -1;
+
 	/// GMA - Write CRCs for files and the overall GMA file when baking
 	bool gma_writeCRCs = true;
 
@@ -13,6 +31,9 @@ struct BakeOptions {
 };
 
 struct EntryOptions {
+	/// BSP/ZIP - The compression format
+	EntryCompressionType zip_compressionType = EntryCompressionType::NO_COMPRESS;
+
 	/// VPK - Save this entry to the directory VPK
 	bool vpk_saveToDirectory = false;
 

include/vpkpp/format/ZIP.h

@@ -5,7 +5,12 @@
 namespace vpkpp {
 
 constexpr std::string_view BMZ_EXTENSION = ".bmz";
+constexpr std::string_view BZ2_EXTENSION = ".bz2";
+constexpr std::string_view GZIP_EXTENSION = ".gz";
+constexpr std::string_view PK3_EXTENSION = ".pk3";
+constexpr std::string_view XZ_EXTENSION = ".xz";
 constexpr std::string_view ZIP_EXTENSION = ".zip";
+constexpr std::string_view ZSTD_EXTENSION = ".zstd";
 
 class ZIP : public PackFile {
 public:
@@ -33,18 +38,12 @@ class ZIP : public PackFile {
 
 	[[nodiscard]] Attribute getSupportedEntryAttributes() const override;
 
-#ifdef VPKPP_ZIP_COMPRESSION
-	[[nodiscard]] uint16_t getCompressionMethod() const;
-
-	void setCompressionMethod(uint16_t compressionMethod);
-#endif
-
 protected:
 	explicit ZIP(const std::string& fullFilePath_);
 
 	void addEntryInternal(Entry& entry, const std::string& path, std::vector<std::byte>& buffer, EntryOptions options) override;
 
-	bool bakeTempZip(const std::string& writeZipPath, const EntryCallback& callback);
+	bool bakeTempZip(const std::string& writeZipPath, BakeOptions options, const EntryCallback& callback);
 
 	bool openZIP(std::string_view path);
 
@@ -60,7 +59,12 @@ class ZIP : public PackFile {
 
 private:
 	VPKPP_REGISTER_PACKFILE_OPEN(BMZ_EXTENSION, &ZIP::open);
+	VPKPP_REGISTER_PACKFILE_OPEN(BZ2_EXTENSION, &ZIP::open);
+	VPKPP_REGISTER_PACKFILE_OPEN(GZIP_EXTENSION, &ZIP::open);
+	VPKPP_REGISTER_PACKFILE_OPEN(PK3_EXTENSION, &ZIP::open);
+	VPKPP_REGISTER_PACKFILE_OPEN(XZ_EXTENSION, &ZIP::open);
 	VPKPP_REGISTER_PACKFILE_OPEN(ZIP_EXTENSION, &ZIP::open);
+	VPKPP_REGISTER_PACKFILE_OPEN(ZSTD_EXTENSION, &ZIP::open);
 };
 
 } // namespace vpkpp

src/vpkpp/format/BSP.cpp

@@ -99,7 +99,7 @@ bool BSP::bake(const std::string& outputDir_, BakeOptions options, const EntryCa
 	std::string outputPath = outputDir + '/' + this->getFilename();
 
 	// Use temp folder so we can read from the current ZIP
-	if (!this->bakeTempZip(this->tempZIPPath, callback)) {
+	if (!this->bakeTempZip(this->tempZIPPath, options, callback)) {
 		return false;
 	}
 	this->mergeUnbakedEntries();

src/vpkpp/format/ZIP.cpp

@@ -1,13 +1,12 @@
 #include <vpkpp/format/ZIP.h>
 
 #include <cstring>
+#include <ctime>
 #include <filesystem>
 
 #include <mz.h>
+#include <mz_os.h>
 #include <mz_strm.h>
-#ifdef VPKPP_ZIP_COMPRESSION
-#include <mz_strm_lzma.h>
-#endif
 #include <mz_strm_os.h>
 #include <mz_zip.h>
 #include <mz_zip_rw.h>
@@ -61,13 +60,14 @@ std::unique_ptr<PackFile> ZIP::open(const std::string& path, const EntryCallback
 	}
 
 	for (auto code = mz_zip_goto_first_entry(zip->zipHandle); code == MZ_OK; code = mz_zip_goto_next_entry(zip->zipHandle)) {
+		if (mz_zip_entry_is_dir(zip->zipHandle) == MZ_OK) {
+			continue;
+		}
+
 		mz_zip_file* fileInfo = nullptr;
 		if (mz_zip_entry_get_info(zip->zipHandle, &fileInfo)) {
 			return nullptr;
 		}
-		if (mz_zip_entry_is_dir(zip->zipHandle) == MZ_OK) {
-			continue;
-		}
 
 		auto entryPath = zip->cleanEntryPath(fileInfo->filename);
 
@@ -122,6 +122,12 @@ void ZIP::addEntryInternal(Entry& entry, const std::string& path, std::vector<st
 	entry.length = buffer.size();
 	entry.compressedLength = 0;
 	entry.crc32 = crypto::computeCRC32(buffer);
+
+	if (options.zip_compressionType != EntryCompressionType::NO_OVERRIDE) {
+		entry.flags = static_cast<uint32_t>(options.zip_compressionType);
+	} else {
+		entry.flags = static_cast<uint32_t>(EntryCompressionType::NO_COMPRESS);
+	}
 }
 
 bool ZIP::bake(const std::string& outputDir_, BakeOptions options, const EntryCallback& callback) {
@@ -130,7 +136,7 @@ bool ZIP::bake(const std::string& outputDir_, BakeOptions options, const EntryCa
 	std::string outputPath = outputDir + '/' + this->getFilename();
 
 	// Use temp folder so we can read from the current ZIP
-	if (!this->bakeTempZip(this->tempZIPPath, callback)) {
+	if (!this->bakeTempZip(this->tempZIPPath, options, callback)) {
 		return false;
 	}
 	this->mergeUnbakedEntries();
@@ -150,27 +156,8 @@ Attribute ZIP::getSupportedEntryAttributes() const {
 	return LENGTH | CRC32;
 }
 
-#ifdef VPKPP_ZIP_COMPRESSION
-uint16_t ZIP::getCompressionMethod() const {
-	return this->options.zip_compressionMethod;
-}
-
-void ZIP::setCompressionMethod(uint16_t compressionMethod) {
-	this->options.zip_compressionMethod = compressionMethod;
-}
-#endif
-
-bool ZIP::bakeTempZip(const std::string& writeZipPath, const EntryCallback& callback) {
-	void* writeStreamHandle;
-#ifdef VPKPP_ZIP_COMPRESSION
-	if (this->options.zip_compressionMethod != MZ_COMPRESS_METHOD_STORE) {
-		writeStreamHandle = mz_stream_lzma_create();
-	} else {
-#endif
-		writeStreamHandle = mz_stream_os_create();
-#ifdef VPKPP_ZIP_COMPRESSION
-	}
-#endif
+bool ZIP::bakeTempZip(const std::string& writeZipPath, BakeOptions options, const EntryCallback& callback) {
+	void* writeStreamHandle = mz_stream_os_create();
 	if (mz_stream_open(writeStreamHandle, writeZipPath.c_str(), MZ_OPEN_MODE_CREATE | MZ_OPEN_MODE_WRITE)) {
 		return false;
 	}
@@ -180,29 +167,35 @@ bool ZIP::bakeTempZip(const std::string& writeZipPath, const EntryCallback& call
 		return false;
 	}
 
-#ifdef VPKPP_ZIP_COMPRESSION
-	if (this->options.zip_compressionMethod != MZ_COMPRESS_METHOD_STORE) {
-		mz_zip_writer_set_compress_level(writeZipHandle, MZ_COMPRESS_LEVEL_DEFAULT);
-		mz_zip_writer_set_compress_method(writeZipHandle, MZ_COMPRESS_METHOD_LZMA);
-	}
-#endif
+	const auto time = std::time(nullptr);
+
+	const bool overrideCompression = options.zip_compressionTypeOverride != EntryCompressionType::NO_OVERRIDE;
+	mz_zip_writer_set_compress_level(writeStreamHandle, options.zip_compressionStrength);
 
 	bool noneFailed = true;
-	this->runForAllEntries([this, &callback, writeZipHandle, &noneFailed](const std::string& path, const Entry& entry) {
+	this->runForAllEntries([this, &options, &callback, writeStreamHandle, writeZipHandle, time, overrideCompression, &noneFailed](const std::string& path, const Entry& entry) {
 		auto binData = this->readEntry(path);
 		if (!binData) {
 			return;
 		}
 
+		if (overrideCompression) {
+			mz_zip_writer_set_compress_method(writeStreamHandle, static_cast<uint16_t>(options.zip_compressionTypeOverride));
+		} else {
+			mz_zip_writer_set_compress_method(writeStreamHandle, entry.flags);
+		}
+
 		mz_zip_file fileInfo;
 		std::memset(&fileInfo, 0, sizeof(mz_zip_entry));
-		fileInfo.flag = MZ_ZIP_FLAG_DATA_DESCRIPTOR;
 		fileInfo.filename = path.c_str();
 		fileInfo.filename_size = path.length();
+		fileInfo.version_madeby = MZ_VERSION_MADEBY;
+		fileInfo.modified_date = time;
+		fileInfo.compression_method = overrideCompression ? static_cast<uint16_t>(options.zip_compressionTypeOverride) : entry.flags;
+		fileInfo.flag = MZ_ZIP_FLAG_DATA_DESCRIPTOR | MZ_ZIP_FLAG_UTF8;
 		fileInfo.uncompressed_size = static_cast<int64_t>(entry.length);
 		fileInfo.compressed_size = static_cast<int64_t>(entry.compressedLength);
 		fileInfo.crc = entry.crc32;
-		fileInfo.compression_method = MZ_COMPRESS_METHOD_STORE;
 		if (mz_zip_writer_add_buffer(writeZipHandle, binData->data(), static_cast<int>(binData->size()), &fileInfo)) {
 			noneFailed = false;
 			return;
@@ -248,9 +241,11 @@ void ZIP::closeZIP() {
 	if (this->zipOpen) {
 		mz_zip_close(this->zipHandle);
 		mz_zip_delete(&this->zipHandle);
+		this->zipOpen = false;
 	}
 	if (this->streamOpen) {
 		mz_stream_close(this->streamHandle);
 		mz_stream_delete(&this->streamHandle);
+		this->streamOpen = false;
 	}
 }