Add ability for PCK to remove files

* This was "kindof" existing in PCK but not properly done or supported. It seems this work was initiated over a decade ago and never completed.
* If a file has offset zero in a PCK, it will be considered removed and it will be removed if previously added.

This adds the possibility to add proper directory level diffing to godotengine#97118.

core/io/file_access_pack.cpp

@@ -96,6 +96,41 @@ void PackedData::add_path(const String &p_pkg_path, const String &p_path, uint64
 	}
 }
 
+void PackedData::remove_path(const String &p_path) {
+	String simplified_path = p_path.simplify_path();
+	PathMD5 pmd5(simplified_path.md5_buffer());
+
+	bool exists = files.has(pmd5);
+
+	if (!exists) {
+		// Do not remove something that does not exist.
+		return;
+	}
+
+	//search for dir
+	String p = simplified_path.replace_first("res://", "");
+	PackedDir *cd = root;
+
+	if (p.contains("/")) { //in a subdir
+
+		Vector<String> ds = p.get_base_dir().split("/");
+
+		for (int j = 0; j < ds.size(); j++) {
+			if (!cd->subdirs.has(ds[j])) {
+				return; // Subdir does not exist, do not bother creating
+			} else {
+				cd = cd->subdirs[ds[j]];
+			}
+		}
+	}
+	String filename = simplified_path.get_file();
+	cd->files.erase(filename);
+
+	// erase file
+
+	files.erase(pmd5);
+}
+
 void PackedData::add_pack_source(PackSource *p_source) {
 	if (p_source != nullptr) {
 		sources.push_back(p_source);
@@ -259,7 +294,11 @@ bool PackedSourcePCK::try_open_pack(const String &p_path, bool p_replace_files,
 		f->get_buffer(md5, 16);
 		uint32_t flags = f->get_32();
 
-		PackedData::get_singleton()->add_path(p_path, path, ofs + p_offset, size, md5, this, p_replace_files, (flags & PACK_FILE_ENCRYPTED));
+		if (ofs == PackedData::FILE_REMOVED_OFFSET) { // The file was removed
+			PackedData::get_singleton()->remove_path(path);
+		} else {
+			PackedData::get_singleton()->add_path(p_path, path, ofs + p_offset, size, md5, this, p_replace_files, (flags & PACK_FILE_ENCRYPTED));
+		}
 	}
 
 	return true;

core/io/file_access_pack.h

@@ -69,6 +69,10 @@ class PackedData {
 		bool encrypted;
 	};
 
+	enum {
+		FILE_REMOVED_OFFSET = 0
+	};
+
 private:
 	struct PackedDir {
 		PackedDir *parent = nullptr;
@@ -111,6 +115,7 @@ class PackedData {
 public:
 	void add_pack_source(PackSource *p_source);
 	void add_path(const String &p_pkg_path, const String &p_path, uint64_t p_ofs, uint64_t p_size, const uint8_t *p_md5, PackSource *p_src, bool p_replace_files, bool p_encrypted = false); // for PackSource
+	void remove_path(const String &p_path); // for PackSource
 
 	void set_disabled(bool p_disabled) { disabled = p_disabled; }
 	_FORCE_INLINE_ bool is_disabled() const { return disabled; }
@@ -193,7 +198,7 @@ Ref<FileAccess> PackedData::try_open_path(const String &p_path) {
 	if (!E) {
 		return nullptr; //not found
 	}
-	if (E->value.offset == 0) {
+	if (E->value.offset == PackedData::FILE_REMOVED_OFFSET) {
 		return nullptr; //was erased
 	}
 

core/io/pck_packer.cpp

@@ -49,6 +49,7 @@ static int _get_pad(int p_alignment, int p_n) {
 void PCKPacker::_bind_methods() {
 	ClassDB::bind_method(D_METHOD("pck_start", "pck_name", "alignment", "key", "encrypt_directory"), &PCKPacker::pck_start, DEFVAL(32), DEFVAL("0000000000000000000000000000000000000000000000000000000000000000"), DEFVAL(false));
 	ClassDB::bind_method(D_METHOD("add_file", "pck_path", "source_path", "encrypt"), &PCKPacker::add_file, DEFVAL(false));
+	ClassDB::bind_method(D_METHOD("add_file_removal", "pck_path"), &PCKPacker::add_file_removal);
 	ClassDB::bind_method(D_METHOD("flush", "verbose"), &PCKPacker::flush, DEFVAL(false));
 }
 
@@ -106,6 +107,24 @@ Error PCKPacker::pck_start(const String &p_file, int p_alignment, const String &
 	return OK;
 }
 
+Error PCKPacker::add_file_removal(const String &p_file) {
+	ERR_FAIL_COND_V_MSG(file.is_null(), ERR_INVALID_PARAMETER, "File must be opened before use.");
+
+	File pf;
+	// Simplify path here and on every 'files' access so that paths that have extra '/'
+	// symbols in them still match to the MD5 hash for the saved path.
+	pf.path = p_file.simplify_path();
+	pf.ofs = 0;
+	pf.size = 0;
+	for (int i = 0; i < 16; i++) {
+		pf.md5.push_back(0);
+	}
+
+	files.push_back(pf);
+
+	return OK;
+}
+
 Error PCKPacker::add_file(const String &p_file, const String &p_src, bool p_encrypt) {
 	ERR_FAIL_COND_V_MSG(file.is_null(), ERR_INVALID_PARAMETER, "File must be opened before use.");
 
@@ -218,6 +237,10 @@ Error PCKPacker::flush(bool p_verbose) {
 
 	int count = 0;
 	for (int i = 0; i < files.size(); i++) {
+		if (files[i].ofs == 0 && files[i].size == 0 && files[i].src_path == String()) {
+			continue;
+		}
+
 		Ref<FileAccess> src = FileAccess::open(files[i].src_path, FileAccess::READ);
 		uint64_t to_write = files[i].size;
 

core/io/pck_packer.h

@@ -60,6 +60,7 @@ class PCKPacker : public RefCounted {
 public:
 	Error pck_start(const String &p_file, int p_alignment = 32, const String &p_key = "0000000000000000000000000000000000000000000000000000000000000000", bool p_encrypt_directory = false);
 	Error add_file(const String &p_file, const String &p_src, bool p_encrypt = false);
+	Error add_file_removal(const String &p_file);
 	Error flush(bool p_verbose = false);
 
 	PCKPacker() {}

doc/classes/PCKPacker.xml

@@ -33,6 +33,13 @@
 				Adds the [param source_path] file to the current PCK package at the [param pck_path] internal path (should start with [code]res://[/code]).
 			</description>
 		</method>
+		<method name="add_file_removal">
+			<return type="int" enum="Error" />
+			<param index="0" name="pck_path" type="String" />
+			<description>
+				Addition a file removal to the PCK. This is used for PCK diffing. If this file has been loaded from a previous PCK, it will be removed.
+			</description>
+		</method>
 		<method name="flush">
 			<return type="int" enum="Error" />
 			<param index="0" name="verbose" type="bool" default="false" />