From 6d48e7069802ba584f29790539a2ea69cc3224f5 Mon Sep 17 00:00:00 2001 From: Juan Date: Mon, 23 Sep 2024 12:39:54 +0200 Subject: [PATCH] Add ability for PCK to remove files * This was "kindof" existing in PCK but not properly done or supported. It seems this work was initiated over a decade ago and never completed. * If a file has offset zero in a PCK, it will be considered removed and it will be removed if previously added. This adds the possibility to add proper directory level diffing to #97118. --- core/io/file_access_pack.cpp | 41 +++++++++++++++++++++++++++++++++++- core/io/file_access_pack.h | 7 +++++- core/io/pck_packer.cpp | 23 ++++++++++++++++++++ core/io/pck_packer.h | 1 + doc/classes/PCKPacker.xml | 7 ++++++ 5 files changed, 77 insertions(+), 2 deletions(-) diff --git a/core/io/file_access_pack.cpp b/core/io/file_access_pack.cpp index eec27ce0aa7f..1f91faa30576 100644 --- a/core/io/file_access_pack.cpp +++ b/core/io/file_access_pack.cpp @@ -96,6 +96,41 @@ void PackedData::add_path(const String &p_pkg_path, const String &p_path, uint64 } } +void PackedData::remove_path(const String &p_path) { + String simplified_path = p_path.simplify_path(); + PathMD5 pmd5(simplified_path.md5_buffer()); + + bool exists = files.has(pmd5); + + if (!exists) { + // Do not remove something that does not exist. + return; + } + + //search for dir + String p = simplified_path.replace_first("res://", ""); + PackedDir *cd = root; + + if (p.contains("/")) { //in a subdir + + Vector ds = p.get_base_dir().split("/"); + + for (int j = 0; j < ds.size(); j++) { + if (!cd->subdirs.has(ds[j])) { + return; // Subdir does not exist, do not bother creating + } else { + cd = cd->subdirs[ds[j]]; + } + } + } + String filename = simplified_path.get_file(); + cd->files.erase(filename); + + // erase file + + files.erase(pmd5); +} + void PackedData::add_pack_source(PackSource *p_source) { if (p_source != nullptr) { sources.push_back(p_source); @@ -259,7 +294,11 @@ bool PackedSourcePCK::try_open_pack(const String &p_path, bool p_replace_files, f->get_buffer(md5, 16); uint32_t flags = f->get_32(); - PackedData::get_singleton()->add_path(p_path, path, ofs + p_offset, size, md5, this, p_replace_files, (flags & PACK_FILE_ENCRYPTED)); + if (ofs == PackedData::FILE_REMOVED_OFFSET) { // The file was removed + PackedData::get_singleton()->remove_path(path); + } else { + PackedData::get_singleton()->add_path(p_path, path, ofs + p_offset, size, md5, this, p_replace_files, (flags & PACK_FILE_ENCRYPTED)); + } } return true; diff --git a/core/io/file_access_pack.h b/core/io/file_access_pack.h index 595a36bca4e8..c63fb864b988 100644 --- a/core/io/file_access_pack.h +++ b/core/io/file_access_pack.h @@ -69,6 +69,10 @@ class PackedData { bool encrypted; }; + enum { + FILE_REMOVED_OFFSET = 0 + }; + private: struct PackedDir { PackedDir *parent = nullptr; @@ -111,6 +115,7 @@ class PackedData { public: void add_pack_source(PackSource *p_source); void add_path(const String &p_pkg_path, const String &p_path, uint64_t p_ofs, uint64_t p_size, const uint8_t *p_md5, PackSource *p_src, bool p_replace_files, bool p_encrypted = false); // for PackSource + void remove_path(const String &p_path); // for PackSource void set_disabled(bool p_disabled) { disabled = p_disabled; } _FORCE_INLINE_ bool is_disabled() const { return disabled; } @@ -193,7 +198,7 @@ Ref PackedData::try_open_path(const String &p_path) { if (!E) { return nullptr; //not found } - if (E->value.offset == 0) { + if (E->value.offset == PackedData::FILE_REMOVED_OFFSET) { return nullptr; //was erased } diff --git a/core/io/pck_packer.cpp b/core/io/pck_packer.cpp index a7c715c31892..9c6b7a6a17ce 100644 --- a/core/io/pck_packer.cpp +++ b/core/io/pck_packer.cpp @@ -49,6 +49,7 @@ static int _get_pad(int p_alignment, int p_n) { void PCKPacker::_bind_methods() { ClassDB::bind_method(D_METHOD("pck_start", "pck_name", "alignment", "key", "encrypt_directory"), &PCKPacker::pck_start, DEFVAL(32), DEFVAL("0000000000000000000000000000000000000000000000000000000000000000"), DEFVAL(false)); ClassDB::bind_method(D_METHOD("add_file", "pck_path", "source_path", "encrypt"), &PCKPacker::add_file, DEFVAL(false)); + ClassDB::bind_method(D_METHOD("add_file_removal", "pck_path"), &PCKPacker::add_file_removal); ClassDB::bind_method(D_METHOD("flush", "verbose"), &PCKPacker::flush, DEFVAL(false)); } @@ -106,6 +107,24 @@ Error PCKPacker::pck_start(const String &p_file, int p_alignment, const String & return OK; } +Error PCKPacker::add_file_removal(const String &p_file) { + ERR_FAIL_COND_V_MSG(file.is_null(), ERR_INVALID_PARAMETER, "File must be opened before use."); + + File pf; + // Simplify path here and on every 'files' access so that paths that have extra '/' + // symbols in them still match to the MD5 hash for the saved path. + pf.path = p_file.simplify_path(); + pf.ofs = 0; + pf.size = 0; + for (int i = 0; i < 16; i++) { + pf.md5.push_back(0); + } + + files.push_back(pf); + + return OK; +} + Error PCKPacker::add_file(const String &p_file, const String &p_src, bool p_encrypt) { ERR_FAIL_COND_V_MSG(file.is_null(), ERR_INVALID_PARAMETER, "File must be opened before use."); @@ -218,6 +237,10 @@ Error PCKPacker::flush(bool p_verbose) { int count = 0; for (int i = 0; i < files.size(); i++) { + if (files[i].ofs == 0 && files[i].size == 0 && files[i].src_path == String()) { + continue; + } + Ref src = FileAccess::open(files[i].src_path, FileAccess::READ); uint64_t to_write = files[i].size; diff --git a/core/io/pck_packer.h b/core/io/pck_packer.h index 8764fc90a015..e0d891c9ce06 100644 --- a/core/io/pck_packer.h +++ b/core/io/pck_packer.h @@ -60,6 +60,7 @@ class PCKPacker : public RefCounted { public: Error pck_start(const String &p_file, int p_alignment = 32, const String &p_key = "0000000000000000000000000000000000000000000000000000000000000000", bool p_encrypt_directory = false); Error add_file(const String &p_file, const String &p_src, bool p_encrypt = false); + Error add_file_removal(const String &p_file); Error flush(bool p_verbose = false); PCKPacker() {} diff --git a/doc/classes/PCKPacker.xml b/doc/classes/PCKPacker.xml index 494e9966ac95..e3185380cc07 100644 --- a/doc/classes/PCKPacker.xml +++ b/doc/classes/PCKPacker.xml @@ -33,6 +33,13 @@ Adds the [param source_path] file to the current PCK package at the [param pck_path] internal path (should start with [code]res://[/code]). + + + + + Addition a file removal to the PCK. This is used for PCK diffing. If this file has been loaded from a previous PCK, it will be removed. + +