Skip to content

feat(vpkpp): cache read-only pack file handles #20

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

feat(vpkpp): cache read-only pack file handles #20

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 15 additions & 9 deletions include/vpkpp/PackFile.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
#include <unordered_map>
#include <vector>

#include <FileStream.h>
#include <sourcepp/math/Integer.h>
#include <sourcepp/Macros.h>
#include <tsl/htrie_map.h>
Expand Down Expand Up @@ -63,21 +64,21 @@ class PackFile {

/// Verify the checksums of each file, if a file fails the check its path will be added to the vector
/// If there is no checksum ability in the format, it will return an empty vector
[[nodiscard]] virtual std::vector<std::string> verifyEntryChecksums() const;
[[nodiscard]] virtual std::vector<std::string> verifyEntryChecksums();

/// Returns true if the entire file has a checksum
[[nodiscard]] virtual bool hasPackFileChecksum() const;

/// Verify the checksum of the entire file, returns true on success
/// Will return true if there is no checksum ability in the format
[[nodiscard]] virtual bool verifyPackFileChecksum() const;
[[nodiscard]] virtual bool verifyPackFileChecksum();

/// Returns true if the file is signed
[[nodiscard]] virtual bool hasPackFileSignature() const;

/// Verify the file signature, returns true on success
/// Will return true if there is no signature ability in the format
[[nodiscard]] virtual bool verifyPackFileSignature() const;
[[nodiscard]] virtual bool verifyPackFileSignature();

/// Does the format support case-sensitive file names?
[[nodiscard]] virtual constexpr bool isCaseSensitive() const noexcept {
Expand All @@ -91,10 +92,10 @@ class PackFile {
[[nodiscard]] std::optional<Entry> findEntry(const std::string& path_, bool includeUnbaked = true) const;

/// Try to read the entry's data to a bytebuffer
[[nodiscard]] virtual std::optional<std::vector<std::byte>> readEntry(const std::string& path_) const = 0;
[[nodiscard]] virtual std::optional<std::vector<std::byte>> readEntry(const std::string& path_) = 0;

/// Try to read the entry's data to a string
[[nodiscard]] std::optional<std::string> readEntryText(const std::string& path) const;
[[nodiscard]] std::optional<std::string> readEntryText(const std::string& path);

[[nodiscard]] virtual constexpr bool isReadOnly() const noexcept {
return false;
Expand Down Expand Up @@ -131,16 +132,16 @@ class PackFile {
virtual bool bake(const std::string& outputDir_ /*= ""*/, BakeOptions options /*= {}*/, const EntryCallback& callback /*= nullptr*/) = 0;

/// Extract the given entry to disk at the given file path
bool extractEntry(const std::string& entryPath, const std::string& filepath) const; // NOLINT(*-use-nodiscard)
bool extractEntry(const std::string& entryPath, const std::string& filepath); // NOLINT(*-use-nodiscard)

/// Extract the given directory to disk under the given output directory
bool extractDirectory(const std::string& dir_, const std::string& outputDir) const; // NOLINT(*-use-nodiscard)
bool extractDirectory(const std::string& dir_, const std::string& outputDir); // NOLINT(*-use-nodiscard)

/// Extract the contents of the pack file to disk at the given directory
bool extractAll(const std::string& outputDir, bool createUnderPackFileDir = true) const; // NOLINT(*-use-nodiscard)
bool extractAll(const std::string& outputDir, bool createUnderPackFileDir = true); // NOLINT(*-use-nodiscard)

/// Extract the contents of the pack file to disk at the given directory - only entries which match the predicate are extracted
bool extractAll(const std::string& outputDir, const EntryPredicate& predicate, bool stripSharedDirs = true) const; // NOLINT(*-use-nodiscard)
bool extractAll(const std::string& outputDir, const EntryPredicate& predicate, bool stripSharedDirs = true); // NOLINT(*-use-nodiscard)

/// Get entries saved to disk
[[nodiscard]] const EntryTrie& getBakedEntries() const;
Expand Down Expand Up @@ -187,10 +188,14 @@ class PackFile {
protected:
explicit PackFile(std::string fullFilePath_);

[[nodiscard]] bool isReadHandleOpen() const;

void runForAllEntriesInternal(const std::function<void(const std::string&, Entry&)>& operation, bool includeUnbaked = true);

void runForAllEntriesInternal(const std::string& parentDir, const std::function<void(const std::string&, Entry&)>& operation, bool recursive = true, bool includeUnbaked = true);

[[nodiscard]] std::vector<std::string> verifyEntryChecksumsUsingCRC32();

[[nodiscard]] std::vector<std::string> verifyEntryChecksumsUsingCRC32() const;

virtual void addEntryInternal(Entry& entry, const std::string& path, std::vector<std::byte>& buffer, EntryOptions options) = 0;
Expand All @@ -214,6 +219,7 @@ class PackFile {
static const OpenFactoryFunction& registerOpenExtensionForTypeFactory(std::string_view extension, const OpenFactoryFunction& factory);

std::string fullFilePath;
std::optional<FileStream> readHandle;

PackFileType type = PackFileType::UNKNOWN;

Expand Down
4 changes: 2 additions & 2 deletions include/vpkpp/format/GCF.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -116,9 +116,9 @@ class GCF : public PackFileReadOnly {
return true;
}

[[nodiscard]] std::vector<std::string> verifyEntryChecksums() const override;
[[nodiscard]] std::vector<std::string> verifyEntryChecksums() override;

[[nodiscard]] std::optional<std::vector<std::byte>> readEntry(const std::string& path_) const override;
[[nodiscard]] std::optional<std::vector<std::byte>> readEntry(const std::string& path_) override;

[[nodiscard]] Attribute getSupportedEntryAttributes() const override;

Expand Down
6 changes: 3 additions & 3 deletions include/vpkpp/format/GMA.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,13 +31,13 @@ class GMA : public PackFile {
return true;
}

[[nodiscard]] std::vector<std::string> verifyEntryChecksums() const override;
[[nodiscard]] std::vector<std::string> verifyEntryChecksums() override;

[[nodiscard]] bool hasPackFileChecksum() const override;

[[nodiscard]] bool verifyPackFileChecksum() const override;
[[nodiscard]] bool verifyPackFileChecksum() override;

[[nodiscard]] std::optional<std::vector<std::byte>> readEntry(const std::string& path_) const override;
[[nodiscard]] std::optional<std::vector<std::byte>> readEntry(const std::string& path_) override;

bool bake(const std::string& outputDir_ /*= ""*/, BakeOptions options /*= {}*/, const EntryCallback& callback /*= nullptr*/) override;

Expand Down
2 changes: 1 addition & 1 deletion include/vpkpp/format/PAK.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ class PAK : public PackFile {
/// Open a PAK file
[[nodiscard]] static std::unique_ptr<PackFile> open(const std::string& path, const EntryCallback& callback = nullptr);

[[nodiscard]] std::optional<std::vector<std::byte>> readEntry(const std::string& path_) const override;
[[nodiscard]] std::optional<std::vector<std::byte>> readEntry(const std::string& path_) override;

bool bake(const std::string& outputDir_ /*= ""*/, BakeOptions options /*= {}*/, const EntryCallback& callback /*= nullptr*/) override;

Expand Down
2 changes: 1 addition & 1 deletion include/vpkpp/format/PCK.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ class PCK : public PackFile {
return true;
}

[[nodiscard]] std::optional<std::vector<std::byte>> readEntry(const std::string& path_) const override;
[[nodiscard]] std::optional<std::vector<std::byte>> readEntry(const std::string& path_) override;

bool bake(const std::string& outputDir_ /*= ""*/, BakeOptions options /*= {}*/, const EntryCallback& callback /*= nullptr*/) override;

Expand Down
9 changes: 4 additions & 5 deletions include/vpkpp/format/VPK.h
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
#pragma once

#include <array>
#include <tuple>

#include "../PackFile.h"

Expand Down Expand Up @@ -75,17 +74,17 @@ class VPK : public PackFile {
return true;
}

[[nodiscard]] std::vector<std::string> verifyEntryChecksums() const override;
[[nodiscard]] std::vector<std::string> verifyEntryChecksums() override;

[[nodiscard]] bool hasPackFileChecksum() const override;

[[nodiscard]] bool verifyPackFileChecksum() const override;
[[nodiscard]] bool verifyPackFileChecksum() override;

[[nodiscard]] bool hasPackFileSignature() const override;

[[nodiscard]] bool verifyPackFileSignature() const override;
[[nodiscard]] bool verifyPackFileSignature() override;

[[nodiscard]] std::optional<std::vector<std::byte>> readEntry(const std::string& path_) const override;
[[nodiscard]] std::optional<std::vector<std::byte>> readEntry(const std::string& path_) override;

bool removeEntry(const std::string& filename_) override;

Expand Down
2 changes: 1 addition & 1 deletion include/vpkpp/format/VPK_VTMB.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ class VPK_VTMB : public PackFile {
/// Open Vampire: The Masquerade - Bloodlines VPK files
[[nodiscard]] static std::unique_ptr<PackFile> open(const std::string& path, const EntryCallback& callback = nullptr);

[[nodiscard]] std::optional<std::vector<std::byte>> readEntry(const std::string& path_) const override;
[[nodiscard]] std::optional<std::vector<std::byte>> readEntry(const std::string& path_) override;

bool bake(const std::string& outputDir_ /*= ""*/, BakeOptions options /*= {}*/, const EntryCallback& callback /*= nullptr*/) override;

Expand Down
4 changes: 2 additions & 2 deletions include/vpkpp/format/ZIP.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,13 +27,13 @@ class ZIP : public PackFile {
return true;
}

[[nodiscard]] std::vector<std::string> verifyEntryChecksums() const override;
[[nodiscard]] std::vector<std::string> verifyEntryChecksums() override;

[[nodiscard]] constexpr bool isCaseSensitive() const noexcept override {
return true;
}

[[nodiscard]] std::optional<std::vector<std::byte>> readEntry(const std::string& path_) const override;
[[nodiscard]] std::optional<std::vector<std::byte>> readEntry(const std::string& path_) override;

bool bake(const std::string& outputDir_ /*= ""*/, BakeOptions options /*= {}*/, const EntryCallback& callback /*= nullptr*/) override;

Expand Down
22 changes: 13 additions & 9 deletions src/vpkpp/PackFile.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -146,23 +146,23 @@ PackFileType PackFile::getType() const {
return this->type;
}

std::vector<std::string> PackFile::verifyEntryChecksums() const {
std::vector<std::string> PackFile::verifyEntryChecksums() {
return {};
}

bool PackFile::hasPackFileChecksum() const {
return false;
}

bool PackFile::verifyPackFileChecksum() const {
bool PackFile::verifyPackFileChecksum() {
return true;
}

bool PackFile::hasPackFileSignature() const {
return false;
}

bool PackFile::verifyPackFileSignature() const {
bool PackFile::verifyPackFileSignature() {
return true;
}

Expand All @@ -183,7 +183,7 @@ std::optional<Entry> PackFile::findEntry(const std::string& path_, bool includeU
return std::nullopt;
}

std::optional<std::string> PackFile::readEntryText(const std::string& path) const {
std::optional<std::string> PackFile::readEntryText(const std::string& path) {
auto bytes = this->readEntry(path);
if (!bytes) {
return std::nullopt;
Expand Down Expand Up @@ -358,7 +358,7 @@ std::size_t PackFile::removeDirectory(const std::string& dirName_) {
return count;
}

bool PackFile::extractEntry(const std::string& entryPath, const std::string& filepath) const {
bool PackFile::extractEntry(const std::string& entryPath, const std::string& filepath) {
if (filepath.empty()) {
return false;
}
Expand All @@ -377,7 +377,7 @@ bool PackFile::extractEntry(const std::string& entryPath, const std::string& fil
return true;
}

bool PackFile::extractDirectory(const std::string& dir_, const std::string& outputDir) const {
bool PackFile::extractDirectory(const std::string& dir_, const std::string& outputDir) {
auto dir = this->cleanEntryPath(dir_);
dir += '/';
if (dir == "/") {
Expand All @@ -402,7 +402,7 @@ bool PackFile::extractDirectory(const std::string& dir_, const std::string& outp
return noneFailed;
}

bool PackFile::extractAll(const std::string& outputDir, bool createUnderPackFileDir) const {
bool PackFile::extractAll(const std::string& outputDir, bool createUnderPackFileDir) {
if (outputDir.empty()) {
return false;
}
Expand All @@ -424,7 +424,7 @@ bool PackFile::extractAll(const std::string& outputDir, bool createUnderPackFile
return noneFailed;
}

bool PackFile::extractAll(const std::string& outputDir, const EntryPredicate& predicate, bool stripSharedDirs) const {
bool PackFile::extractAll(const std::string& outputDir, const EntryPredicate& predicate, bool stripSharedDirs) {
if (outputDir.empty() || !predicate) {
return false;
}
Expand Down Expand Up @@ -633,7 +633,11 @@ std::string PackFile::escapeEntryPathForWrite(const std::string& path) {
#endif
}

std::vector<std::string> PackFile::verifyEntryChecksumsUsingCRC32() const {
bool PackFile::isReadHandleOpen() const {
return this->readHandle && *this->readHandle;
}

std::vector<std::string> PackFile::verifyEntryChecksumsUsingCRC32() {
std::vector<std::string> out;
this->runForAllEntries([this, &out](const std::string& path, const Entry& entry) {
if (!entry.crc32) {
Expand Down
19 changes: 2 additions & 17 deletions src/vpkpp/format/BSP.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ std::unique_ptr<PackFile> BSP::open(const std::string& path, const EntryCallback
}

auto* bsp = new BSP{path};
auto packFile = std::unique_ptr<PackFile>(bsp);
auto packFile = std::unique_ptr<PackFile>{bsp};

if (!(*bsp)) {
// File failed to load, or has an invalid signature
Expand All @@ -44,22 +44,7 @@ std::unique_ptr<PackFile> BSP::open(const std::string& path, const EntryCallback
writer.write(*pakFileLump);
} else {
// No paklump, create an empty zip
void* writeStreamHandle = mz_stream_os_create();
if (mz_stream_os_open(writeStreamHandle, bsp->tempBSPPakLumpPath.c_str(), MZ_OPEN_MODE_CREATE | MZ_OPEN_MODE_WRITE)) {
return nullptr;
}
void* writeZipHandle = mz_zip_writer_create();
if (mz_zip_writer_open(writeZipHandle, writeStreamHandle, 0)) {
return nullptr;
}
if (mz_zip_writer_close(writeZipHandle)) {
return nullptr;
}
mz_zip_writer_delete(&writeZipHandle);
if (mz_stream_os_close(writeStreamHandle)) {
return nullptr;
}
mz_stream_os_delete(&writeStreamHandle);
ZIP::create(bsp->tempBSPPakLumpPath);
}

if (!bsp->openZIP(bsp->tempBSPPakLumpPath)) {
Expand Down
11 changes: 7 additions & 4 deletions src/vpkpp/format/FPX.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,11 +44,14 @@ std::unique_ptr<PackFile> FPX::openInternal(const std::string& path, const Entry
}

auto* fpx = new FPX{path};
auto packFile = std::unique_ptr<PackFile>(fpx);
auto packFile = std::unique_ptr<PackFile>{fpx};

FileStream reader{fpx->fullFilePath};
reader.seek_in(0);
reader.read(fpx->header1);
auto& reader = (fpx->readHandle = FileStream{path}).value();
if (!reader) {
return nullptr;
}

reader.seek_in(0).read(fpx->header1);
if (fpx->header1.signature != FPX_SIGNATURE) {
// File is not an FPX
return nullptr;
Expand Down
29 changes: 15 additions & 14 deletions src/vpkpp/format/GCF.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,14 +27,16 @@ std::unique_ptr<PackFile> GCF::open(const std::string& path, const EntryCallback

// Create the pack file
auto* gcf = new GCF{path};
auto packFile = std::unique_ptr<PackFile>(gcf);
auto packFile = std::unique_ptr<PackFile>{gcf};

// open file
FileStream reader(gcf->fullFilePath);
reader.seek_in(0);
auto& reader = (gcf->readHandle = FileStream{path}).value();
if (!reader) {
return nullptr;
}

// we read the main header here (not the block header)
reader.read(gcf->header);
reader.seek_in(0).read(gcf->header);

if (gcf->header.dummy1 != 1 && gcf->header.dummy2 != 1 && gcf->header.gcfversion != 6) {
/**
Expand Down Expand Up @@ -198,7 +200,7 @@ std::unique_ptr<PackFile> GCF::open(const std::string& path, const EntryCallback
return packFile;
}

std::vector<std::string> GCF::verifyEntryChecksums() const {
std::vector<std::string> GCF::verifyEntryChecksums() {
std::vector<std::string> bad;
this->runForAllEntries([this, &bad](const std::string& path, const Entry& entry) {
auto bytes = this->readEntry(path);
Expand All @@ -223,7 +225,12 @@ std::vector<std::string> GCF::verifyEntryChecksums() const {
return bad;
}

std::optional<std::vector<std::byte>> GCF::readEntry(const std::string& path_) const {
std::optional<std::vector<std::byte>> GCF::readEntry(const std::string& path_) {
if (!this->isReadHandleOpen()) {
return std::nullopt;
}
auto& reader = this->readHandle.value();

auto path = this->cleanEntryPath(path_);
auto entry = this->findEntry(path);
if (!entry) {
Expand Down Expand Up @@ -257,22 +264,16 @@ std::optional<std::vector<std::byte>> GCF::readEntry(const std::string& path_) c
return lhs.file_data_offset < rhs.file_data_offset;
});

FileStream stream{this->fullFilePath};
if (!stream) {
//printf("!stream\n");
return std::nullopt;
}

uint64_t remaining = entry->length;

for (const auto& block : toread) {
uint32_t currindex = block.first_data_block_index;
while (currindex <= this->blockheader.count) {
uint64_t curfilepos = static_cast<uint64_t>(this->datablockheader.firstblockoffset) + (static_cast<std::uint64_t>(0x2000) * static_cast<std::uint64_t>(currindex));
stream.seek_in_u(curfilepos);
reader.seek_in_u(curfilepos);
//printf("off %lli block %lu toread %lli should be %llu\n", stream.tellInput(), currindex, remaining, curfilepos);
uint32_t toreadAmt = std::min(remaining, static_cast<uint64_t>(0x2000));
auto streamvec = stream.read_bytes(toreadAmt);
auto streamvec = reader.read_bytes(toreadAmt);
filedata.insert(filedata.end(), streamvec.begin(), streamvec.end());
remaining -= toreadAmt;
currindex = this->fragmap[currindex];
Expand Down
Loading