Merge pull request #918 from yuriks/romfs

Do not load entire RomFS to memory, read from the file as needed instead (rebased)

src/core/file_sys/archive_romfs.cpp

@@ -17,16 +17,15 @@
 
 namespace FileSys {
 
-ArchiveFactory_RomFS::ArchiveFactory_RomFS(const Loader::AppLoader& app_loader)
-        : romfs_data(std::make_shared<std::vector<u8>>()) {
+ArchiveFactory_RomFS::ArchiveFactory_RomFS(Loader::AppLoader& app_loader) {
     // Load the RomFS from the app
-    if (Loader::ResultStatus::Success != app_loader.ReadRomFS(*romfs_data)) {
+    if (Loader::ResultStatus::Success != app_loader.ReadRomFS(romfs_file, data_offset, data_size)) {
         LOG_ERROR(Service_FS, "Unable to read RomFS!");
     }
 }
 
 ResultVal<std::unique_ptr<ArchiveBackend>> ArchiveFactory_RomFS::Open(const Path& path) {
-    auto archive = Common::make_unique<IVFCArchive>(romfs_data);
+    auto archive = Common::make_unique<IVFCArchive>(romfs_file, data_offset, data_size);
     return MakeResult<std::unique_ptr<ArchiveBackend>>(std::move(archive));
 }
 

src/core/file_sys/archive_romfs.h

@@ -22,14 +22,16 @@ namespace FileSys {
 /// File system interface to the RomFS archive
 class ArchiveFactory_RomFS final : public ArchiveFactory {
 public:
-    ArchiveFactory_RomFS(const Loader::AppLoader& app_loader);
+    ArchiveFactory_RomFS(Loader::AppLoader& app_loader);
 
     std::string GetName() const override { return "RomFS"; }
     ResultVal<std::unique_ptr<ArchiveBackend>> Open(const Path& path) override;
     ResultCode Format(const Path& path) override;
 
 private:
-    std::shared_ptr<std::vector<u8>> romfs_data;
+    std::shared_ptr<FileUtil::IOFile> romfs_file;
+    u64 data_offset;
+    u64 data_size;
 };
 
 } // namespace FileSys

src/core/file_sys/archive_savedatacheck.cpp

@@ -37,17 +37,14 @@ ResultVal<std::unique_ptr<ArchiveBackend>> ArchiveFactory_SaveDataCheck::Open(co
     auto vec = path.AsBinary();
     const u32* data = reinterpret_cast<u32*>(vec.data());
     std::string file_path = GetSaveDataCheckPath(mount_point, data[1], data[0]);
-    FileUtil::IOFile file(file_path, "rb");
+    auto file = std::make_shared<FileUtil::IOFile>(file_path, "rb");
 
-    if (!file.IsOpen()) {
+    if (!file->IsOpen()) {
         return ResultCode(-1); // TODO(Subv): Find the right error code
     }
-    auto size = file.GetSize();
-    auto raw_data = std::make_shared<std::vector<u8>>(size);
-    file.ReadBytes(raw_data->data(), size);
-    file.Close();
+    auto size = file->GetSize();
 
-    auto archive = Common::make_unique<IVFCArchive>(std::move(raw_data));
+    auto archive = Common::make_unique<IVFCArchive>(file, 0, size);
     return MakeResult<std::unique_ptr<ArchiveBackend>>(std::move(archive));
 }
 

src/core/file_sys/disk_archive.cpp

@@ -105,21 +105,21 @@ bool DiskFile::Open() {
     return true;
 }
 
-size_t DiskFile::Read(const u64 offset, const u32 length, u8* buffer) const {
+size_t DiskFile::Read(const u64 offset, const size_t length, u8* buffer) const {
     file->Seek(offset, SEEK_SET);
     return file->ReadBytes(buffer, length);
 }
 
-size_t DiskFile::Write(const u64 offset, const u32 length, const u32 flush, const u8* buffer) const {
+size_t DiskFile::Write(const u64 offset, const size_t length, const bool flush, const u8* buffer) const {
     file->Seek(offset, SEEK_SET);
     size_t written = file->WriteBytes(buffer, length);
     if (flush)
         file->Flush();
     return written;
 }
 
-size_t DiskFile::GetSize() const {
-    return static_cast<size_t>(file->GetSize());
+u64 DiskFile::GetSize() const {
+    return file->GetSize();
 }
 
 bool DiskFile::SetSize(const u64 size) const {

src/core/file_sys/disk_archive.h

@@ -55,10 +55,10 @@ class DiskFile : public FileBackend {
     DiskFile(const DiskArchive& archive, const Path& path, const Mode mode);
 
     bool Open() override;
-    size_t Read(const u64 offset, const u32 length, u8* buffer) const override;
-    size_t Write(const u64 offset, const u32 length, const u32 flush, const u8* buffer) const override;
-    size_t GetSize() const override;
-    bool SetSize(const u64 size) const override;
+    size_t Read(u64 offset, size_t length, u8* buffer) const override;
+    size_t Write(u64 offset, size_t length, bool flush, const u8* buffer) const override;
+    u64 GetSize() const override;
+    bool SetSize(u64 size) const override;
     bool Close() const override;
 
     void Flush() const override {

src/core/file_sys/file_backend.h

@@ -31,7 +31,7 @@ class FileBackend : NonCopyable {
      * @param buffer Buffer to read data into
      * @return Number of bytes read
      */
-    virtual size_t Read(const u64 offset, const u32 length, u8* buffer) const = 0;
+    virtual size_t Read(u64 offset, size_t length, u8* buffer) const = 0;
 
     /**
      * Write data to the file
@@ -41,20 +41,20 @@ class FileBackend : NonCopyable {
      * @param buffer Buffer to read data from
      * @return Number of bytes written
      */
-    virtual size_t Write(const u64 offset, const u32 length, const u32 flush, const u8* buffer) const = 0;
+    virtual size_t Write(u64 offset, size_t length, bool flush, const u8* buffer) const = 0;
 
     /**
      * Get the size of the file in bytes
      * @return Size of the file in bytes
      */
-    virtual size_t GetSize() const = 0;
+    virtual u64 GetSize() const = 0;
 
     /**
      * Set the size of the file in bytes
      * @param size New size of the file
      * @return true if successful
      */
-    virtual bool SetSize(const u64 size) const = 0;
+    virtual bool SetSize(u64 size) const = 0;
 
     /**
      * Close the file

src/core/file_sys/ivfc_archive.cpp

@@ -16,15 +16,12 @@
 
 namespace FileSys {
 
-IVFCArchive::IVFCArchive(std::shared_ptr<const std::vector<u8>> data) : data(data) {
-}
-
 std::string IVFCArchive::GetName() const {
     return "IVFC";
 }
 
 std::unique_ptr<FileBackend> IVFCArchive::OpenFile(const Path& path, const Mode mode) const {
-    return Common::make_unique<IVFCFile>(data);
+    return Common::make_unique<IVFCFile>(romfs_file, data_offset, data_size);
 }
 
 bool IVFCArchive::DeleteFile(const Path& path) const {
@@ -64,19 +61,21 @@ std::unique_ptr<DirectoryBackend> IVFCArchive::OpenDirectory(const Path& path) c
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
-size_t IVFCFile::Read(const u64 offset, const u32 length, u8* buffer) const {
+size_t IVFCFile::Read(const u64 offset, const size_t length, u8* buffer) const {
     LOG_TRACE(Service_FS, "called offset=%llu, length=%d", offset, length);
-    memcpy(buffer, data->data() + offset, length);
-    return length;
+    romfs_file->Seek(data_offset + offset, SEEK_SET);
+    size_t read_length = (size_t)std::min((u64)length, data_size - offset);
+
+    return romfs_file->ReadBytes(buffer, read_length);
 }
 
-size_t IVFCFile::Write(const u64 offset, const u32 length, const u32 flush, const u8* buffer) const {
+size_t IVFCFile::Write(const u64 offset, const size_t length, const bool flush, const u8* buffer) const {
     LOG_ERROR(Service_FS, "Attempted to write to IVFC file");
     return 0;
 }
 
-size_t IVFCFile::GetSize() const {
-    return sizeof(u8) * data->size();
+u64 IVFCFile::GetSize() const {
+    return data_size;
 }
 
 bool IVFCFile::SetSize(const u64 size) const {

src/core/file_sys/ivfc_archive.h

@@ -10,6 +10,7 @@
 #include <vector>
 
 #include "common/common_types.h"
+#include "common/file_util.h"
 
 #include "core/file_sys/archive_backend.h"
 #include "core/file_sys/directory_backend.h"
@@ -28,7 +29,8 @@ namespace FileSys {
  */
 class IVFCArchive : public ArchiveBackend {
 public:
-    IVFCArchive(std::shared_ptr<const std::vector<u8>> data);
+    IVFCArchive(std::shared_ptr<FileUtil::IOFile> file, u64 offset, u64 size)
+        : romfs_file(file), data_offset(offset), data_size(size) {}
 
     std::string GetName() const override;
 
@@ -42,23 +44,28 @@ class IVFCArchive : public ArchiveBackend {
     std::unique_ptr<DirectoryBackend> OpenDirectory(const Path& path) const override;
 
 protected:
-    std::shared_ptr<const std::vector<u8>> data;
+    std::shared_ptr<FileUtil::IOFile> romfs_file;
+    u64 data_offset;
+    u64 data_size;
 };
 
 class IVFCFile : public FileBackend {
 public:
-    IVFCFile(std::shared_ptr<const std::vector<u8>> data) : data(data) {}
+    IVFCFile(std::shared_ptr<FileUtil::IOFile> file, u64 offset, u64 size)
+        : romfs_file(file), data_offset(offset), data_size(size) {}
 
     bool Open() override { return true; }
-    size_t Read(const u64 offset, const u32 length, u8* buffer) const override;
-    size_t Write(const u64 offset, const u32 length, const u32 flush, const u8* buffer) const override;
-    size_t GetSize() const override;
-    bool SetSize(const u64 size) const override;
+    size_t Read(u64 offset, size_t length, u8* buffer) const override;
+    size_t Write(u64 offset, size_t length, bool flush, const u8* buffer) const override;
+    u64 GetSize() const override;
+    bool SetSize(u64 size) const override;
     bool Close() const override { return false; }
     void Flush() const override { }
 
 private:
-    std::shared_ptr<const std::vector<u8>> data;
+    std::shared_ptr<FileUtil::IOFile> romfs_file;
+    u64 data_offset;
+    u64 data_size;
 };
 
 class IVFCDirectory : public DirectoryBackend {

src/core/hle/service/fs/archive.cpp

@@ -116,7 +116,7 @@ ResultVal<bool> File::SyncRequest() {
             u32 address = cmd_buff[6];
             LOG_TRACE(Service_FS, "Write %s %s: offset=0x%llx length=%d address=0x%x, flush=0x%x",
                       GetTypeName().c_str(), GetName().c_str(), offset, length, address, flush);
-            cmd_buff[2] = static_cast<u32>(backend->Write(offset, length, flush, Memory::GetPointer(address)));
+            cmd_buff[2] = static_cast<u32>(backend->Write(offset, length, flush != 0, Memory::GetPointer(address)));
             break;
         }
 

src/core/loader/3dsx.cpp

@@ -246,11 +246,11 @@ ResultStatus AppLoader_THREEDSX::Load() {
     if (is_loaded)
         return ResultStatus::ErrorAlreadyLoaded;
 
-    if (!file->IsOpen())
+    if (!file.IsOpen())
         return ResultStatus::Error;
 
     SharedPtr<CodeSet> codeset;
-    if (Load3DSXFile(*file, Memory::PROCESS_IMAGE_VADDR, &codeset) != ERROR_NONE)
+    if (Load3DSXFile(file, Memory::PROCESS_IMAGE_VADDR, &codeset) != ERROR_NONE)
         return ResultStatus::Error;
     codeset->name = filename;
 

src/core/loader/3dsx.h

@@ -17,7 +17,7 @@ namespace Loader {
 /// Loads an 3DSX file
 class AppLoader_THREEDSX final : public AppLoader {
 public:
-    AppLoader_THREEDSX(std::unique_ptr<FileUtil::IOFile>&& file, std::string filename)
+    AppLoader_THREEDSX(FileUtil::IOFile&& file, std::string filename)
         : AppLoader(std::move(file)), filename(std::move(filename)) {}
 
     /**

src/core/loader/elf.cpp

@@ -273,7 +273,6 @@ SharedPtr<CodeSet> ElfReader::LoadInto(u32 vaddr) {
     LOG_DEBUG(Loader, "%i segments:", header->e_phnum);
 
     // First pass : Get the bits into RAM
-    u32 segment_addr[32];
     u32 base_addr = relocate ? vaddr : 0;
 
     u32 total_image_size = 0;
@@ -392,15 +391,15 @@ ResultStatus AppLoader_ELF::Load() {
     if (is_loaded)
         return ResultStatus::ErrorAlreadyLoaded;
 
-    if (!file->IsOpen())
+    if (!file.IsOpen())
         return ResultStatus::Error;
 
     // Reset read pointer in case this file has been read before.
-    file->Seek(0, SEEK_SET);
+    file.Seek(0, SEEK_SET);
 
-    u32 size = static_cast<u32>(file->GetSize());
+    size_t size = file.GetSize();
     std::unique_ptr<u8[]> buffer(new u8[size]);
-    if (file->ReadBytes(&buffer[0], size) != size)
+    if (file.ReadBytes(&buffer[0], size) != size)
         return ResultStatus::Error;
 
     ElfReader elf_reader(&buffer[0]);

src/core/loader/elf.h

@@ -17,7 +17,7 @@ namespace Loader {
 /// Loads an ELF/AXF file
 class AppLoader_ELF final : public AppLoader {
 public:
-    AppLoader_ELF(std::unique_ptr<FileUtil::IOFile>&& file, std::string filename)
+    AppLoader_ELF(FileUtil::IOFile&& file, std::string filename)
         : AppLoader(std::move(file)), filename(std::move(filename)) { }
 
     /**

src/core/loader/loader.cpp

@@ -90,16 +90,16 @@ static const char* GetFileTypeString(FileType type) {
 }
 
 ResultStatus LoadFile(const std::string& filename) {
-    std::unique_ptr<FileUtil::IOFile> file(new FileUtil::IOFile(filename, "rb"));
-    if (!file->IsOpen()) {
+    FileUtil::IOFile file(filename, "rb");
+    if (!file.IsOpen()) {
         LOG_ERROR(Loader, "Failed to load file %s", filename.c_str());
         return ResultStatus::Error;
     }
 
     std::string filename_filename, filename_extension;
     Common::SplitPath(filename, nullptr, &filename_filename, &filename_extension);
 
-    FileType type = IdentifyFile(*file);
+    FileType type = IdentifyFile(file);
     FileType filename_type = GuessFromExtension(filename_extension);
 
     if (type != filename_type) {
@@ -124,7 +124,7 @@ ResultStatus LoadFile(const std::string& filename) {
     case FileType::CXI:
     case FileType::CCI:
     {
-        AppLoader_NCCH app_loader(std::move(file));
+        AppLoader_NCCH app_loader(std::move(file), filename);
 
         // Load application and RomFS
         if (ResultStatus::Success == app_loader.Load()) {

src/core/loader/loader.h

@@ -52,7 +52,7 @@ static inline u32 MakeMagic(char a, char b, char c, char d) {
 /// Interface for loading an application
 class AppLoader : NonCopyable {
 public:
-    AppLoader(std::unique_ptr<FileUtil::IOFile>&& file) : file(std::move(file)) { }
+    AppLoader(FileUtil::IOFile&& file) : file(std::move(file)) { }
     virtual ~AppLoader() { }
 
     /**
@@ -66,7 +66,7 @@ class AppLoader : NonCopyable {
      * @param buffer Reference to buffer to store data
      * @return ResultStatus result of function
      */
-    virtual ResultStatus ReadCode(std::vector<u8>& buffer) const {
+    virtual ResultStatus ReadCode(std::vector<u8>& buffer) {
         return ResultStatus::ErrorNotImplemented;
     }
 
@@ -75,7 +75,7 @@ class AppLoader : NonCopyable {
      * @param buffer Reference to buffer to store data
      * @return ResultStatus result of function
      */
-    virtual ResultStatus ReadIcon(std::vector<u8>& buffer) const {
+    virtual ResultStatus ReadIcon(std::vector<u8>& buffer) {
         return ResultStatus::ErrorNotImplemented;
     }
 
@@ -84,7 +84,7 @@ class AppLoader : NonCopyable {
      * @param buffer Reference to buffer to store data
      * @return ResultStatus result of function
      */
-    virtual ResultStatus ReadBanner(std::vector<u8>& buffer) const {
+    virtual ResultStatus ReadBanner(std::vector<u8>& buffer) {
         return ResultStatus::ErrorNotImplemented;
     }
 
@@ -93,22 +93,25 @@ class AppLoader : NonCopyable {
      * @param buffer Reference to buffer to store data
      * @return ResultStatus result of function
      */
-    virtual ResultStatus ReadLogo(std::vector<u8>& buffer) const {
+    virtual ResultStatus ReadLogo(std::vector<u8>& buffer) {
         return ResultStatus::ErrorNotImplemented;
     }
 
     /**
      * Get the RomFS of the application
-     * @param buffer Reference to buffer to store data
+     * Since the RomFS can be huge, we return a file reference instead of copying to a buffer
+     * @param romfs_file The file containing the RomFS
+     * @param offset The offset the romfs begins on
+     * @param size The size of the romfs
      * @return ResultStatus result of function
      */
-    virtual ResultStatus ReadRomFS(std::vector<u8>& buffer) const {
+    virtual ResultStatus ReadRomFS(std::shared_ptr<FileUtil::IOFile>& romfs_file, u64& offset, u64& size) {
         return ResultStatus::ErrorNotImplemented;
     }
 
 protected:
-    std::unique_ptr<FileUtil::IOFile> file;
-    bool                              is_loaded = false;
+    FileUtil::IOFile file;
+    bool is_loaded = false;
 };
 
 /**