feat: generate asset lists for linked zones

src/Linking/Linker.cpp

@@ -8,6 +8,7 @@
 #include "SearchPath/SearchPaths.h"
 #include "Utils/Logging/Log.h"
 #include "Zone/AssetList/AssetList.h"
+#include "Zone/AssetList/AssetListOutputStream.h"
 #include "Zone/AssetList/AssetListReader.h"
 #include "Zone/Definition/ZoneDefinitionStream.h"
 #include "ZoneCreation/ZoneCreationContext.h"
@@ -336,6 +337,36 @@ namespace
             return true;
         }
 
+        static bool WriteAssetList(IOutputPath& outPath, const fs::path& outDir, const Zone& zone)
+        {
+            const auto assetListPath = fs::path("assetlist") / std::format("{}.csv", zone.m_name);
+            const auto stream = outPath.Open(assetListPath.string());
+            if (!stream)
+            {
+                con::error("Failed to open assetlist for zone: {}", zone.m_name);
+                return false;
+            }
+
+            AssetListOutputStream assetListStream(*stream, zone.m_game_id);
+            for (const auto* asset : zone.m_pools)
+            {
+                if (asset->IsReference())
+                    assetListStream.WriteEntry(AssetListEntry(asset->m_type, asset->ReferencedAssetName(), true));
+                else
+                    assetListStream.WriteEntry(AssetListEntry(asset->m_type, asset->m_name, false));
+            }
+
+            stream->flush();
+            if (!*stream)
+            {
+                con::error("Writing assetlist for zone \"{}\" failed.", zone.m_name);
+                return false;
+            }
+
+            con::info("Created assetlist \"{}\"", (outDir / assetListPath).string());
+            return true;
+        }
+
         bool BuildFastFile(LinkerPathManager& paths, const std::string& projectName, const std::string& targetName, ZoneDefinition& zoneDefinition) const
         {
             const fs::path outDir(paths.m_linker_paths->BuildOutputFolderPath(projectName, zoneDefinition.m_game));
@@ -348,7 +379,11 @@ namespace
             const auto zone = CreateZoneForDefinition(paths, outDir, cacheDir, targetName, zoneDefinition);
             auto result = zone != nullptr;
             if (zone)
+            {
                 result = WriteZoneToFile(outputPath, *zone);
+                if (result && m_args.m_generate_asset_lists)
+                    result = WriteAssetList(outputPath, outDir, *zone);
+            }
 
             return result;
         }

src/Linking/LinkerArgs.cpp

@@ -57,6 +57,12 @@ const CommandLineOption* const OPTION_OUTPUT_FOLDER =
     .WithParameter("outputFolderPath")
     .Build();
 
+const CommandLineOption* const OPTION_NO_ASSET_LIST =
+    CommandLineOption::Builder::Create()
+    .WithLongName("no-assetlist")
+    .WithDescription("Disables generating an assetlist after successfully linking a zone.")
+    .Build();
+
 const CommandLineOption* const OPTION_ADD_ASSET_SEARCH_PATH =
     CommandLineOption::Builder::Create()
     .WithLongName("add-asset-search-path")
@@ -125,6 +131,7 @@ const CommandLineOption* const COMMAND_LINE_OPTIONS[]{
     OPTION_NO_COLOR,
     OPTION_BASE_FOLDER,
     OPTION_OUTPUT_FOLDER,
+    OPTION_NO_ASSET_LIST,
     OPTION_ADD_ASSET_SEARCH_PATH,
     OPTION_ASSET_SEARCH_PATH,
     OPTION_GDT_SEARCH_PATH,
@@ -136,7 +143,8 @@ const CommandLineOption* const COMMAND_LINE_OPTIONS[]{
 };
 
 LinkerArgs::LinkerArgs()
-    : m_argument_parser(COMMAND_LINE_OPTIONS, std::extent_v<decltype(COMMAND_LINE_OPTIONS)>)
+    : m_generate_asset_lists(true),
+      m_argument_parser(COMMAND_LINE_OPTIONS, std::extent_v<decltype(COMMAND_LINE_OPTIONS)>)
 {
 }
 
@@ -222,6 +230,9 @@ bool LinkerArgs::ParseArgs(const int argc, const char** argv, bool& shouldContin
     else
         m_out_folder = DEFAULT_OUTPUT_FOLDER;
 
+    // --no-assetlist
+    m_generate_asset_lists = !m_argument_parser.IsOptionSpecified(OPTION_NO_ASSET_LIST);
+
     // --asset-search-path
     if (m_argument_parser.IsOptionSpecified(OPTION_ASSET_SEARCH_PATH))
     {

src/Linking/LinkerArgs.h

@@ -23,6 +23,7 @@ class LinkerArgs
     std::string m_bin_folder;
     std::string m_base_folder;
     std::string m_out_folder;
+    bool m_generate_asset_lists;
 
     std::set<std::string> m_asset_search_paths;
     std::set<std::string> m_gdt_search_paths;

src/ObjCommon/Csv/CsvStream.cpp

@@ -217,11 +217,12 @@ namespace
     }
 } // namespace
 
-CsvOutputStream::CsvOutputStream(std::ostream& stream)
+CsvOutputStream::CsvOutputStream(std::ostream& stream, const bool padColumns)
     : m_stream(stream),
       m_column_count(0),
       m_current_column(0),
-      m_first_row(true)
+      m_first_row(true),
+      m_pad_columns(padColumns)
 {
 }
 
@@ -265,7 +266,7 @@ void CsvOutputStream::NextRow()
         m_first_row = false;
         m_column_count = m_current_column;
     }
-    else
+    else if (m_pad_columns)
     {
         while (m_current_column < m_column_count)
         {

src/ObjCommon/Csv/CsvStream.h

@@ -37,7 +37,7 @@ class CsvInputStream
 class CsvOutputStream
 {
 public:
-    explicit CsvOutputStream(std::ostream& stream);
+    explicit CsvOutputStream(std::ostream& stream, bool padColumns = true);
 
     void WriteColumn(const std::string& value);
     void NextRow();
@@ -47,4 +47,5 @@ class CsvOutputStream
     unsigned m_column_count;
     unsigned m_current_column;
     bool m_first_row;
+    bool m_pad_columns;
 };

src/ZoneCommon/Zone/AssetList/AssetListOutputStream.cpp

@@ -1,14 +1,16 @@
 #include "AssetListOutputStream.h"
 
 AssetListOutputStream::AssetListOutputStream(std::ostream& stream, const GameId game)
-    : m_stream(stream),
+    : m_stream(stream, false),
       m_game(IGame::GetGameById(game))
 {
 }
 
 void AssetListOutputStream::WriteEntry(const AssetListEntry& entry)
 {
     m_stream.WriteColumn(*m_game->GetAssetTypeName(entry.m_type));
+    if (entry.m_is_reference)
+        m_stream.WriteColumn("");
     m_stream.WriteColumn(entry.m_name);
     m_stream.NextRow();
 }

test/ObjCommonTests/Csv/CsvStreamTests.cpp

@@ -84,6 +84,22 @@ namespace
         }
     }
 
+    TEST_CASE("CsvOutputStream can write rows with different column counts", "[csv]")
+    {
+        std::ostringstream ss;
+        CsvOutputStream outputStream(ss, false);
+
+        outputStream.WriteColumn("one");
+        outputStream.WriteColumn("two");
+        outputStream.WriteColumn("three");
+        outputStream.NextRow();
+        outputStream.WriteColumn("foo");
+        outputStream.WriteColumn("bar");
+        outputStream.NextRow();
+
+        REQUIRE(ss.str() == ("one,two,three" NEW_LINE "foo,bar" NEW_LINE));
+    }
+
     TEST_CASE("CsvInputStream", "[csv]")
     {
         SECTION("Ensure can write normal single-line csv")

test/SystemTests/Game/IW4/AssetList/SimpleZoneIW4.zone

@@ -0,0 +1,4 @@
+>game,IW4
+
+material,,white
+rawfile,SimpleZone.txt

test/SystemTests/Game/IW4/AssetListIgnore/IgnoreAssetListIW4.zone

@@ -0,0 +1,7 @@
+>game,IW4
+
+ignore,SimpleZoneIW4
+
+material,white
+rawfile,SimpleZone.txt
+rawfile,NotIgnored.txt

test/SystemTests/Game/IW4/Simple/NotIgnored.txt

@@ -0,0 +1 @@
+This asset is not ignored.

test/SystemTests/Game/IW4/SimpleZoneIW4.cpp

@@ -7,6 +7,8 @@
 #include <catch2/catch_test_macros.hpp>
 #include <filesystem>
 #include <format>
+#include <fstream>
+#include <iterator>
 #include <memory>
 #include <string>
 
@@ -58,4 +60,148 @@ namespace
         REQUIRE(zone->m_pools.GetTotalAssetCount() == 1);
         REQUIRE(zone->m_pools.GetAsset<IW4::AssetRawFile>("SimpleZone.txt"));
     }
+
+    TEST_CASE("Linker generates an assetlist(IW4)", "[iw4][system][assetlist]")
+    {
+        const auto assetSearchPath = (oat::paths::GetSystemTestsDirectory() / "Game/IW4/Simple").string();
+        const auto sourceSearchPath = (oat::paths::GetSystemTestsDirectory() / "Game/IW4/AssetList").string();
+        const auto outputPath = oat::paths::GetTempDirectory("SimpleZoneIW4AssetList").string();
+        fs::create_directories(fs::path(outputPath) / ".oat/cache/SimpleZoneIW4");
+
+        const char* argStrings[]{
+            "SystemTests", // bin
+            "--asset-search-path",
+            assetSearchPath.c_str(),
+            "--source-search-path",
+            sourceSearchPath.c_str(),
+            "--base-folder",
+            outputPath.c_str(),
+            "--output-folder",
+            outputPath.c_str(),
+            "SimpleZoneIW4",
+        };
+
+        LinkerArgs args;
+
+        bool shouldContinue = true;
+        const auto couldParseArgs = args.ParseArgs(std::extent_v<decltype(argStrings)>, argStrings, shouldContinue);
+
+        REQUIRE(couldParseArgs);
+        REQUIRE(shouldContinue);
+
+        const auto linker = Linker::Create(std::move(args));
+        REQUIRE(linker->Start());
+
+        const auto expectedAssetListPath = fs::path(outputPath) / "assetlist/SimpleZoneIW4.csv";
+        std::ifstream assetListStream(expectedAssetListPath, std::ios::binary);
+        REQUIRE(assetListStream.is_open());
+
+        const std::string assetListContent(std::istreambuf_iterator<char>(assetListStream), {});
+        REQUIRE(assetListContent == "material,,white\nrawfile,SimpleZone.txt\n");
+    }
+
+    TEST_CASE("Linker can omit the assetlist(IW4)", "[iw4][system][assetlist]")
+    {
+        const auto assetSearchPath = (oat::paths::GetSystemTestsDirectory() / "Game/IW4/Simple").string();
+        const auto sourceSearchPath = (oat::paths::GetSystemTestsDirectory() / "Game/IW4/Simple").string();
+        const auto outputPath = oat::paths::GetTempDirectory("SimpleZoneIW4NoAssetList").string();
+        fs::create_directories(fs::path(outputPath) / ".oat/cache/SimpleZoneIW4");
+
+        const char* argStrings[]{
+            "SystemTests", // bin
+            "--asset-search-path",
+            assetSearchPath.c_str(),
+            "--source-search-path",
+            sourceSearchPath.c_str(),
+            "--base-folder",
+            outputPath.c_str(),
+            "--output-folder",
+            outputPath.c_str(),
+            "--no-assetlist",
+            "SimpleZoneIW4",
+        };
+
+        LinkerArgs args;
+
+        bool shouldContinue = true;
+        const auto couldParseArgs = args.ParseArgs(std::extent_v<decltype(argStrings)>, argStrings, shouldContinue);
+
+        REQUIRE(couldParseArgs);
+        REQUIRE(shouldContinue);
+
+        const auto linker = Linker::Create(std::move(args));
+        REQUIRE(linker->Start());
+
+        const auto expectedAssetListPath = fs::path(outputPath) / "assetlist/SimpleZoneIW4.csv";
+        REQUIRE_FALSE(fs::exists(expectedAssetListPath));
+    }
+
+    TEST_CASE("Linker can use a generated assetlist as ignore(IW4)", "[iw4][system][assetlist]")
+    {
+        const auto assetSearchPath = (oat::paths::GetSystemTestsDirectory() / "Game/IW4/Simple").string();
+        const auto assetListSourcePath = (oat::paths::GetSystemTestsDirectory() / "Game/IW4/AssetList").string();
+        const auto ignoreSourcePath = (oat::paths::GetSystemTestsDirectory() / "Game/IW4/AssetListIgnore").string();
+        const auto outputPath = oat::paths::GetTempDirectory("GeneratedAssetListIgnoreIW4").string();
+        fs::create_directories(fs::path(outputPath) / ".oat/cache/SimpleZoneIW4");
+        fs::create_directories(fs::path(outputPath) / ".oat/cache/IgnoreAssetListIW4");
+
+        const char* generateArgStrings[]{
+            "SystemTests", // bin
+            "--asset-search-path",
+            assetSearchPath.c_str(),
+            "--source-search-path",
+            assetListSourcePath.c_str(),
+            "--base-folder",
+            outputPath.c_str(),
+            "--output-folder",
+            outputPath.c_str(),
+            "SimpleZoneIW4",
+        };
+
+        LinkerArgs generateArgs;
+        bool shouldContinue = true;
+        REQUIRE(generateArgs.ParseArgs(std::extent_v<decltype(generateArgStrings)>, generateArgStrings, shouldContinue));
+        REQUIRE(shouldContinue);
+        REQUIRE(Linker::Create(std::move(generateArgs))->Start());
+
+        const char* ignoreArgStrings[]{
+            "SystemTests", // bin
+            "--asset-search-path",
+            assetSearchPath.c_str(),
+            "--source-search-path",
+            outputPath.c_str(),
+            "--add-source-search-path",
+            ignoreSourcePath.c_str(),
+            "--base-folder",
+            outputPath.c_str(),
+            "--output-folder",
+            outputPath.c_str(),
+            "IgnoreAssetListIW4",
+        };
+
+        LinkerArgs ignoreArgs;
+        shouldContinue = true;
+        REQUIRE(ignoreArgs.ParseArgs(std::extent_v<decltype(ignoreArgStrings)>, ignoreArgStrings, shouldContinue));
+        REQUIRE(shouldContinue);
+        REQUIRE(Linker::Create(std::move(ignoreArgs))->Start());
+
+        const auto expectedZonePath = (fs::path(outputPath) / "IgnoreAssetListIW4.ff").string();
+        auto maybeZone = ZoneLoading::LoadZone(expectedZonePath, std::nullopt);
+        REQUIRE(maybeZone);
+
+        auto zone = std::move(*maybeZone);
+        REQUIRE(zone->m_pools.GetTotalAssetCount() == 3);
+
+        const auto* material = zone->m_pools.GetAssetOrAssetReference<IW4::AssetMaterial>("white");
+        REQUIRE(material);
+        REQUIRE(material->IsReference());
+
+        const auto* rawFile = zone->m_pools.GetAssetOrAssetReference<IW4::AssetRawFile>("SimpleZone.txt");
+        REQUIRE(rawFile);
+        REQUIRE(rawFile->IsReference());
+
+        const auto* notIgnoredRawFile = zone->m_pools.GetAsset<IW4::AssetRawFile>("NotIgnored.txt");
+        REQUIRE(notIgnoredRawFile);
+        REQUIRE_FALSE(notIgnoredRawFile->IsReference());
+    }
 } // namespace