[OTR Archive] Store soh version in OTR files and verify on launch/ask to regenerate

CMake/Packaging-2.cmake

@@ -24,7 +24,6 @@ if (CPACK_GENERATOR MATCHES "Bundle")
     set(CPACK_BUNDLE_NAME "soh")
     set(CPACK_BUNDLE_PLIST "macosx/Info.plist")
     set(CPACK_BUNDLE_ICON "macosx/soh.icns")
-    set(CPACK_BUNDLE_STARTUP_COMMAND "../soh/macosx/soh-macos.sh")
+    set(CPACK_BUNDLE_STARTUP_COMMAND "macosx/soh-macos.sh")
     set(CPACK_BUNDLE_APPLE_CERT_APP "-")
 endif()
-

CMakeLists.txt

@@ -97,7 +97,7 @@ set_property(TARGET soh PROPERTY APPIMAGE_DESKTOP_FILE "${CMAKE_SOURCE_DIR}/scri
 set_property(TARGET soh PROPERTY APPIMAGE_ICON_FILE "${CMAKE_BINARY_DIR}/sohIcon.png")
 
 if("${CMAKE_SYSTEM_NAME}" STREQUAL "Linux")
-install(PROGRAMS "${CMAKE_SOURCE_DIR}/scripts/linux/appimage/soh.sh" DESTINATION . COMPONENT appimage)
+install(PROGRAMS "${CMAKE_BINARY_DIR}/linux/soh.sh" DESTINATION . COMPONENT appimage)
 install(FILES "${CMAKE_SOURCE_DIR}/soh.otr" DESTINATION . COMPONENT ship)
 install(TARGETS ZAPD DESTINATION ./assets/extractor COMPONENT extractor)
 install(DIRECTORY "${CMAKE_SOURCE_DIR}/soh/assets/extractor/" DESTINATION ./assets/extractor COMPONENT extractor)
@@ -115,7 +115,7 @@ add_custom_target(
     ExtractAssets
     # CMake versions prior to 3.17 do not have the rm command, use remove instead for older versions
     COMMAND ${CMAKE_COMMAND} -E $<IF:$<VERSION_LESS:${CMAKE_VERSION},3.17>,remove,rm> -f oot.otr oot-mq.otr soh.otr
-    COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/OTRExporter/extract_assets.py -z "$<TARGET_FILE:ZAPD>" --non-interactive
+    COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/OTRExporter/extract_assets.py -z "$<TARGET_FILE:ZAPD>" --non-interactive --sohver "${CMAKE_PROJECT_VERSION}"
     COMMAND ${CMAKE_COMMAND} -DSYSTEM_NAME=${CMAKE_SYSTEM_NAME} -DTARGET_DIR="$<TARGET_FILE_DIR:ZAPD>" -DSOURCE_DIR=${CMAKE_CURRENT_SOURCE_DIR} -DBINARY_DIR=${CMAKE_BINARY_DIR} -P ${CMAKE_CURRENT_SOURCE_DIR}/copy-existing-otrs.cmake
     WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/OTRExporter
     COMMENT "Running asset extraction..."
@@ -137,7 +137,7 @@ add_custom_target(
     GenerateSohOtr
     # CMake versions prior to 3.17 do not have the rm command, use remove instead for older versions
     COMMAND ${CMAKE_COMMAND} -E $<IF:$<VERSION_LESS:${CMAKE_VERSION},3.17>,remove,rm> -f soh.otr
-    COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/OTRExporter/extract_assets.py -z "$<TARGET_FILE:ZAPD>" --norom
+    COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/OTRExporter/extract_assets.py -z "$<TARGET_FILE:ZAPD>" --norom --sohver "${CMAKE_PROJECT_VERSION}"
     COMMAND ${CMAKE_COMMAND} -DSYSTEM_NAME=${CMAKE_SYSTEM_NAME} -DTARGET_DIR="$<TARGET_FILE_DIR:ZAPD>" -DSOURCE_DIR=${CMAKE_CURRENT_SOURCE_DIR} -DBINARY_DIR=${CMAKE_BINARY_DIR} -DONLYSOHOTR=On -P ${CMAKE_CURRENT_SOURCE_DIR}/copy-existing-otrs.cmake
     WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/OTRExporter
     COMMENT "Generating soh.otr..."
@@ -210,4 +210,4 @@ elseif(CMAKE_SYSTEM_NAME MATCHES "Darwin")
 endif()
 
 set(CPACK_PROJECT_CONFIG_FILE ${CMAKE_SOURCE_DIR}/CMake/Packaging-2.cmake)
-include(CMake/Packaging.cmake)
+include(CMake/Packaging.cmake)

OTRExporter/OTRExporter/Main.cpp

@@ -27,6 +27,7 @@
 #include <mutex>
 
 std::string otrFileName = "oot.otr";
+std::string sohVersionString = "0.0.0";
 std::shared_ptr<LUS::Archive> otrArchive;
 BinaryWriter* fileWriter;
 std::chrono::steady_clock::time_point fileStart, resStart;
@@ -67,6 +68,47 @@ static void ExporterProgramEnd()
 {
 	uint32_t crc = 0xFFFFFFFF;
 	const uint8_t endianness = (uint8_t)Endianness::Big;
+
+	std::vector<int16_t> sohVersion = {};
+	std::vector<std::string> versionParts = StringHelper::Split(sohVersionString, ".");
+
+	for (auto val : versionParts)
+	{
+		int16_t num = 0;
+		try
+		{
+			num = std::stoi(val, nullptr);
+		}
+		catch (std::invalid_argument &e)
+		{
+			num = 0;
+		}
+		catch (std::out_of_range &e)
+		{
+			num = 0;
+		}
+
+		sohVersion.push_back(num);
+
+		// Ignore any extra values passed in
+		if (sohVersion.size() == 3)
+			break;
+	}
+
+	// Padded out missing values
+	while (sohVersion.size() < 3)
+	{
+		sohVersion.push_back(0);
+	}
+
+	MemoryStream *sohVersionStream = new MemoryStream();
+	BinaryWriter sohWriter(sohVersionStream);
+	sohWriter.SetEndianness(Endianness::Big);
+	sohWriter.Write(endianness);
+	sohWriter.Write(sohVersion[0]); // Major
+	sohWriter.Write(sohVersion[1]); // Minor
+	sohWriter.Write(sohVersion[2]); // Patch
+	sohWriter.Close();
 
 	if (Globals::Instance->fileMode == ZFileMode::ExtractDirectory)
 	{
@@ -92,8 +134,12 @@ static void ExporterProgramEnd()
 		printf("Generating OTR Archive...\n");
 		otrArchive = LUS::Archive::CreateArchive(otrFileName, 40000);
 
+		printf("Adding version file.\n");
 		otrArchive->AddFile("version", (uintptr_t)versionStream->ToVector().data(), versionStream->GetLength());
 
+		printf("Adding sohVersion file.\n");
+		otrArchive->AddFile("sohVersion", (uintptr_t)sohVersionStream->ToVector().data(), sohVersionStream->GetLength());
+
 		for (const auto& item : files)
 		{
 			std::string fName = item.first;
@@ -110,6 +156,7 @@ static void ExporterProgramEnd()
 								fileData.size());
 		}
 	}
+
 	otrArchive = nullptr;
 	delete fileWriter;
 	files.clear();
@@ -118,9 +165,25 @@ static void ExporterProgramEnd()
 	if (DiskFile::Exists("soh.otr")) {
 		return;
 	}
+
 	const auto& lst = Directory::ListFiles("Extract");
+
+	printf("Generating SoH OTR Archive...\n");
 	std::shared_ptr<LUS::Archive> sohOtr = LUS::Archive::CreateArchive("soh.otr", 4096);
-	//sohOtr->AddFile("version", (uintptr_t)versionStream->ToVector().data(), versionStream->GetLength());
+
+	// OTRTODO: Remove this dummy version after https://github.com/Kenix3/libultraship/pull/345 is merged
+	MemoryStream *dummyVersionStream = new MemoryStream();
+	BinaryWriter dummyWriter(dummyVersionStream);
+	dummyWriter.SetEndianness(Endianness::Big);
+	dummyWriter.Write(endianness);
+	dummyWriter.Write((uint32_t)0); // dummy version to satisfy LUS
+	dummyWriter.Close();
+
+	printf("Adding version file.\n");
+	sohOtr->AddFile("version", (uintptr_t)dummyVersionStream->ToVector().data(), dummyVersionStream->GetLength());
+
+	printf("Adding sohVersion file.\n");
+	sohOtr->AddFile("sohVersion", (uintptr_t)sohVersionStream->ToVector().data(), sohVersionStream->GetLength());
 
 	for (const auto& item : lst)
 	{
@@ -185,6 +248,11 @@ static void ExporterParseArgs(int argc, char* argv[], int& i)
 		otrFileName = argv[i + 1];
 		i++;
 	}
+	else if (arg == "--sohver")
+	{
+		sohVersionString = argv[i + 1];
+		i++;
+	}
 }
 
 static bool ExporterProcessFileMode(ZFileMode fileMode)

OTRExporter/extract_assets.py

@@ -8,7 +8,7 @@
 import subprocess
 import argparse
 
-def BuildOTR(xmlPath, rom, zapd_exe=None, genHeaders=None):
+def BuildOTR(xmlPath, rom, zapd_exe=None, sohver=None, genHeaders=None):
     shutil.copytree("assets", "Extract/assets")
 
     if not zapd_exe:
@@ -23,7 +23,7 @@ def BuildOTR(xmlPath, rom, zapd_exe=None, genHeaders=None):
     else:
         # generate otrs, but not headers
         exec_cmd.extend(["-gsf", "0", "-se", "OTR", "--otrfile",
-                "oot-mq.otr" if Z64Rom.isMqRom(rom) else "oot.otr"])
+                "oot-mq.otr" if Z64Rom.isMqRom(rom) else "oot.otr"] + ["--sohver", sohver] if sohver else [])
 
     print(exec_cmd)
     exitValue = subprocess.call(exec_cmd)
@@ -33,13 +33,13 @@ def BuildOTR(xmlPath, rom, zapd_exe=None, genHeaders=None):
         print("Aborting...", file=os.sys.stderr)
         print("\n")
 
-def BuildSohOtr(zapd_exe=None):
+def BuildSohOtr(zapd_exe=None, sohver=None):
     shutil.copytree("assets", "Extract/assets")
 
     if not zapd_exe:
         zapd_exe = "x64\\Release\\ZAPD.exe" if sys.platform == "win32" else "../ZAPDTR/ZAPD.out"
 
-    exec_cmd = [zapd_exe, "botr", "-se", "OTR", "--norom"]
+    exec_cmd = [zapd_exe, "botr", "-se", "OTR", "--norom"] + ["--sohver", sohver] if sohver else []
 
     print(exec_cmd)
     exitValue = subprocess.call(exec_cmd)
@@ -57,22 +57,23 @@ def main():
     parser.add_argument("-v", "--verbose", help="Display rom's header checksums and their corresponding xml folder", dest="verbose", action="store_true")
     parser.add_argument("--gen-headers", help="Generate source headers to be checked in", dest="gen_headers", action="store_true")
     parser.add_argument("--norom", help="Generate only soh.otr to be bundled to the game", dest="norom", action="store_true")
+    parser.add_argument("--sohver", help="Store the SoH version in the otr", dest="sohver", type=str)
 
     args = parser.parse_args()
 
     if args.norom:
         if (os.path.exists("Extract")):
             shutil.rmtree("Extract")
 
-        BuildSohOtr(args.zapd_exe)
+        BuildSohOtr(args.zapd_exe, sohver=args.sohver)
         return
 
     roms = [ Z64Rom(args.rom) ] if args.rom else rom_chooser.chooseROM(args.verbose, args.non_interactive)
     for rom in roms:
         if (os.path.exists("Extract")):
             shutil.rmtree("Extract")
 
-        BuildOTR("../soh/assets/xml/" + rom.version.xml_ver + "/", rom.file_path, zapd_exe=args.zapd_exe, genHeaders=args.gen_headers)
+        BuildOTR("../soh/assets/xml/" + rom.version.xml_ver + "/", rom.file_path, zapd_exe=args.zapd_exe, sohver=args.sohver, genHeaders=args.gen_headers)
 
 if __name__ == "__main__":
     main()

ZAPDTR/ZAPD/Main.cpp

@@ -226,17 +226,29 @@ extern "C" int zapd_main(int argc, char* argv[])
 		}
 	}
 
-
 	// Parse File Mode
 	ExporterSet* exporterSet = Globals::Instance->GetExporterSet();
-
-	if(Globals::Instance->onlyGenSohOtr) {
-		exporterSet->endProgramFunc();
+
+	if (Globals::Instance->onlyGenSohOtr)
+	{
+		if (exporterSet != nullptr)
+		{
+			// parse args for exporter
+			if (exporterSet->parseArgsFunc != nullptr)
+			{
+				for (int32_t i = 2; i < argc; i++)
+					exporterSet->parseArgsFunc(argc, argv, i);
+			}
+
+			exporterSet->endProgramFunc();
+		} else {
+			printf("Error: No exporter set, unable to make soh.otr.\n");
+		}
 
 		delete g;
 		return 0;
 	}
-	
+
 	std::string buildMode = argv[1];
 	ZFileMode fileMode = ZFileMode::Invalid;
 

scripts/linux/appimage/soh.sh → scripts/linux/appimage/soh.sh.in

@@ -159,7 +159,7 @@ while [[ (! -e "$SHIP_HOME"/oot.otr) || (! -e "$SHIP_HOME"/oot-mq.otr) ]]; do
                     else
                         echo "Processing..."
                     fi
-                    assets/extractor/ZAPD.out ed -eh -i assets/extractor/xmls/"${ROM}" -b tmp/rom.z64 -fl assets/extractor/filelists -o placeholder -osf placeholder -gsf 1 -rconf assets/extractor/Config_"${ROM}".xml -se OTR --otrfile "${OTRNAME}" > /dev/null 2>&1
+                    assets/extractor/ZAPD.out ed -eh -i assets/extractor/xmls/"${ROM}" -b tmp/rom.z64 -fl assets/extractor/filelists -o placeholder -osf placeholder -gsf 1 -rconf assets/extractor/Config_"${ROM}".xml -se OTR --otrfile "${OTRNAME}" --sohver "@CMAKE_PROJECT_VERSION@" > /dev/null 2>&1
                     cp "$ASSETDIR"/"$OTRNAME" "$SHIP_HOME"
                 fi
             else

soh/CMakeLists.txt

@@ -761,9 +761,11 @@ execute_process(COMMAND ${CURL} -sSfL https://raw.githubusercontent.com/gabomdq/
 
 if("${CMAKE_SYSTEM_NAME}" STREQUAL "Darwin")
 configure_file( ${CMAKE_CURRENT_SOURCE_DIR}/macosx/Info.plist.in ${CMAKE_BINARY_DIR}/macosx/Info.plist @ONLY)
+configure_file( ${CMAKE_CURRENT_SOURCE_DIR}/macosx/soh-macos.sh.in ${CMAKE_BINARY_DIR}/macosx/soh-macos.sh @ONLY)
 INSTALL(FILES ${CMAKE_BINARY_DIR}/gamecontrollerdb.txt DESTINATION ../MacOS COMPONENT ship)
 INSTALL(FILES ${CMAKE_BINARY_DIR}/soh/soh.otr DESTINATION ../Resources COMPONENT ship)
 elseif(NOT "${CMAKE_SYSTEM_NAME}" MATCHES "NintendoSwitch|CafeOS")
+configure_file( ${CMAKE_CURRENT_SOURCE_DIR}/../scripts/linux/appimage/soh.sh.in ${CMAKE_BINARY_DIR}/linux/soh.sh @ONLY)
 INSTALL(FILES ${CMAKE_BINARY_DIR}/gamecontrollerdb.txt DESTINATION . COMPONENT ship)
 endif()
 

soh/macosx/soh-macos.sh → soh/macosx/soh-macos.sh.in

@@ -220,7 +220,7 @@ if [ ! -e "$SHIP_HOME"/oot.otr ] || [ ! -e "$SHIP_HOME"/oot-mq.otr ]; then
 		fi
 
 		osascript -e 'display notification "Generating OTR..." with title "Ship Of Harkinian"'
-		assets/extractor/ZAPD.out ed -i assets/extractor/xmls/"${ROM}" -b tmp/rom.z64 -fl assets/extractor/filelists -o placeholder -osf placeholder -gsf 1 -rconf assets/extractor/Config_"${ROM}".xml -se OTR
+		assets/extractor/ZAPD.out ed -i assets/extractor/xmls/"${ROM}" -b tmp/rom.z64 -fl assets/extractor/filelists -o placeholder -osf placeholder -gsf 1 -rconf assets/extractor/Config_"${ROM}".xml -se OTR --sohver "@CMAKE_PROJECT_VERSION@"
 		if [ -e "$ASSETDIR"/oot.otr ]; then
 			osascript -e 'display notification "OTR successfully generated" with title "Ship Of Harkinian"'
 			cp "$ASSETDIR"/oot.otr "$SHIP_HOME"/"$OTRNAME"

soh/soh/Extractor/Extract.cpp

@@ -7,6 +7,7 @@
 #include "Extract.h"
 #include "portable-file-dialogs.h"
 #include <Utils/BitConverter.h>
+#include "variables.h"
 
 #ifdef unix
 #include <dirent.h>
@@ -527,9 +528,10 @@ std::string Extractor::Mkdtemp() {
 extern "C" int zapd_main(int argc, char** argv);
 
 bool Extractor::CallZapd(std::string installPath, std::string exportdir) {
-    constexpr int argc = 16;
+    constexpr int argc = 18;
     char xmlPath[1024];
     char confPath[1024];
+    char sohVersion[18]; // 5 digits for int16_max (x3) + separators + terminator
     std::array<const char*, argc> argv;
     const char* version = GetZapdVerStr();
     const char* otrFile = IsMasterQuest() ? "oot-mq.otr" : "oot.otr";
@@ -551,6 +553,7 @@ bool Extractor::CallZapd(std::string installPath, std::string exportdir) {
 
     snprintf(xmlPath, 1024, "assets/extractor/xmls/%s", version);
     snprintf(confPath, 1024, "assets/extractor/Config_%s.xml", version);
+    snprintf(sohVersion, 18, "%d.%d.%d", gBuildVersionMajor, gBuildVersionMinor, gBuildVersionPatch);
 
     argv[0] = "ZAPD";
     argv[1] = "ed";
@@ -568,6 +571,8 @@ bool Extractor::CallZapd(std::string installPath, std::string exportdir) {
     argv[13] = "OTR";
     argv[14] = "--otrfile";
     argv[15] = otrFile;
+    argv[16] = "--sohver";
+    argv[17] = sohVersion;
 
 #ifdef _WIN32
     // Grab a handle to the command window.
@@ -593,4 +598,3 @@ bool Extractor::CallZapd(std::string installPath, std::string exportdir) {
 
     return 0;
 }
-