Savestates: Multi-threaded compression, use ZSTD

kd-11 · May 14, 2024 · 66008d5 · 66008d5
1 parent 7c31370
commit 66008d5
Show file tree

Hide file tree

Showing 17 changed files with 985 additions and 24 deletions.
diff --git a/.gitmodules b/.gitmodules
@@ -88,3 +88,7 @@
 	path = 3rdparty/rtmidi/rtmidi
 	url = ../../thestk/rtmidi
 	ignore = dirty
+[submodule "3rdparty/zstd/zstd"]
+	path = 3rdparty/zstd/zstd
+	url = ../../facebook/zstd
+	ignore = dirty
diff --git a/3rdparty/CMakeLists.txt b/3rdparty/CMakeLists.txt
@@ -16,6 +16,9 @@ add_library(3rdparty_dummy_lib INTERFACE)
 # ZLib
 add_subdirectory(zlib EXCLUDE_FROM_ALL)
 
+# ZSTD
+add_subdirectory(zstd EXCLUDE_FROM_ALL)
+
 # 7z sdk
 add_subdirectory(7z EXCLUDE_FROM_ALL)
 
@@ -356,6 +359,7 @@ else()
 	add_library(3rdparty::libusb ALIAS usb-1.0-static)
 endif()
 add_library(3rdparty::zlib ALIAS 3rdparty_zlib)
+add_library(3rdparty::zstd ALIAS 3rdparty_zstd)
 add_library(3rdparty::7z ALIAS 3rdparty_7z)
 add_library(3rdparty::flatbuffers ALIAS 3rdparty_flatbuffers)
 add_library(3rdparty::pugixml ALIAS pugixml)

diff --git a/3rdparty/rtmidi/CMakeLists.txt b/3rdparty/rtmidi/CMakeLists.txt
@@ -1,2 +1,3 @@
 option(RTMIDI_API_JACK "Compile with JACK support." OFF)
+set(RTMIDI_TARGETNAME_UNINSTALL "uninstall-rpcs3-rtmidi")
 add_subdirectory(rtmidi EXCLUDE_FROM_ALL)
diff --git a/3rdparty/zstd/CMakeLists.txt b/3rdparty/zstd/CMakeLists.txt
@@ -0,0 +1,9 @@
+
+project(3rdparty_zstd)
+
+add_library(3rdparty_zstd INTERFACE)
+
+add_subdirectory(zstd/build/cmake EXLUDE_FROM_ALL)
+
+target_include_directories(3rdparty_zstd INTERFACE zstd zstd/lib)
+
diff --git a/3rdparty/zstd/zstd b/3rdparty/zstd/zstd
diff --git a/3rdparty/zstd/zstd.rc b/3rdparty/zstd/zstd.rc
@@ -0,0 +1,51 @@
+// Microsoft Visual C++ generated resource script.
+//
+
+#include "zstd\lib\zstd.h" /* ZSTD_VERSION_STRING */
+#define APSTUDIO_READONLY_SYMBOLS
+#include "verrsrc.h"
+#undef APSTUDIO_READONLY_SYMBOLS
+
+
+#if !defined(AFX_RESOURCE_DLL) || defined(AFX_TARG_ENU)
+LANGUAGE 9, 1
+
+/////////////////////////////////////////////////////////////////////////////
+//
+// Version
+//
+
+VS_VERSION_INFO     VERSIONINFO
+  FILEVERSION       ZSTD_VERSION_MAJOR,ZSTD_VERSION_MINOR,ZSTD_VERSION_RELEASE,0
+  PRODUCTVERSION    ZSTD_VERSION_MAJOR,ZSTD_VERSION_MINOR,ZSTD_VERSION_RELEASE,0
+ FILEFLAGSMASK VS_FFI_FILEFLAGSMASK
+#ifdef _DEBUG
+ FILEFLAGS VS_FF_DEBUG
+#else
+ FILEFLAGS 0x0L
+#endif
+ FILEOS VOS_NT_WINDOWS32
+ FILETYPE VFT_DLL
+ FILESUBTYPE VFT2_UNKNOWN
+BEGIN
+    BLOCK "StringFileInfo"
+    BEGIN
+        BLOCK "040904B0"
+        BEGIN
+            VALUE "CompanyName", "Meta Platforms, Inc."
+            VALUE "FileDescription", "Zstandard - Fast and efficient compression algorithm"
+            VALUE "FileVersion", ZSTD_VERSION_STRING
+            VALUE "InternalName", "zstd.exe"
+            VALUE "LegalCopyright", "Copyright (c) Meta Platforms, Inc. and affiliates."
+            VALUE "OriginalFilename", "zstd.exe"
+            VALUE "ProductName", "Zstandard"
+            VALUE "ProductVersion", ZSTD_VERSION_STRING
+        END
+    END
+    BLOCK "VarFileInfo"
+    BEGIN
+        VALUE "Translation", 0x0409, 1200
+    END
+END
+
+#endif
diff --git a/3rdparty/zstd/zstd.vcxproj b/3rdparty/zstd/zstd.vcxproj
@@ -0,0 +1,193 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="14.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <ItemGroup Label="ProjectConfigurations">
+    <ProjectConfiguration Include="Debug|x64">
+      <Configuration>Debug</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|x64">
+      <Configuration>Release</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+  </ItemGroup>
+  <ItemGroup>
+    <ClCompile Include=".\zstd\lib\common\debug.c" />
+    <ClCompile Include=".\zstd\lib\common\entropy_common.c" />
+    <ClCompile Include=".\zstd\lib\common\error_private.c" />
+    <ClCompile Include=".\zstd\lib\common\pool.c" />
+    <ClCompile Include=".\zstd\lib\common\threading.c" />
+    <ClCompile Include=".\zstd\lib\common\xxhash.c" />
+    <ClCompile Include=".\zstd\lib\common\zstd_common.c" />
+    <ClCompile Include=".\zstd\lib\common\fse_decompress.c" />
+    <ClCompile Include=".\zstd\lib\compress\hist.c" />
+    <ClCompile Include=".\zstd\lib\compress\fse_compress.c" />
+    <ClCompile Include=".\zstd\lib\compress\huf_compress.c" />
+    <ClCompile Include=".\zstd\lib\compress\zstdmt_compress.c" />
+    <ClCompile Include=".\zstd\lib\compress\zstd_compress.c" />
+    <ClCompile Include=".\zstd\lib\compress\zstd_compress_literals.c" />
+    <ClCompile Include=".\zstd\lib\compress\zstd_compress_sequences.c" />
+    <ClCompile Include=".\zstd\lib\compress\zstd_compress_superblock.c" />
+    <ClCompile Include=".\zstd\lib\compress\zstd_fast.c" />
+    <ClCompile Include=".\zstd\lib\compress\zstd_double_fast.c" />
+    <ClCompile Include=".\zstd\lib\compress\zstd_lazy.c" />
+    <ClCompile Include=".\zstd\lib\compress\zstd_opt.c" />
+    <ClCompile Include=".\zstd\lib\compress\zstd_ldm.c" />
+    <ClCompile Include=".\zstd\lib\decompress\huf_decompress.c" />
+    <ClCompile Include=".\zstd\lib\decompress\zstd_decompress.c" />
+    <ClCompile Include=".\zstd\lib\decompress\zstd_decompress_block.c" />
+    <ClCompile Include=".\zstd\lib\decompress\zstd_ddict.c" />
+    <ClCompile Include=".\zstd\lib\dictBuilder\cover.c" />
+    <ClCompile Include=".\zstd\lib\dictBuilder\fastcover.c" />
+    <ClCompile Include=".\zstd\lib\dictBuilder\divsufsort.c" />
+    <ClCompile Include=".\zstd\lib\dictBuilder\zdict.c" />
+    <ClCompile Include=".\zstd\lib\legacy\zstd_v01.c" />
+    <ClCompile Include=".\zstd\lib\legacy\zstd_v02.c" />
+    <ClCompile Include=".\zstd\lib\legacy\zstd_v03.c" />
+    <ClCompile Include=".\zstd\lib\legacy\zstd_v04.c" />
+    <ClCompile Include=".\zstd\lib\legacy\zstd_v05.c" />
+    <ClCompile Include=".\zstd\lib\legacy\zstd_v06.c" />
+    <ClCompile Include=".\zstd\lib\legacy\zstd_v07.c" />
+    <ClCompile Include=".\zstd\programs\util.c" />
+    <ClCompile Include=".\zstd\programs\timefn.c" />
+    <ClCompile Include=".\zstd\programs\benchfn.c" />
+    <ClCompile Include=".\zstd\programs\benchzstd.c" />
+    <ClCompile Include=".\zstd\programs\datagen.c" />
+    <ClCompile Include=".\zstd\programs\dibio.c" />
+    <ClCompile Include=".\zstd\programs\fileio.c" />
+    <ClCompile Include=".\zstd\programs\fileio_asyncio.c" />
+    <ClCompile Include=".\zstd\programs\lorem.c" />
+    <ClCompile Include=".\zstd\programs\zstdcli.c" />
+    <ClCompile Include=".\zstd\programs\zstdcli_trace.c" />
+  </ItemGroup>
+  <ItemGroup>
+    <ClInclude Include=".\zstd\lib\common\pool.h" />
+    <ClInclude Include=".\zstd\lib\common\threading.h" />
+    <ClInclude Include=".\zstd\lib\common\xxhash.h" />
+    <ClInclude Include=".\zstd\lib\compress\zstdmt_compress.h" />
+    <ClInclude Include=".\zstd\lib\zdict.h" />
+    <ClInclude Include=".\zstd\lib\dictBuilder\cover.h" />
+    <ClInclude Include=".\zstd\lib\dictBuilder\divsufsort.h" />
+    <ClInclude Include=".\zstd\lib\common\fse.h" />
+    <ClInclude Include=".\zstd\lib\common\huf.h" />
+    <ClInclude Include=".\zstd\lib\zstd.h" />
+    <ClInclude Include=".\zstd\lib\common\zstd_internal.h" />
+    <ClInclude Include=".\zstd\lib\zstd_errors.h" />
+    <ClInclude Include=".\zstd\lib\compress\zstd_compress.h" />
+    <ClInclude Include=".\zstd\lib\compress\zstd_compress_literals.h" />
+    <ClInclude Include=".\zstd\lib\compress\zstd_compress_sequences.h" />
+    <ClInclude Include=".\zstd\lib\compress\zstd_cwksp.h" />
+    <ClInclude Include=".\zstd\lib\compress\zstd_compress_superblock.h" />
+    <ClInclude Include=".\zstd\lib\compress\zstd_fast.h" />
+    <ClInclude Include=".\zstd\lib\compress\zstd_double_fast.h" />
+    <ClInclude Include=".\zstd\lib\compress\zstd_lazy.h" />
+    <ClInclude Include=".\zstd\lib\compress\zstd_opt.h" />
+    <ClInclude Include=".\zstd\lib\compress\zstd_ldm.h" />
+    <ClInclude Include=".\zstd\lib\decompress\zstd_ddict.h" />
+    <ClInclude Include=".\zstd\lib\legacy\zstd_legacy.h" />
+    <ClInclude Include=".\zstd\lib\legacy\zstd_v01.h" />
+    <ClInclude Include=".\zstd\lib\legacy\zstd_v02.h" />
+    <ClInclude Include=".\zstd\lib\legacy\zstd_v03.h" />
+    <ClInclude Include=".\zstd\lib\legacy\zstd_v04.h" />
+    <ClInclude Include=".\zstd\lib\legacy\zstd_v05.h" />
+    <ClInclude Include=".\zstd\lib\legacy\zstd_v06.h" />
+    <ClInclude Include=".\zstd\lib\legacy\zstd_v07.h" />
+    <ClInclude Include=".\zstd\programs\benchzstd.h" />
+    <ClInclude Include=".\zstd\programs\datagen.h" />
+    <ClInclude Include=".\zstd\programs\dibio.h" />
+    <ClInclude Include=".\zstd\programs\fileio.h" />
+    <ClInclude Include=".\zstd\programs\platform.h" />
+    <ClInclude Include=".\zstd\programs\util.h" />
+  </ItemGroup>
+  <ItemGroup>
+    <ResourceCompile Include="zstd.rc" />
+  </ItemGroup>
+  <PropertyGroup Label="Globals">
+    <ProjectGuid>{4E52A41A-F33B-4C7A-8C36-A1A6B4F4277C}</ProjectGuid>
+    <Keyword>Win32Proj</Keyword>
+    <RootNamespace>zstd</RootNamespace>
+    <OutDir>$(SolutionDir)lib\$(Configuration)-$(Platform)\</OutDir>
+    <IntDir>$(SolutionDir)tmp\$(ProjectName)-$(Configuration)-$(Platform)\</IntDir>
+    <InstructionSet>NotSet</InstructionSet>
+    <WindowsTargetPlatformVersion>10.0</WindowsTargetPlatformVersion>
+  </PropertyGroup>
+  <Import Project="$(SolutionDir)\buildfiles\msvc\common_default.props" />
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+  <Import Project="$(SolutionDir)\buildfiles\msvc\common_default_macros.props" />
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <UseDebugLibraries>true</UseDebugLibraries>
+    <CharacterSet>MultiByte</CharacterSet>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+    <ConfigurationType>StaticLibrary</ConfigurationType>
+    <UseDebugLibraries>false</UseDebugLibraries>
+    <WholeProgramOptimization>true</WholeProgramOptimization>
+    <CharacterSet>MultiByte</CharacterSet>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+  <ImportGroup Label="ExtensionSettings">
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <PropertyGroup Label="UserMacros" />
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+    <LinkIncremental>true</LinkIncremental>
+    <IncludePath>$(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\compress;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
+    <RunCodeAnalysis>false</RunCodeAnalysis>
+    <LibraryPath>$(LibraryPath);</LibraryPath>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+    <LinkIncremental>false</LinkIncremental>
+    <IncludePath>$(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\compress;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
+    <RunCodeAnalysis>false</RunCodeAnalysis>
+    <LibraryPath>$(LibraryPath);</LibraryPath>
+  </PropertyGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+    <ClCompile>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <WarningLevel>Level4</WarningLevel>
+      <Optimization>Disabled</Optimization>
+      <PreprocessorDefinitions>ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <TreatWarningAsError>true</TreatWarningAsError>
+      <EnablePREfast>false</EnablePREfast>
+      <EnableEnhancedInstructionSet>$(InstructionSet)</EnableEnhancedInstructionSet>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <AdditionalDependencies>setargv.obj;%(AdditionalDependencies)</AdditionalDependencies>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+    <ClCompile>
+      <WarningLevel>Level4</WarningLevel>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <Optimization>MaxSpeed</Optimization>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <IntrinsicFunctions>true</IntrinsicFunctions>
+      <PreprocessorDefinitions>ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <TreatWarningAsError>false</TreatWarningAsError>
+      <EnablePREfast>false</EnablePREfast>
+      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
+      <AdditionalOptions>/DZSTD_MULTITHREAD %(AdditionalOptions)</AdditionalOptions>
+      <EnableEnhancedInstructionSet>$(InstructionSet)</EnableEnhancedInstructionSet>
+      <LanguageStandard_C>Default</LanguageStandard_C>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <EnableCOMDATFolding>true</EnableCOMDATFolding>
+      <OptimizeReferences>true</OptimizeReferences>
+      <AdditionalDependencies>setargv.obj;%(AdditionalDependencies)</AdditionalDependencies>
+    </Link>
+  </ItemDefinitionGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+  <ImportGroup Label="ExtensionTargets">
+  </ImportGroup>
+</Project>
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -114,6 +114,8 @@ if(MSVC)
     add_compile_options(/wd4530 /utf-8) # C++ exception handler used, but unwind semantics are not enabled
 endif()
 
+set(ALLOW_DUPLICATE_CUSTOM_TARGETS TRUE)
+
 add_subdirectory(3rdparty)
 
 if (DISABLE_LTO)

diff --git a/rpcs3.sln b/rpcs3.sln
@@ -100,6 +100,8 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "miniupnpc_static", "3rdpart
 EndProject
 Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "rtmidi", "3rdparty\rtmidi\rtmidi.vcxproj", "{2C902C67-985C-4BE0-94A3-E0FE2EB929A3}"
 EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "zstd", "3rdparty\zstd\zstd.vcxproj", "{4E52A41A-F33B-4C7A-8C36-A1A6B4F4277C}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|x64 = Debug|x64
@@ -202,6 +204,10 @@ Global
 		{2C902C67-985C-4BE0-94A3-E0FE2EB929A3}.Debug|x64.Build.0 = Debug|x64
 		{2C902C67-985C-4BE0-94A3-E0FE2EB929A3}.Release|x64.ActiveCfg = Release|x64
 		{2C902C67-985C-4BE0-94A3-E0FE2EB929A3}.Release|x64.Build.0 = Release|x64
+		{4E52A41A-F33B-4C7A-8C36-A1A6B4F4277C}.Debug|x64.ActiveCfg = Debug|x64
+		{4E52A41A-F33B-4C7A-8C36-A1A6B4F4277C}.Debug|x64.Build.0 = Debug|x64
+		{4E52A41A-F33B-4C7A-8C36-A1A6B4F4277C}.Release|x64.ActiveCfg = Release|x64
+		{4E52A41A-F33B-4C7A-8C36-A1A6B4F4277C}.Release|x64.Build.0 = Release|x64
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -233,6 +239,7 @@ Global
 		{A55DA1B5-CC17-4525-BE7F-1659CE17BB56} = {6C3B64A0-8F8A-4DC4-8C0B-D71EBEED7FA8}
 		{5228F863-E0DD-4DE7-AA7B-5C52B14CD4D0} = {6C3B64A0-8F8A-4DC4-8C0B-D71EBEED7FA8}
 		{2C902C67-985C-4BE0-94A3-E0FE2EB929A3} = {6C3B64A0-8F8A-4DC4-8C0B-D71EBEED7FA8}
+		{4E52A41A-F33B-4C7A-8C36-A1A6B4F4277C} = {6C3B64A0-8F8A-4DC4-8C0B-D71EBEED7FA8}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {06CC7920-E085-4B81-9582-8DE8AAD42510}