Skip to content
This repository has been archived by the owner on May 11, 2024. It is now read-only.

Commit

Permalink
- [Core] Remove K4os.Compression.LZ4
Browse files Browse the repository at this point in the history
  • Loading branch information
Razmoth committed Nov 11, 2023
1 parent 740b872 commit 5e5b6d1
Show file tree
Hide file tree
Showing 7 changed files with 165 additions and 84 deletions.
15 changes: 11 additions & 4 deletions AssetStudio.Utility/ShaderConverter.cs
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
using K4os.Compression.LZ4;
using SpirV;
using SpirV;
using System;
using System.Globalization;
using System.IO;
Expand All @@ -16,7 +15,11 @@ public static string Convert(this Shader shader)
if (shader.m_SubProgramBlob != null) //5.3 - 5.4
{
var decompressedBytes = new byte[shader.decompressedSize];
LZ4Codec.Decode(shader.m_SubProgramBlob, decompressedBytes);
var numWrite = LZ4.LZ4.Decompress(shader.m_SubProgramBlob, decompressedBytes);
if (numWrite != shader.decompressedSize)
{
throw new IOException($"Lz4 decompression error, write {numWrite} bytes but expected {shader.decompressedSize} bytes");
}
using (var blobReader = new EndianBinaryReader(new MemoryStream(decompressedBytes), EndianType.LittleEndian))
{
var program = new ShaderProgram(blobReader, shader);
Expand Down Expand Up @@ -51,7 +54,11 @@ private static string ConvertSerializedShader(Shader shader)
}
else
{
LZ4Codec.Decode(shader.compressedBlob, (int)offset, (int)compressedLength, decompressedBytes, 0, (int)decompressedLength);
var numWrite = LZ4.LZ4.Decompress(shader.compressedBlob.AsSpan().Slice((int)offset, (int)compressedLength), decompressedBytes.AsSpan().Slice(0, (int)decompressedLength));
if (numWrite != decompressedLength)
{
throw new IOException($"Lz4 decompression error, write {numWrite} bytes but expected {decompressedLength} bytes");
}
}
using (var blobReader = new EndianBinaryReader(new MemoryStream(decompressedBytes), EndianType.LittleEndian))
{
Expand Down
1 change: 0 additions & 1 deletion AssetStudio/AssetStudio.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@
</PropertyGroup>

<ItemGroup>
<PackageReference Include="K4os.Compression.LZ4" Version="1.3.6" />
<PackageReference Include="MessagePack" Version="2.6.100-alpha" />
<PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
<PackageReference Include="ZstdSharp.Port" Version="0.7.2" />
Expand Down
3 changes: 1 addition & 2 deletions AssetStudio/BlbFile.cs
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
using System;
using System.IO;
using System.Linq;
using K4os.Compression.LZ4;

namespace AssetStudio
{
Expand Down Expand Up @@ -131,7 +130,7 @@ private void ReadBlocks(EndianBinaryReader reader, Stream blocksStream)
var compressedBytesSpan = compressedBytes.AsSpan(0, compressedSize);
var uncompressedBytesSpan = uncompressedBytes.AsSpan(0, uncompressedSize);

var numWrite = LZ4Codec.Decode(compressedBytesSpan, uncompressedBytesSpan);
var numWrite = LZ4.LZ4.Decompress(compressedBytesSpan, uncompressedBytesSpan);
if (numWrite != uncompressedSize)
{
throw new IOException($"Lz4 decompression error, write {numWrite} bytes but expected {uncompressedSize} bytes");
Expand Down
13 changes: 8 additions & 5 deletions AssetStudio/BundleFile.cs
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
using K4os.Compression.LZ4;
using ZstdSharp;
using ZstdSharp;
using System;
using System.Data;
using System.IO;
Expand Down Expand Up @@ -438,7 +437,7 @@ private void ReadBlocksInfoAndDirectory(FileReader reader)
case CompressionType.Lz4HC: //LZ4HC
{
var uncompressedBytes = new byte[uncompressedSize];
var numWrite = LZ4Codec.Decode(blocksInfoBytesSpan, uncompressedBytes);
var numWrite = LZ4.LZ4.Decompress(blocksInfoBytesSpan, uncompressedBytes);
if (numWrite != uncompressedSize)
{
throw new IOException($"Lz4 decompression error, write {numWrite} bytes but expected {uncompressedSize} bytes");
Expand Down Expand Up @@ -554,7 +553,7 @@ private void ReadBlocks(FileReader reader, Stream blocksStream)
var uncompressedSize = (int)blockInfo.uncompressedSize;
var uncompressedBytes = BigArrayPool<byte>.Shared.Rent(uncompressedSize);
var uncompressedBytesSpan = uncompressedBytes.AsSpan(0, uncompressedSize);
var numWrite = LZ4Codec.Decode(compressedBytesSpan, uncompressedBytesSpan);
var numWrite = LZ4.LZ4.Decompress(compressedBytesSpan, uncompressedBytesSpan);
if (numWrite != uncompressedSize)
{
throw new IOException($"Lz4 decompression error, write {numWrite} bytes but expected {uncompressedSize} bytes");
Expand All @@ -578,7 +577,11 @@ private void ReadBlocks(FileReader reader, Stream blocksStream)
var uncompressedSize = (int)blockInfo.uncompressedSize;
var uncompressedBytes = BigArrayPool<byte>.Shared.Rent(uncompressedSize);
var uncompressedBytesSpan = uncompressedBytes.AsSpan(0, uncompressedSize);
FairGuardUtils.Lz4.Decompress(compressedBytesSpan, uncompressedBytesSpan);
var numWrite = LZ4.LZ4Inv.Decompress(compressedBytesSpan, uncompressedBytesSpan);
if (numWrite != uncompressedSize)
{
throw new IOException($"Lz4 decompression error, write {numWrite} bytes but expected {uncompressedSize} bytes");
}
blocksStream.Write(uncompressedBytes, 0, uncompressedSize);
BigArrayPool<byte>.Shared.Return(compressedBytes);
BigArrayPool<byte>.Shared.Return(uncompressedBytes);
Expand Down
69 changes: 0 additions & 69 deletions AssetStudio/Crypto/FairGuardUtils.cs
Original file line number Diff line number Diff line change
Expand Up @@ -232,74 +232,5 @@ public static void RC4(Span<byte> data, byte[] key)
data[iteration] ^= (byte)(k - 0x61);
}
}

public static class Lz4
{
public static void Decompress(ReadOnlySpan<byte> cmp, Span<byte> dec)
{
int cmpPos = 0;
int decPos = 0;

// ReSharper disable once VariableHidesOuterVariable
int GetLength(int length, ReadOnlySpan<byte> cmp)
{
byte sum;

if (length == 0xf)
{
do
{
length += sum = cmp[cmpPos++];
} while (sum == 0xff);
}

return length;
}

do
{
byte token = cmp[cmpPos++];

int encCount = (token >> 4) & 0xf;
int litCount = (token >> 0) & 0xf;

//Copy literal chunk
litCount = GetLength(litCount, cmp);

cmp.Slice(cmpPos, litCount).CopyTo(dec.Slice(decPos));

cmpPos += litCount;
decPos += litCount;

if (cmpPos >= cmp.Length)
{
break;
}

//Copy compressed chunk
int back = cmp[cmpPos++] << 8 |
cmp[cmpPos++] << 0;

encCount = GetLength(encCount, cmp) + 4;

int encPos = decPos - back;

if (encCount <= back)
{
dec.Slice(encPos, encCount).CopyTo(dec.Slice(decPos));

decPos += encCount;
}
else
{
while (encCount-- > 0)
{
dec[decPos++] = dec[encPos++];
}
}
} while (cmpPos < cmp.Length &&
decPos < dec.Length);
}
}
}
}
143 changes: 143 additions & 0 deletions AssetStudio/LZ4/LZ4Utils.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,143 @@
using System;

namespace AssetStudio.LZ4;
public static class LZ4
{
public static int Decompress(ReadOnlySpan<byte> cmp, Span<byte> dec)
{
int cmpPos = 0;
int decPos = 0;

// ReSharper disable once VariableHidesOuterVariable
int GetLength(int length, ReadOnlySpan<byte> cmp)
{
byte sum;

if (length == 0xf)
{
do
{
length += sum = cmp[cmpPos++];
} while (sum == 0xff);
}

return length;
}

do
{
byte token = cmp[cmpPos++];

int encCount = (token >> 0) & 0xf;
int litCount = (token >> 4) & 0xf;

//Copy literal chunk
litCount = GetLength(litCount, cmp);

cmp.Slice(cmpPos, litCount).CopyTo(dec.Slice(decPos));

cmpPos += litCount;
decPos += litCount;

if (cmpPos >= cmp.Length)
{
break;
}

//Copy compressed chunk
int back = cmp[cmpPos++] << 0 |
cmp[cmpPos++] << 8;

encCount = GetLength(encCount, cmp) + 4;

int encPos = decPos - back;

if (encCount <= back)
{
dec.Slice(encPos, encCount).CopyTo(dec.Slice(decPos));

decPos += encCount;
}
else
{
while (encCount-- > 0)
{
dec[decPos++] = dec[encPos++];
}
}
} while (cmpPos < cmp.Length &&
decPos < dec.Length);

return decPos;
}
}
public static class LZ4Inv
{
public static int Decompress(ReadOnlySpan<byte> cmp, Span<byte> dec)
{
int cmpPos = 0;
int decPos = 0;

// ReSharper disable once VariableHidesOuterVariable
int GetLength(int length, ReadOnlySpan<byte> cmp)
{
byte sum;

if (length == 0xf)
{
do
{
length += sum = cmp[cmpPos++];
} while (sum == 0xff);
}

return length;
}

do
{
byte token = cmp[cmpPos++];

int encCount = (token >> 4) & 0xf;
int litCount = (token >> 0) & 0xf;

//Copy literal chunk
litCount = GetLength(litCount, cmp);

cmp.Slice(cmpPos, litCount).CopyTo(dec.Slice(decPos));

cmpPos += litCount;
decPos += litCount;

if (cmpPos >= cmp.Length)
{
break;
}

//Copy compressed chunk
int back = cmp[cmpPos++] << 8 |
cmp[cmpPos++] << 0;

encCount = GetLength(encCount, cmp) + 4;

int encPos = decPos - back;

if (encCount <= back)
{
dec.Slice(encPos, encCount).CopyTo(dec.Slice(decPos));

decPos += encCount;
}
else
{
while (encCount-- > 0)
{
dec[decPos++] = dec[encPos++];
}
}
} while (cmpPos < cmp.Length &&
decPos < dec.Length);

return decPos;
}
}
5 changes: 2 additions & 3 deletions AssetStudio/Mhy0File.cs
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
using System;
using System.IO;
using System.Linq;
using K4os.Compression.LZ4;

namespace AssetStudio
{
Expand Down Expand Up @@ -55,7 +54,7 @@ private void ReadBlocksInfoAndDirectory(FileReader reader)
Logger.Verbose($"uncompressed blocksInfo size: 0x{m_Header.uncompressedBlocksInfoSize:X8}");
var compressedBlocksInfo = blocksInfoReader.ReadBytes((int)blocksInfoReader.Remaining);
var uncompressedBlocksInfo = new byte[(int)m_Header.uncompressedBlocksInfoSize];
var numWrite = LZ4Codec.Decode(compressedBlocksInfo, uncompressedBlocksInfo);
var numWrite = LZ4.LZ4.Decompress(compressedBlocksInfo, uncompressedBlocksInfo);
if (numWrite != m_Header.uncompressedBlocksInfoSize)
{
throw new IOException($"Lz4 decompression error, write {numWrite} bytes but expected {m_Header.uncompressedBlocksInfoSize} bytes");
Expand Down Expand Up @@ -128,7 +127,7 @@ private void ReadBlocks(EndianBinaryReader reader, Stream blocksStream)
DescrambleEntry(compressedBytesSpan);

Logger.Verbose($"Descrambled block signature {Convert.ToHexString(compressedBytes, 0, 4)}");
var numWrite = LZ4Codec.Decode(compressedBytesSpan[0xC..compressedSize], uncompressedBytesSpan);
var numWrite = LZ4.LZ4.Decompress(compressedBytesSpan[0xC..compressedSize], uncompressedBytesSpan);
if (numWrite != uncompressedSize)
{
throw new IOException($"Lz4 decompression error, write {numWrite} bytes but expected {uncompressedSize} bytes");
Expand Down

0 comments on commit 5e5b6d1

Please sign in to comment.