Skip to content

Commit cb0b2d6

Browse files
kalimagkalimag
authored
Support passing bytes as Lua strings (#4501)
* NLua: support passing `byte[]` and `Memory<byte>` as Lua strings * Add `read`/`write_bytes_as_string` APIs * Add tests for `params` string arguments * Reduce code duplication * Show as string type in Lua docs * Rename to `as_binary_string` * Add mainmemory methods * Add explanation to wiki text
1 parent 1d6d3d8 commit cb0b2d6

File tree

7 files changed

+235
-6
lines changed

7 files changed

+235
-6
lines changed

‎ExternalProjects/NLua/src/CheckType.cs‎

Lines changed: 15 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,8 @@ public CheckType(ObjectTranslator translator)
3434
_extractValues.Add(typeof(string), GetAsString);
3535
_extractValues.Add(typeof(char[]), GetAsCharArray);
3636
_extractValues.Add(typeof(byte[]), GetAsByteArray);
37+
_extractValues.Add(typeof(Memory<byte>), GetAsByteMemory);
38+
_extractValues.Add(typeof(ReadOnlyMemory<byte>), GetAsByteReadOnlyMemory);
3739
_extractValues.Add(typeof(LuaFunction), GetAsFunction);
3840
_extractValues.Add(typeof(LuaTable), GetAsTable);
3941
_extractValues.Add(typeof(LuaThread), GetAsThread);
@@ -93,9 +95,10 @@ internal ExtractValue CheckLuaType(LuaState luaState, int stackPos, Type paramTy
9395
if (luatype == LuaType.Nil)
9496
{
9597
// Return the correct extractor anyways
96-
if (netParamIsNumeric || paramType == typeof(bool))
98+
// Otherwise the wrong extractor will be cached inside MethodCache.argTypes
99+
if (_extractValues.TryGetValue(paramType, out var extractor))
97100
{
98-
return _extractValues[paramType];
101+
return extractor;
99102
}
100103

101104
return _extractNetObject;
@@ -134,7 +137,8 @@ internal ExtractValue CheckLuaType(LuaState luaState, int stackPos, Type paramTy
134137
}
135138
}
136139

137-
var netParamIsString = paramType == typeof(string) || paramType == typeof(char[]) || paramType == typeof(byte[]);
140+
var netParamIsString = paramType == typeof(string) || paramType == typeof(char[]) ||
141+
paramType == typeof(byte[]) || paramType == typeof(ReadOnlyMemory<byte>) || paramType == typeof(Memory<byte>);
138142

139143
if (netParamIsNumeric)
140144
{
@@ -432,17 +436,22 @@ private static object GetAsCharArray(LuaState luaState, int stackPos)
432436
return retVal.ToCharArray();
433437
}
434438

435-
private static object GetAsByteArray(LuaState luaState, int stackPos)
439+
private static byte[] GetAsByteArray(LuaState luaState, int stackPos)
436440
{
437441
if (!luaState.IsStringOrNumber(stackPos))
438442
{
439443
return null;
440444
}
441445

442-
var retVal = luaState.ToBuffer(stackPos, false);
443-
return retVal;
446+
return luaState.ToBuffer(stackPos, false);
444447
}
445448

449+
private static object GetAsByteMemory(LuaState luaState, int stackPos)
450+
=> GetAsByteArray(luaState, stackPos) is { } bytes ? (Memory<byte>?) bytes : null; // Simply casting null arrays to Memory<T>? actually gives you Memory<T>.Empty
451+
452+
private static object GetAsByteReadOnlyMemory(LuaState luaState, int stackPos)
453+
=> GetAsByteArray(luaState, stackPos) is { } bytes ? (ReadOnlyMemory<byte>?) bytes : null;
454+
446455
private static object GetAsString(LuaState luaState, int stackPos)
447456
=> !luaState.IsStringOrNumber(stackPos) ? null : luaState.ToString(stackPos, false);
448457

‎ExternalProjects/NLua/src/ObjectTranslator.cs‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1057,6 +1057,15 @@ internal void Push(LuaState luaState, object o)
10571057
case bool b:
10581058
luaState.PushBoolean(b);
10591059
break;
1060+
case byte[] bytes:
1061+
luaState.PushBuffer(bytes);
1062+
break;
1063+
case Memory<byte> bytes:
1064+
luaState.PushBuffer(bytes.Span);
1065+
break;
1066+
case ReadOnlyMemory<byte> bytes:
1067+
luaState.PushBuffer(bytes.Span);
1068+
break;
10601069
default:
10611070
{
10621071
if (IsILua(o))

‎References/NLua.dll‎

1 KB
Binary file not shown.

‎src/BizHawk.Client.Common/lua/CommonLibs/MemoryLuaLibrary.cs‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,21 @@ public LuaTable ReadBytesAsArray(long addr, int length, string domain = null)
6969
public LuaTable ReadBytesAsDict(long addr, int length, string domain = null)
7070
=> _th.MemoryBlockToTable(APIs.Memory.ReadByteRange(addr, length, domain), addr);
7171

72+
#pragma warning disable MA0136 // [LuaMethodExample] normalizes line endings
73+
[LuaMethodExample("""
74+
local data = memory.read_bytes_as_binary_string(0x100, 32, "WRAM")
75+
local some_s32_le, some_float = string.unpack("<i4f", data)
76+
for i = 1, #data do
77+
print(data:byte(i))
78+
end
79+
""")]
80+
[LuaMethod("read_bytes_as_binary_string", "Reads {{length}} bytes starting at {{addr}} into a binary string. This string can be read with functions such as {{string.byte}} and {{string.unpack}}. This string can contain any bytes including null bytes, and is not suitable for display as text.")]
81+
public byte[] ReadBytesAsString(long addr, int length, string domain = null)
82+
{
83+
var bytes = APIs.Memory.ReadByteRange(addr, length, domain);
84+
return bytes as byte[] ?? bytes.ToArray();
85+
}
86+
7287
[LuaDeprecatedMethod]
7388
[LuaMethod("writebyterange", "Writes the given values to the given addresses as unsigned bytes")]
7489
public void WriteByteRange(LuaTable memoryblock, string domain = null)
@@ -114,6 +129,16 @@ public void WriteBytesAsDict(LuaTable addrMap, string domain = null)
114129
}
115130
}
116131

132+
[LuaMethodExample("""
133+
memory.write_bytes_as_binary_string(0x100, string.pack("<i4f", 1234, 456.789), "WRAM")
134+
memory.write_bytes_as_binary_string(0x108, "\xFE\xED", "WRAM")
135+
memory.write_bytes_as_binary_string(0x10A, string.char(0xBE, 0xEF), "WRAM")
136+
""")]
137+
[LuaMethod("write_bytes_as_binary_string", "Writes bytes from a binary string to {{addr}}. The string can be created with functions such as {{string.pack}}, and can contain any bytes including null bytes. This is not a text encoding function.")]
138+
public void WriteBytesAsString(long addr, byte[] bytes, string domain = null)
139+
=> APIs.Memory.WriteByteRange(addr, bytes, domain);
140+
#pragma warning restore MA0136
141+
117142
[LuaMethodExample("local simemrea = memory.readfloat( 0x100, false, mainmemory.getname( ) );")]
118143
[LuaMethod("readfloat", "Reads the given address as a 32-bit float value from the main memory domain with th e given endian")]
119144
public float ReadFloat(long addr, bool bigendian, string domain = null)

‎src/BizHawk.Client.Common/lua/LuaDocumentation.cs‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,12 @@ public string ToTASVideosWikiMarkup()
4646
** Some callbacks will be called with arguments, if the function you register has the right number of parameters. This will be noted in the registration function's docs.
4747
* table
4848
** A standard Lua table
49+
* binary string
50+
** A regular Lua string containing raw bytes instead of text, used in some memory functions. Can contain any bytes including null bytes.
51+
** Not encoded as hex or any valid text encoding. Should not be written to the console.
52+
** Can be read using {{string.byte(some_string, index)}} or {{some_string:byte(index)}}
53+
** Created as {{""\x00\x01\x02\x03""}} or {{string.char(0x00, 0x01, 0x02, 0x03)}}
54+
** Can be converted from and to other data types with [https://www.lua.org/manual/5.4/manual.html#pdf-string.pack|string.pack] and [https://www.lua.org/manual/5.4/manual.html#pdf-string.unpack|string.unpack]
4955
* something else
5056
** check the .NET documentation on MSDN")
5157
.AppendLine()
@@ -281,6 +287,11 @@ private static string TypeCleanup(string str)
281287
.Replace("LuaFunction", "func ")
282288
.Replace("Nullable`1[Int32]", "int? ")
283289
.Replace("Nullable`1[UInt32]", "uint? ")
290+
.Replace("Byte[]", "string ")
291+
.Replace("Nullable`1[ReadOnlyMemory`1[Byte]]", "string? ")
292+
.Replace("Nullable`1[Memory`1[Byte]]", "string? ")
293+
.Replace("ReadOnlyMemory`1[Byte]", "string ")
294+
.Replace("Memory`1[Byte]", "string ")
284295
.Replace("Byte", "byte ")
285296
.Replace("Int16", "short ")
286297
.Replace("Int32", "int ")

‎src/BizHawk.Client.Common/lua/LuaHelperLibs/MainMemoryLuaLibrary.cs‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,21 @@ public LuaTable ReadBytesAsArray(long addr, int length)
6969
public LuaTable ReadBytesAsDict(long addr, int length)
7070
=> _th.MemoryBlockToTable(APIs.Memory.ReadByteRange(addr, length, MainMemName), addr);
7171

72+
#pragma warning disable MA0136 // [LuaMethodExample] normalizes line endings
73+
[LuaMethodExample("""
74+
local data = mainmemory.read_bytes_as_binary_string(0x100, 32)
75+
local some_s32_le, some_float = string.unpack("<i4f", data)
76+
for i = 1, #data do
77+
print(data:byte(i))
78+
end
79+
""")]
80+
[LuaMethod("read_bytes_as_binary_string", "Reads {{length}} bytes starting at {{addr}} into a binary string. This string can be read with functions such as {{string.byte}} and {{string.unpack}}. This string can contain any bytes including null bytes, and is not suitable for display as text.")]
81+
public byte[] ReadBytesAsString(long addr, int length)
82+
{
83+
var bytes = APIs.Memory.ReadByteRange(addr, length, MainMemName);
84+
return bytes as byte[] ?? bytes.ToArray();
85+
}
86+
7287
[LuaDeprecatedMethod]
7388
[LuaMethod("writebyterange", "Writes the given values to the given addresses as unsigned bytes")]
7489
public void WriteByteRange(LuaTable memoryblock)
@@ -114,6 +129,16 @@ public void WriteBytesAsDict(LuaTable addrMap)
114129
}
115130
}
116131

132+
[LuaMethodExample("""
133+
mainmemory.write_bytes_as_binary_string(0x100, string.pack("<i4f", 1234, 456.789))
134+
mainmemory.write_bytes_as_binary_string(0x108, "\xFE\xED")
135+
mainmemory.write_bytes_as_binary_string(0x10A, string.char(0xBE, 0xEF))
136+
""")]
137+
[LuaMethod("write_bytes_as_binary_string", "Writes bytes from a binary string to {{addr}}. The string can be created with functions such as {{string.pack}}, and can contain any bytes including null bytes. This is not a text encoding function.")]
138+
public void WriteBytesAsString(long addr, byte[] bytes)
139+
=> APIs.Memory.WriteByteRange(addr, bytes, MainMemName);
140+
#pragma warning restore MA0136
141+
117142
[LuaMethodExample("local simairea = mainmemory.readfloat(0x100, false);")]
118143
[LuaMethod("readfloat", "Reads the given address as a 32-bit float value from the main memory domain with th e given endian")]
119144
public float ReadFloat(long addr, bool bigendian)

‎src/BizHawk.Tests.Client.Common/lua/LuaTests.cs‎

Lines changed: 150 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,38 @@ public static void PassChar(char? o)
8888
public static void PassString(string? o)
8989
=> Assert.IsTrue(o == (string?)ExpectedValue);
9090

91+
[LuaMethod("pass_byte_array", "")]
92+
public static void PassByteArray(byte[]? o)
93+
=> CollectionAssert.AreEqual((byte[]?) ExpectedValue, o);
94+
95+
[LuaMethod("pass_byte_memory", "")]
96+
public static void PassByteMemory(Memory<byte>? o)
97+
{
98+
Assert.AreEqual(ExpectedValue == null, o == null);
99+
if (o is { } memory)
100+
CollectionAssert.AreEqual((byte[]?) ExpectedValue, memory.ToArray());
101+
}
102+
103+
// Separate method to ensure it's not in the NLua method cache yet
104+
[LuaMethod("pass_byte_memory2", "")]
105+
public static void PassByteMemory2(Memory<byte>? o) => PassByteMemory(o);
106+
107+
[LuaMethod("pass_byte_readonlymemory", "")]
108+
public static void PassByteReadOnlyMemory(ReadOnlyMemory<byte>? o)
109+
{
110+
Assert.AreEqual(ExpectedValue == null, o == null);
111+
if (o is { } memory)
112+
CollectionAssert.AreEqual((byte[]?) ExpectedValue, memory.ToArray());
113+
}
114+
115+
[LuaMethod("pass_params_strings", "")]
116+
public static void PassParamsStrings(params string[] o)
117+
=> CollectionAssert.AreEqual((string[]?) ExpectedValue, o);
118+
119+
[LuaMethod("pass_params_objects", "")]
120+
public static void PassParamsObjects(params object[] o)
121+
=> CollectionAssert.AreEqual((object[]?) ExpectedValue, o);
122+
91123
[LuaMethod("pass_color", "")]
92124
public static void PassColor(object? o)
93125
#pragma warning disable MSTEST0026 // "Prefer adding an additional assertion that checks for null" ??? maybe structure like the below method?
@@ -231,6 +263,18 @@ public static void PassCallback(NLua.LuaFunction? o)
231263
public static string? ReturnString()
232264
=> (string?)ReturnValue;
233265

266+
[LuaMethod("return_byte_array", "")]
267+
public static byte[]? ReturnByteArray()
268+
=> (byte[]?)ReturnValue;
269+
270+
[LuaMethod("return_byte_memory", "")]
271+
public static Memory<byte>? ReturnByteMemory()
272+
=> (byte[]?) ReturnValue;
273+
274+
[LuaMethod("return_byte_readonlymemory", "")]
275+
public static ReadOnlyMemory<byte>? ReturnByteReadOnlyMemory()
276+
=> (byte[]?) ReturnValue;
277+
234278
[LuaMethod("return_table", "")]
235279
public static NLua.LuaTable? ReturnTable()
236280
=> (NLua.LuaTable?)ReturnValue;
@@ -487,6 +531,9 @@ public void Net_Return_Nullable()
487531
Assert.IsTrue((bool)LuaInstance.DoString("return return_string() == nil")[0]);
488532
Assert.IsTrue((bool)LuaInstance.DoString("return return_table() == nil")[0]);
489533
Assert.IsTrue((bool)LuaInstance.DoString("return return_callback() == nil")[0]);
534+
Assert.IsTrue((bool)LuaInstance.DoString("return return_byte_array() == nil")[0]);
535+
Assert.IsTrue((bool)LuaInstance.DoString("return return_byte_memory() == nil")[0]);
536+
Assert.IsTrue((bool)LuaInstance.DoString("return return_byte_readonlymemory() == nil")[0]);
490537
}
491538

492539
[TestMethod]
@@ -631,6 +678,48 @@ public void Net_Return_String_Utf8()
631678
}
632679
#pragma warning restore BHI1600
633680

681+
private static (byte[] Bytes, string LuaString) GetTestBytes()
682+
{
683+
var array = Enumerable.Range(0, 256).Select(i => (byte) i).ToArray();
684+
var str = string.Concat(array.Select(value => $@"\{value}"));
685+
return (array, str);
686+
}
687+
688+
[TestMethod]
689+
public void Net_Return_ByteArray()
690+
{
691+
(ReturnValue, var luastr) = GetTestBytes();
692+
Assert.IsTrue(LuaInstance.DoString($"return return_byte_array() == \"{luastr}\"") is [ true ]);
693+
}
694+
695+
[TestMethod]
696+
public void Net_Return_ByteArray_Empty()
697+
{
698+
ReturnValue = Array.Empty<byte>();
699+
Assert.IsTrue(LuaInstance.DoString("return return_byte_array() == \"\"") is [ true ]);
700+
}
701+
702+
[TestMethod]
703+
public void Net_Return_ByteArray_NullTerminated()
704+
{
705+
ReturnValue = new byte[] { 0 };
706+
Assert.IsTrue(LuaInstance.DoString("return return_byte_array() == \"\\0\"") is [ true ]);
707+
}
708+
709+
[TestMethod]
710+
public void Net_Return_ByteMemory()
711+
{
712+
(ReturnValue, var luastr) = GetTestBytes();
713+
Assert.IsTrue(LuaInstance.DoString($"return return_byte_memory() == \"{luastr}\"") is [ true ]);
714+
}
715+
716+
[TestMethod]
717+
public void Net_Return_ByteReadOnlyMemory()
718+
{
719+
(ReturnValue, var luastr) = GetTestBytes();
720+
Assert.IsTrue(LuaInstance.DoString($"return return_byte_readonlymemory() == \"{luastr}\"") is [ true ]);
721+
}
722+
634723
[TestMethod]
635724
public void Net_Return_Color()
636725
{
@@ -693,6 +782,9 @@ public void Net_Argument_Nullable()
693782
LuaInstance.DoString("pass_color(nil)");
694783
LuaInstance.DoString("pass_table(nil)");
695784
LuaInstance.DoString("pass_callback(nil)");
785+
LuaInstance.DoString("pass_byte_array(nil)");
786+
LuaInstance.DoString("pass_byte_memory(nil)");
787+
LuaInstance.DoString("pass_byte_readonlymemory(nil)");
696788
}
697789

698790
[TestMethod]
@@ -851,6 +943,20 @@ public void Net_Argument_String_Implicit_Number_Conversion()
851943
LuaInstance.DoString("pass_string(-0.321)");
852944
}
853945

946+
[TestMethod]
947+
public void Net_Argument_ParamsStrings()
948+
{
949+
ExpectedValue = new string[] { "foo", "bar", "baz" };
950+
LuaInstance.DoString("pass_params_strings(\"foo\", \"bar\", \"baz\")");
951+
}
952+
953+
[TestMethod]
954+
public void Net_Argument_ParamsObjectStrings()
955+
{
956+
ExpectedValue = new object[] { "foo", "bar", "baz" };
957+
LuaInstance.DoString("pass_params_objects(\"foo\", \"bar\", \"baz\")");
958+
}
959+
854960
[TestMethod]
855961
public void Net_Argument_Color()
856962
{
@@ -892,6 +998,50 @@ public void Net_Argument_LuaFunction()
892998
LuaInstance.DoString("pass_callback(function() pass_f64(123.0) end)");
893999
}
8941000

1001+
[TestMethod]
1002+
public void Net_Argument_StringToByteArray()
1003+
{
1004+
(ExpectedValue, var luastr) = GetTestBytes();
1005+
LuaInstance.DoString($"pass_byte_array(\"{luastr}\")");
1006+
}
1007+
1008+
[TestMethod]
1009+
public void Net_Argument_StringToByteMemory()
1010+
{
1011+
(ExpectedValue, var luastr) = GetTestBytes();
1012+
LuaInstance.DoString($"pass_byte_memory(\"{luastr}\")");
1013+
}
1014+
1015+
[TestMethod]
1016+
public void Net_Argument_StringToByteReadOnlyMemory ()
1017+
{
1018+
(ExpectedValue, var luastr) = GetTestBytes();
1019+
LuaInstance.DoString($"pass_byte_readonlymemory(\"{luastr}\")");
1020+
}
1021+
1022+
[TestMethod]
1023+
public void Net_Argument_StringToByteMemory_Empty()
1024+
{
1025+
ExpectedValue = Array.Empty<byte>();
1026+
LuaInstance.DoString("""pass_byte_memory("")""");
1027+
}
1028+
1029+
[TestMethod]
1030+
public void Net_Argument_StringToByteMemory_NullTerminated()
1031+
{
1032+
ExpectedValue = new byte[] { 0 };
1033+
LuaInstance.DoString("""pass_byte_memory("\0")""");
1034+
}
1035+
1036+
[TestMethod]
1037+
public void Net_Argument_StringToByteMemory_NullThenNotNull()
1038+
{
1039+
ExpectedValue = null;
1040+
LuaInstance.DoString("pass_byte_memory2(nil)");
1041+
ExpectedValue = new byte[] { 1, 2, 3 };
1042+
LuaInstance.DoString("""pass_byte_memory2("\1\2\3")""");
1043+
}
1044+
8951045
[DataRow(new long[0], "pass_table_ipairs({})")]
8961046
[DataRow(new[] { 0x11L, 0x22L, 0x33L, 0x44L }, "pass_table_ipairs({ 0x11, 0x22, 0x33, 0x44 })")]
8971047
[DataRow(new[] { 0x11L, 0x22L, 0x33L }, "pass_table_ipairs({ 0x11, 0x22, 0x33, [true] = 0x44 })")]

0 commit comments

Comments
 (0)