Skip to content

Commit 58d67cc

Browse files
lemirelemire
committed
fix: complete correction
1 parent 900472e commit 58d67cc

File tree

1 file changed

+34
-46
lines changed

1 file changed

+34
-46
lines changed

‎test/UTF8ValidationTests.cs‎

Lines changed: 34 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -13,12 +13,12 @@ public unsafe class Utf8SIMDValidationTests
1313
{
1414

1515

16-
private const int NumTrials = 1000;
16+
private const int NumTrials = 100;
1717
private static readonly RandomUtf8 generator = new RandomUtf8(1234, 1, 1, 1, 1);
1818
private static readonly Random rand = new Random();
1919

2020
// int[] outputLengths = { 128, 192, 256, 320, 384, 448, 512, 576, 640, 704, 768, 832, 896, 960, 1024, 1088, 1152, 1216, 1280, 1344, 1408, 1472, 1536, 1600, 1664, 1728, 1792, 1856, 1920, 1984, 2048, 2112, 2176, 2240, 2304, 2368, 2432, 2496, 2560, 2624, 2688, 2752, 2816, 2880, 2944, 3008, 3072, 3136, 3200, 3264, 3328, 3392, 3456, 3520, 3584, 3648, 3712, 3776, 3840, 3904, 3968, 4032, 4096, 4160, 4224, 4288, 4352, 4416, 4480, 4544, 4608, 4672, 4736, 4800, 4864, 4928, 4992, 5056, 5120, 5184, 5248, 5312, 5376, 5440, 5504, 5568, 5632, 5696, 5760, 5824, 5888, 5952, 6016, 6080, 6144, 6208, 6272, 6336, 6400, 6464, 6528, 6592, 6656, 6720, 6784, 6848, 6912, 6976, 7040, 7104, 7168, 7232, 7296, 7360, 7424, 7488, 7552, 7616, 7680, 7744, 7808, 7872, 7936, 8000, 8064, 8128, 8192, 8256, 8320, 8384, 8448, 8512, 8576, 8640, 8704, 8768, 8832, 8896, 8960, 9024, 9088, 9152, 9216, 9280, 9344, 9408, 9472, 9536, 9600, 9664, 9728, 9792, 9856, 9920, 9984, 10000 };
21-
static int[] outputLengths = { 128, 256,345, 512,968, 1024, 1000 };
21+
static int[] outputLengths = { 128, 345, 1000 };
2222

2323
[Flags]
2424
public enum TestSystemRequirements
@@ -76,7 +76,7 @@ public TestIfCondition(Func<bool> condition, string skipReason)
7676

7777

7878

79-
public void simpleGoodSequences(Utf8ValidationDelegate utf8ValidationDelegate)
79+
private void simpleGoodSequences(Utf8ValidationDelegate utf8ValidationDelegate)
8080
{
8181
string[] goodSequences = {
8282
"a",
@@ -144,7 +144,7 @@ public void simpleGoodSequencesAVX()
144144
}
145145

146146

147-
public void BadSequences(Utf8ValidationDelegate utf8ValidationDelegate)
147+
private void BadSequences(Utf8ValidationDelegate utf8ValidationDelegate)
148148
{
149149
string[] badSequences = {
150150
"\xC3\x28",
@@ -231,13 +231,13 @@ public void BadSequencesAVX()
231231
}
232232

233233
// this was in the C++ code
234-
public void Node48995Test(Utf8ValidationDelegate utf8ValidationDelegate)
234+
private void Node48995Test(Utf8ValidationDelegate utf8ValidationDelegate)
235235
{
236236
byte[] bad = new byte[] { 0x80 };
237237
Assert.False(ValidateUtf8(bad,utf8ValidationDelegate));
238238
}
239239

240-
public void NoError(Utf8ValidationDelegate utf8ValidationDelegate)
240+
private void NoError(Utf8ValidationDelegate utf8ValidationDelegate)
241241
{
242242
foreach (int outputLength in outputLengths)
243243
{
@@ -299,7 +299,7 @@ public void NoErrorAVX()
299299
NoError(SimdUnicode.UTF8.GetPointerToFirstInvalidByteAvx2);
300300
}
301301

302-
public void NoErrorSpecificByteCount(Utf8ValidationDelegate utf8ValidationDelegate)
302+
private void NoErrorSpecificByteCount(Utf8ValidationDelegate utf8ValidationDelegate)
303303
{
304304
RunTestForByteLength(1,utf8ValidationDelegate);
305305
RunTestForByteLength(2,utf8ValidationDelegate);
@@ -369,7 +369,7 @@ public void NoErrorSpecificByteCountAVX()
369369
NoErrorSpecificByteCount(SimdUnicode.UTF8.GetPointerToFirstInvalidByteAvx2);
370370
}
371371

372-
public void NoErrorIncompleteThenASCII(Utf8ValidationDelegate utf8ValidationDelegate)
372+
private void NoErrorIncompleteThenASCII(Utf8ValidationDelegate utf8ValidationDelegate)
373373
{
374374
foreach (int outputLength in outputLengths){
375375
for (int trial = 0; trial < NumTrials; trial++)
@@ -438,20 +438,18 @@ public void NoErrorIncompleteThenASCIIScalar()
438438
// NoErrorIncompleteThenASCII(SimdUnicode.UTF8.GetPointerToFirstInvalidByteArm64);
439439
// }
440440

441-
[Fact]
442-
[Trait("Category", "avx")]
441+
[FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)]
443442
public void NoErrorIncompleteThenASCIIAVX()
444443
{
445444
NoErrorIncompleteThenASCII(SimdUnicode.UTF8.GetPointerToFirstInvalidByteAvx2);
446445
}
447446

448447

449448

450-
public void NoErrorIncompleteAt256Vector(Utf8ValidationDelegate utf8ValidationDelegate)
449+
private void NoErrorIncompleteAt256Vector(Utf8ValidationDelegate utf8ValidationDelegate)
451450
{
452-
// foreach (int outputLength in outputLengths)
451+
foreach (int outputLength in outputLengths)
453452
{
454-
int outputLength = 256;
455453
for (int trial = 0; trial < NumTrials; trial++)
456454
{
457455

@@ -515,14 +513,13 @@ public void NoErrorIncompleteAt256VectorScalar()
515513
// NoErrorIncompleteAt256Vector(SimdUnicode.UTF8.GetPointerToFirstInvalidByteArm64);
516514
// }
517515

518-
[Fact]
519-
[Trait("Category", "avx")]
516+
[FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)]
520517
public void NoErrorIncompleteAt256VectorAVX()
521518
{
522519
NoErrorIncompleteAt256Vector(SimdUnicode.UTF8.GetPointerToFirstInvalidByteAvx2);
523520
}
524521

525-
public void BadHeaderBits(Utf8ValidationDelegate utf8ValidationDelegate)
522+
private void BadHeaderBits(Utf8ValidationDelegate utf8ValidationDelegate)
526523
{
527524
foreach (int outputLength in outputLengths)
528525
{
@@ -590,14 +587,13 @@ public void BadHeaderBitsScalar()
590587
// NoErrorSpecificByteCount(SimdUnicode.UTF8.GetPointerToFirstInvalidByteArm64);
591588
// }
592589

593-
[Fact]
594-
[Trait("Category", "avx")]
590+
[FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)]
595591
public void BadHeaderBitsAVX()
596592
{
597593
BadHeaderBits(SimdUnicode.UTF8.GetPointerToFirstInvalidByteAvx2);
598594
}
599595

600-
public void TooShortError(Utf8ValidationDelegate utf8ValidationDelegate)
596+
private void TooShortError(Utf8ValidationDelegate utf8ValidationDelegate)
601597
{
602598
foreach (int outputLength in outputLengths)
603599
{
@@ -663,14 +659,13 @@ public void TooShortErrorScalar()
663659
// TooShortError(SimdUnicode.UTF8.GetPointerToFirstInvalidByteArm64);
664660
// }
665661

666-
[Fact]
667-
[Trait("Category", "avx")]
662+
[FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)]
668663
public void TooShortErrorAVX()
669664
{
670665
TooShortError(SimdUnicode.UTF8.GetPointerToFirstInvalidByteAvx2);
671666
}
672667

673-
public void TooLongError(Utf8ValidationDelegate utf8ValidationDelegate)
668+
private void TooLongError(Utf8ValidationDelegate utf8ValidationDelegate)
674669
{
675670

676671
foreach (int outputLength in outputLengths)
@@ -736,14 +731,13 @@ public void TooLongErrorScalar()
736731
// TooLongError(SimdUnicode.UTF8.GetPointerToFirstInvalidByteArm64);
737732
// }
738733

739-
[Fact]
740-
[Trait("Category", "avx")]
734+
[FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)]
741735
public void TooLongErrorAVX()
742736
{
743737
TooLongError(SimdUnicode.UTF8.GetPointerToFirstInvalidByteAvx2);
744738
}
745739

746-
public void OverlongError(Utf8ValidationDelegate utf8ValidationDelegate)
740+
private void OverlongError(Utf8ValidationDelegate utf8ValidationDelegate)
747741
{
748742
for (int trial = 0; trial < NumTrials; trial++)
749743
{
@@ -818,15 +812,14 @@ public void OverlongErrorScalar()
818812
// OverlongError(SimdUnicode.UTF8.GetPointerToFirstInvalidByteArm64);
819813
// }
820814

821-
[Fact]
822-
[Trait("Category", "avx")]
815+
[FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)]
823816
public void OverlongErrorAVX()
824817
{
825818
OverlongError(SimdUnicode.UTF8.GetPointerToFirstInvalidByteAvx2);
826819
}
827820

828821

829-
public void TooShortErrorAtEnd(Utf8ValidationDelegate utf8ValidationDelegate)
822+
private void TooShortErrorAtEnd(Utf8ValidationDelegate utf8ValidationDelegate)
830823
{
831824
for (int trial = 0; trial < NumTrials; trial++)
832825
{
@@ -912,15 +905,15 @@ public void TooShortErrorAtEndAVX()
912905
TooShortErrorAtEnd(SimdUnicode.UTF8.GetPointerToFirstInvalidByteAvx2);
913906
}
914907

915-
[Fact]
908+
[FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)]
916909
public void TooShortErrorAtEndAvx2()
917910
{
918911
TooShortErrorAtEnd(SimdUnicode.UTF8.GetPointerToFirstInvalidByteAvx2);
919912
}
920913

921914

922915
//corresponds to condition 5.4.1 in the paper
923-
public void Invalid0xf50xff(Utf8ValidationDelegate utf8ValidationDelegate)
916+
private void Invalid0xf50xff(Utf8ValidationDelegate utf8ValidationDelegate)
924917
{
925918

926919
var invalidBytes = Enumerable.Range(0xF5, 0x100 - 0xF5).Select(i => (byte)i).ToArray(); // 0xF5 to 0xFF
@@ -972,14 +965,13 @@ public void Invalid0xf50xffScalar()
972965
// Invalid0xf50xff(SimdUnicode.UTF8.GetPointerToFirstInvalidByteArm64);
973966
// }
974967

975-
[Fact]
976-
[Trait("Category", "avx")]
968+
[FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)]
977969
public void Invalid0xf50xffAVX()
978970
{
979971
Invalid0xf50xff(SimdUnicode.UTF8.GetPointerToFirstInvalidByteAvx2);
980972
}
981973

982-
[Fact]
974+
[FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)]
983975
public void Invalid0xf50xffAvx2()
984976
{
985977
Invalid0xf50xff(SimdUnicode.UTF8.GetPointerToFirstInvalidByteAvx2);
@@ -1049,7 +1041,7 @@ static void PrintHexAndBinary(byte[] bytes, int highlightIndex = -1)
10491041
}
10501042

10511043

1052-
public void TooLargeError(Utf8ValidationDelegate utf8ValidationDelegate)
1044+
private void TooLargeError(Utf8ValidationDelegate utf8ValidationDelegate)
10531045
{
10541046
foreach (int outputLength in outputLengths)
10551047
{
@@ -1106,15 +1098,14 @@ public void TooLargeErrorScalar()
11061098
// TooLargeError(SimdUnicode.UTF8.GetPointerToFirstInvalidByteArm64);
11071099
// }
11081100

1109-
[Fact]
1110-
[Trait("Category", "avx")]
1101+
[FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)]
11111102
public void TooLargeErrorAvx()
11121103
{
11131104
TooLargeError(SimdUnicode.UTF8.GetPointerToFirstInvalidByteAvx2);
11141105
}
11151106

11161107

1117-
public void AsciiPlusContinuationAtEndError(Utf8ValidationDelegate utf8ValidationDelegate)
1108+
private void AsciiPlusContinuationAtEndError(Utf8ValidationDelegate utf8ValidationDelegate)
11181109
{
11191110
foreach (int outputLength in outputLengths)
11201111
{
@@ -1169,20 +1160,19 @@ public void AsciiPlusContinuationAtEndErrorScalar()
11691160
// AsciiPlusContinuationAtEndError(SimdUnicode.UTF8.GetPointerToFirstInvalidByteArm64);
11701161
// }
11711162

1172-
[Fact]
1173-
[Trait("Category", "avx")]
1163+
[FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)]
11741164
public void AsciiPlusContinuationAtEndErrorAVX()
11751165
{
11761166
AsciiPlusContinuationAtEndError(SimdUnicode.UTF8.GetPointerToFirstInvalidByteAvx2);
11771167
}
11781168

1179-
[Fact]
1169+
[FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)]
11801170
public void AsciiPlusContinuationAtEndErrorAvx2()
11811171
{
11821172
AsciiPlusContinuationAtEndError(SimdUnicode.UTF8.GetPointerToFirstInvalidByteAvx2);
11831173
}
11841174

1185-
public void SurrogateErrorTest(Utf8ValidationDelegate utf8ValidationDelegate)
1175+
private void SurrogateErrorTest(Utf8ValidationDelegate utf8ValidationDelegate)
11861176
{
11871177
foreach (int outputLength in outputLengths)
11881178
{
@@ -1249,15 +1239,14 @@ public void SurrogateErrorTestScalar()
12491239
// SurrogateErrorTest(SimdUnicode.UTF8.GetPointerToFirstInvalidByteArm64);
12501240
// }
12511241

1252-
[Fact]
1253-
[Trait("Category", "avx")]
1242+
[FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)]
12541243
public void SurrogateErrorTestAVX()
12551244
{
12561245
SurrogateErrorTest(SimdUnicode.UTF8.GetPointerToFirstInvalidByteAvx2);
12571246
}
12581247

12591248

1260-
public void BruteForceTest(Utf8ValidationDelegate utf8ValidationDelegate)
1249+
private void BruteForceTest(Utf8ValidationDelegate utf8ValidationDelegate)
12611250
{
12621251
foreach (int outputLength in outputLengths)
12631252
{
@@ -1336,8 +1325,7 @@ public void BruteForceTestScalar()
13361325
// BruteForceTest(SimdUnicode.UTF8.GetPointerToFirstInvalidByteArm64);
13371326
// }
13381327

1339-
[Fact]
1340-
[Trait("Category", "avx")]
1328+
[FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)]
13411329
public void BruteForceTestAVX()
13421330
{
13431331
BruteForceTest(SimdUnicode.UTF8.GetPointerToFirstInvalidByteAvx2);

0 commit comments

Comments
 (0)