Skip to content

Commit 90f0111

Browse files
committed
Emit Warning for Byte String Chars > 127 but < 256
>= 256 (more than 1 byte) are already an error ```fsharp let _ = "ä"B // ^^^^ // This byte array literal contains 1 non-ASCII characters. All characters should be < 128y. // Note: In future F# versions this Warning might be promoted to Error! ``` Note: Issue with trigraph and > 255: ```fsharp let _ = "\973"B // ^^^^ // Warning: '\973' is not a valid character literal. // Note: Currently the value is wrapped around byte range to '\169' [...] // // ^^^^^^^ // Warning: This byte array literal contains 1 non-ASCII characters. ``` -> Two warnings for same trigraph: * Warning for trigraph `\973` > 255. Detected while parsing trigraph. Don't know yet if String or Byte String. * warning spans just trigraph * Warning for wrapped value `\169` > 127. Detected while checking Byte String for correct values. Don't have any infos about value (range & notation) any more. * warning spans full byte string -> Each Warning is emitted in a different parsing steps. -> No easy way to reduce to just one warning. However: Both warnings are correct and warn about a different issues (albeit in same value) And: should get automatically reduced once out-of-trigraph-range gets promoted to error **Enhancement**: Count number of invalid chars (2-bytes, >128) in byte array ```fsharp let _ = "ΩäΩüΩ"B ``` * prev: `This byte array literal contains characters that do not encode as a single byte` * now: `This byte array literal contains 3 characters that do not encode as a single byte` && `This byte array literal contains 2 non-ASCII characters. All characters should be < 128y.` Note: When checking valid bytes in Byte String, there's no direct mapping to range and notation any more. -> cannot highlight exact byte string part, but only full byte string. * Possible Enhancement?: List invalid chars in a certain notation? For example in `\u` notation. * Pro: * Gives an idea which chars are incorrect * Con: * Might be difficult to process by user because output might be different notation than written in string * Might be a long list with invalid chars
1 parent 9ce73bb commit 90f0111

File tree

18 files changed

+177
-46
lines changed

18 files changed

+177
-46
lines changed

src/Compiler/FSComp.txt

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1019,7 +1019,7 @@ lexfltSeparatorTokensOfPatternMatchMisaligned,"The '|' tokens separating rules o
10191019
# -----------------------------------------------------------------------------
10201020
lexCharNotAllowedInOperatorNames,"'%s' is not permitted as a character in operator names and is reserved for future use"
10211021
lexUnexpectedChar,"Unexpected character '%s'"
1022-
1140,lexByteArrayCannotEncode,"This byte array literal contains characters that do not encode as a single byte"
1022+
1140,lexByteArrayCannotEncode,"This byte array literal contains %d characters that do not encode as a single byte"
10231023
1141,lexIdentEndInMarkReserved,"Identifiers followed by '%s' are reserved for future use"
10241024
1142,lexOutsideEightBitSigned,"This number is outside the allowable range for 8-bit signed integers"
10251025
1143,lexOutsideEightBitSignedHex,"This number is outside the allowable range for hexadecimal 8-bit signed integers"
@@ -1132,6 +1132,7 @@ lexIfOCaml,"IF-FSHARP/IF-CAML regions are no longer supported"
11321132
1250,lexTooManyPercentsInTripleQuote,"The interpolated triple quoted string literal does not start with enough '$' characters to allow this many consecutive '%%' characters."
11331133
1251,lexExtendedStringInterpolationNotSupported,"Extended string interpolation is not supported in this version of F#."
11341134
1252,lexInvalidCharLiteralInString,"'%s' is not a valid character literal.\nNote: Currently the value is wrapped around byte range to '%s'. In a future F# version this Warning will be promoted to Error!"
1135+
1253,lexByteArrayOutisdeAscii,"This byte array literal contains %d non-ASCII characters. All characters should be < 128y."
11351136
# reshapedmsbuild.fs
11361137
1300,toolLocationHelperUnsupportedFrameworkVersion,"The specified .NET Framework version '%s' is not supported. Please specify a value from the enumeration Microsoft.Build.Utilities.TargetDotNetFrameworkVersion."
11371138
# -----------------------------------------------------------------------------

src/Compiler/SyntaxTree/LexHelpers.fs

Lines changed: 19 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -222,16 +222,31 @@ let addUnicodeChar buf c = addIntChar buf (int c)
222222

223223
let addByteChar buf (c: char) = addIntChar buf (int32 c % 256)
224224

225+
type LargerThanOneByte = int
226+
type LargerThan127ButInsideByte = int
225227
/// Sanity check that high bytes are zeros. Further check each low byte <= 127
226-
let stringBufferIsBytes (buf: ByteBuffer) =
228+
let errorsInByteStringBuffer (buf: ByteBuffer) =
227229
let bytes = buf.AsMemory()
228-
let mutable ok = true
230+
assert(bytes.Length % 2 = 0)
231+
232+
// Enhancement?: return faulty values?
233+
// But issue: we don't know range of values -> no direct mapping from value to range & notation
234+
235+
// values with high byte <> 0
236+
let mutable largerThanOneByteCount = 0
237+
// values with high byte = 0, but low byte > 127
238+
let mutable largerThan127ButSingleByteCount = 0
229239

230240
for i = 0 to bytes.Length / 2 - 1 do
231241
if bytes.Span[i * 2 + 1] <> 0uy then
232-
ok <- false
242+
largerThanOneByteCount <- largerThanOneByteCount + 1
243+
elif bytes.Span[i * 2] > 127uy then
244+
largerThan127ButSingleByteCount <- largerThan127ButSingleByteCount + 1
233245

234-
ok
246+
if largerThanOneByteCount + largerThan127ButSingleByteCount > 0 then
247+
Some (largerThanOneByteCount, largerThan127ButSingleByteCount)
248+
else
249+
None
235250

236251
let newline (lexbuf: LexBuffer<_>) = lexbuf.EndPos <- lexbuf.EndPos.NextLine
237252

src/Compiler/SyntaxTree/LexHelpers.fsi

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,9 @@ val stringBufferAsString: ByteBuffer -> string
9797

9898
val stringBufferAsBytes: ByteBuffer -> byte[]
9999

100-
val stringBufferIsBytes: ByteBuffer -> bool
100+
type LargerThanOneByte = int
101+
type LargerThan127ButInsideByte = int
102+
val errorsInByteStringBuffer: ByteBuffer -> Option<LargerThanOneByte * LargerThan127ButInsideByte>
101103

102104
val newline: Lexing.LexBuffer<'a> -> unit
103105

src/Compiler/lex.fsl

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -133,10 +133,14 @@ let startString args (lexbuf: UnicodeLexing.Lexbuf) =
133133
if kind.IsInterpolated then
134134
fail args lexbuf (FSComp.SR.lexByteStringMayNotBeInterpolated()) ()
135135
BYTEARRAY (Lexhelp.stringBufferAsBytes buf, synByteStringKind, cont)
136-
elif Lexhelp.stringBufferIsBytes buf then
137-
BYTEARRAY (Lexhelp.stringBufferAsBytes buf, synByteStringKind, cont)
138136
else
139-
fail args lexbuf (FSComp.SR.lexByteArrayCannotEncode()) ()
137+
match Lexhelp.errorsInByteStringBuffer buf with
138+
| Some (largerThanOneByte, largerThan127) ->
139+
if largerThanOneByte > 0 then
140+
fail args lexbuf (FSComp.SR.lexByteArrayCannotEncode(largerThanOneByte)) ()
141+
if largerThan127 > 0 then
142+
warning (Error(FSComp.SR.lexByteArrayOutisdeAscii(largerThan127), lexbuf.LexemeRange))
143+
| None -> ()
140144
BYTEARRAY (Lexhelp.stringBufferAsBytes buf, synByteStringKind, cont)
141145
elif kind.IsInterpolated then
142146
let s = Lexhelp.stringBufferAsString buf

src/Compiler/xlf/FSComp.txt.cs.xlf

Lines changed: 7 additions & 2 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

src/Compiler/xlf/FSComp.txt.de.xlf

Lines changed: 7 additions & 2 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

src/Compiler/xlf/FSComp.txt.es.xlf

Lines changed: 7 additions & 2 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

src/Compiler/xlf/FSComp.txt.fr.xlf

Lines changed: 7 additions & 2 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

src/Compiler/xlf/FSComp.txt.it.xlf

Lines changed: 7 additions & 2 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

src/Compiler/xlf/FSComp.txt.ja.xlf

Lines changed: 7 additions & 2 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)