Add number suffixes and allow more number underscores (#481)

* Allow Rusty float literals with underscores * Implement number suffixes * Add fuzzer support and fix fuzzer-found issue * Add CHANGELOG entry * Fix grammar and tests to include (inf|NaN)_f(32|64) * Clean up integer and float parsing * Honour grammar and accept `NaNf32` instead of `NaN_f32`
ron-rs · Aug 23, 2023 · 388cfe3 · 388cfe3
1 parent 0982767
commit 388cfe3
Show file tree

Hide file tree

Showing 10 changed files with 1,130 additions and 178 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -21,6 +21,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Add a benchmark for PRs that runs over the latest fuzzer corpus ([#465](https://github.com/ron-rs/ron/pull/465))
 - Fix issue [#445](https://github.com/ron-rs/ron/issues/445) and allow parsing `+unsigned` as an unsigned int ([#479](https://github.com/ron-rs/ron/pull/479))
 - Breaking: Expand the `value::Number` enum to explicitly encode all possible number types ([#479](https://github.com/ron-rs/ron/pull/479))
+- Allow parsing floating point literals with underscores ([#481](https://github.com/ron-rs/ron/pull/481))
+- Fix issue [#241](https://github.com/ron-rs/ron/issues/241) and allow parsing numbers with explicit type suffixes, e.g. `1u8` or `-1f32` ([#481](https://github.com/ron-rs/ron/pull/481))
+- Add `number_suffixes` option to `PrettyConfig` to allow serialising numbers with their explicit type suffix, e.g. `42i32` ([#481](https://github.com/ron-rs/ron/pull/481))
 
 ## [0.8.1] - 2023-08-17
 

diff --git a/docs/grammar.md b/docs/grammar.md
@@ -47,16 +47,26 @@ value = integer | float | string | char | bool | option | list | map | tuple | s
 
 ```ebnf
 digit = "0" | "1" | "2" | "3" | "4" | "5" | "6" | "7" | "8" | "9";
-hex_digit = "A" | "a" | "B" | "b" | "C" | "c" | "D" | "d" | "E" | "e" | "F" | "f";
-unsigned = (["0", ("b" | "o")], digit, { digit | '_' } |
-             "0x", (digit | hex_digit), { digit | hex_digit | '_' });
-integer = ["+" | "-"], unsigned;
-float = ["+" | "-"], ("inf" | "NaN" | float_num);
+digit_binary = "0" | "1";
+digit_octal = "0" | "1" | "2" | "3" | "4" | "5" | "6" | "7";
+digit_hexadecimal = digit | "A" | "a" | "B" | "b" | "C" | "c" | "D" | "d" | "E" | "e" | "F" | "f";
+
+integer = ["+" | "-"], unsigned, [integer_suffix];
+integer_suffix = ("i", "u"), ("8", "16", "32", "64", "128");
+
+unsigned = unsigned_binary | unsigned_octal | unsigned_hexadecimal | unsigned_decimal;
+unsigned_binary = "0b", digit_binary, { digit_binary | "_" };
+unsigned_octal = "0o", digit_octal, { digit_octal | "_" };
+unsigned_hexadecimal = "0x", digit_hexadecimal, { digit_hexadecimal | "_" };
+unsigned_decimal = digit, { digit | "_" };
+
+float = ["+" | "-"], ("inf" | "NaN" | float_num), [float_suffix];
 float_num = (float_int | float_std | float_frac), [float_exp];
-float_int = digit, { digit };
-float_std = digit, { digit }, ".", {digit};
-float_frac = ".", digit, {digit};
-float_exp = ("e" | "E"), ["+" | "-"], digit, {digit};
+float_int = digit, { digit | "_" };
+float_std = digit, { digit | "_" }, ".", [digit, { digit | "_" }];
+float_frac = ".", digit, { digit | "_" };
+float_exp = ("e" | "E"), ["+" | "-"], { digit | "_" }, digit, { digit | "_" };
+float_suffix = "f", ("32", "64");
 ```
 
 ## String

diff --git a/fuzz/fuzz_targets/bench/lib.rs b/fuzz/fuzz_targets/bench/lib.rs
@@ -84,6 +84,8 @@ struct ArbitraryPrettyConfig {
     /// Enable compact maps, which do not insert new lines and indentation
     ///  between the entries of a struct
     compact_maps: bool,
+    /// Enable explicit number type suffixes like `1u16`
+    number_suffixes: bool,
 }
 
 fn arbitrary_ron_extensions(u: &mut Unstructured) -> arbitrary::Result<Extensions> {
@@ -102,6 +104,7 @@ impl From<ArbitraryPrettyConfig> for PrettyConfig {
             .escape_strings(arbitrary.escape_strings)
             .compact_structs(arbitrary.compact_structs)
             .compact_maps(arbitrary.compact_maps)
+            .number_suffixes(arbitrary.number_suffixes)
     }
 }
 

diff --git a/src/de/mod.rs b/src/de/mod.rs
@@ -293,10 +293,14 @@ impl<'de, 'a> de::Deserializer<'de> for &'a mut Deserializer<'de> {
             return visitor.visit_none();
         } else if self.bytes.consume("()") {
             return visitor.visit_unit();
-        } else if self.bytes.consume_ident("inf") {
+        } else if self.bytes.consume_ident("inf") || self.bytes.consume_ident("inff32") {
             return visitor.visit_f32(std::f32::INFINITY);
-        } else if self.bytes.consume_ident("NaN") {
+        } else if self.bytes.consume_ident("inff64") {
+            return visitor.visit_f64(std::f64::INFINITY);
+        } else if self.bytes.consume_ident("NaN") || self.bytes.consume_ident("NaNf32") {
             return visitor.visit_f32(std::f32::NAN);
+        } else if self.bytes.consume_ident("NaNf64") {
+            return visitor.visit_f64(std::f64::NAN);
         }
 
         // `identifier` does not change state if it fails
@@ -330,72 +334,72 @@ impl<'de, 'a> de::Deserializer<'de> for &'a mut Deserializer<'de> {
     where
         V: Visitor<'de>,
     {
-        visitor.visit_i8(self.bytes.signed_integer()?)
+        visitor.visit_i8(self.bytes.integer()?)
     }
 
     fn deserialize_i16<V>(self, visitor: V) -> Result<V::Value>
     where
         V: Visitor<'de>,
     {
-        visitor.visit_i16(self.bytes.signed_integer()?)
+        visitor.visit_i16(self.bytes.integer()?)
     }
 
     fn deserialize_i32<V>(self, visitor: V) -> Result<V::Value>
     where
         V: Visitor<'de>,
     {
-        visitor.visit_i32(self.bytes.signed_integer()?)
+        visitor.visit_i32(self.bytes.integer()?)
     }
 
     fn deserialize_i64<V>(self, visitor: V) -> Result<V::Value>
     where
         V: Visitor<'de>,
     {
-        visitor.visit_i64(self.bytes.signed_integer()?)
+        visitor.visit_i64(self.bytes.integer()?)
     }
 
     #[cfg(feature = "integer128")]
     fn deserialize_i128<V>(self, visitor: V) -> Result<V::Value>
     where
         V: Visitor<'de>,
     {
-        visitor.visit_i128(self.bytes.signed_integer()?)
+        visitor.visit_i128(self.bytes.integer()?)
     }
 
     fn deserialize_u8<V>(self, visitor: V) -> Result<V::Value>
     where
         V: Visitor<'de>,
     {
-        visitor.visit_u8(self.bytes.unsigned_integer()?)
+        visitor.visit_u8(self.bytes.integer()?)
     }
 
     fn deserialize_u16<V>(self, visitor: V) -> Result<V::Value>
     where
         V: Visitor<'de>,
     {
-        visitor.visit_u16(self.bytes.unsigned_integer()?)
+        visitor.visit_u16(self.bytes.integer()?)
     }
 
     fn deserialize_u32<V>(self, visitor: V) -> Result<V::Value>
     where
         V: Visitor<'de>,
     {
-        visitor.visit_u32(self.bytes.unsigned_integer()?)
+        visitor.visit_u32(self.bytes.integer()?)
     }
 
     fn deserialize_u64<V>(self, visitor: V) -> Result<V::Value>
     where
         V: Visitor<'de>,
     {
-        visitor.visit_u64(self.bytes.unsigned_integer()?)
+        visitor.visit_u64(self.bytes.integer()?)
     }
 
     #[cfg(feature = "integer128")]
     fn deserialize_u128<V>(self, visitor: V) -> Result<V::Value>
     where
         V: Visitor<'de>,
     {
-        visitor.visit_u128(self.bytes.unsigned_integer()?)
+        visitor.visit_u128(self.bytes.integer()?)
     }
 
     fn deserialize_f32<V>(self, visitor: V) -> Result<V::Value>

diff --git a/src/error.rs b/src/error.rs
@@ -52,6 +52,10 @@ pub enum Error {
     InvalidEscape(&'static str),
 
     IntegerOutOfBounds,
+    InvalidIntegerDigit {
+        digit: char,
+        base: u8,
+    },
 
     NoSuchExtension(String),
 
@@ -152,13 +156,16 @@ impl fmt::Display for Error {
             Error::ExpectedIdentifier => f.write_str("Expected identifier"),
             Error::InvalidEscape(s) => f.write_str(s),
             Error::IntegerOutOfBounds => f.write_str("Integer is out of bounds"),
+            Error::InvalidIntegerDigit { digit, base } => {
+                write!(f, "Invalid digit {:?} for base {} integers", digit, base)
+            },
             Error::NoSuchExtension(ref name) => {
                 write!(f, "No RON extension named {}", Identifier(name))
             }
             Error::Utf8Error(ref e) => fmt::Display::fmt(e, f),
             Error::UnclosedBlockComment => f.write_str("Unclosed block comment"),
             Error::UnderscoreAtBeginning => {
-                f.write_str("Unexpected leading underscore in an integer")
+                f.write_str("Unexpected leading underscore in a number")
             }
             Error::UnexpectedByte(ref byte) => write!(f, "Unexpected byte {:?}", byte),
             Error::TrailingCharacters => f.write_str("Non-whitespace trailing characters"),
@@ -256,7 +263,7 @@ impl fmt::Display for Error {
     }
 }
 
-#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
 pub struct Position {
     pub line: usize,
     pub col: usize,