richer consume_time validator

Author: d1manson

README.md

@@ -52,7 +52,7 @@ The supported types are inspired by - but not a perfect match to - BigQuery data
 - `INT64` - a json number without any exponent, between int64 min and max.
 - `DECIMAL_29_9` - a json number without an exponent, with up to 29 digits before the decimal point, and up to 9 after (aka BigQuery `NUMERIC`, with default decimal point position)
 - `DATE`* - date as a string, without a timezone, as `YYYY-MM-DD`, or `YYYY/MM/DD` or `YYYY.MM.DD`.
-- `TIME`* - time as a string, without a timezone, as `HH:MM:SS` (fractional seconds not supported currently)
+- `TIME`* - time as a string, without a timezone. `HH:MM[:SS[.SSSSSS]]`
 - `DATETIME`* - date and time as a string, without a timezone, as `YYYY-MM-DDTHH:MM::SS` (in the date part, `-` can be swapped for `/`, `.`).
 - `BYTES` - a base64 string.
 - `STRUCT` - a sub schema. In this case you need to provide a `"fields": [...]` property in the schema definition, with a list of sub fields. You can nest arbitrarily deeply, and/or use `REPEATED` mode if needed.

benches/micro_util_bench.rs

@@ -492,6 +492,25 @@ fn criterion_benchmark(c: &mut Criterion) {
     group.finish();
 
 
+
+    let mut group = c.benchmark_group("consume_time");
+    // build 10k int/null values (we don't include valid json numbers that are invalid ints here)
+    let mut values: Vec<u8pOwned> = Vec::with_capacity(10000);
+    for _ in 0..values.capacity() {
+        if rng.random_bool(0.1){
+            values.push(u8pOwned::from("null"));
+        } if rng.random_bool(0.4) {
+            values.push(u8pOwned::from("\"23:11\"  "));
+        } if rng.random_bool(0.5) {
+            values.push(u8pOwned::from("\"23:11:12.01\" "));
+        } else {
+            values.push(u8pOwned::from("\"23:11:12.0111\"  "));
+        }
+    }
+    let mut value_iter = values.iter().cycle(); 
+    group.bench_function("simd", |b| b.iter(|| micro_util::consume_time( black_box(&value_iter.next().unwrap().as_borrowed()))));
+    group.finish();
+
 }
 
 criterion_group!(benches, criterion_benchmark);

src/micro_util.rs

@@ -390,6 +390,33 @@ pub fn consume_decimal_29_9(json: &u8p) -> usize {
 
 }
 
+
+
+/// This assumes the json slice is the start of a spec-compliant value (not neccessarily a time string, but definitely compliant value).
+/// For a string of the form: `HH:MM[:SS[.SSSSSS]]` it returns the number of bytes, including the start and end double quotes, otherwise zero.
+pub fn consume_time(json: &u8p) -> usize {
+    let lower = u8x16::from(*b"\"00:00:00.000000");
+    let upper = u8x16::from(*b"\"29:59:59.999999");
+
+    let data = json.initial_lane::<16, 0>();
+    let matched  =  data.simd_le(upper) & data.simd_ge(lower);
+    let mut ret = matched.to_bitmask().trailing_ones() as usize;
+    let next_char = json.raw_u8s().get(ret).unwrap_or(&0);
+    ret += 1; // add closing quote
+
+    let check1= *next_char == b'"';
+    let check2 = (ret == "\"00:00\"".len()) | (ret >= b"\"00:00:00\"".len());
+    let check3 = (json.raw_u8s()[1] < b'2') | (json.raw_u8s()[2] <= b'3');
+
+    if check1 && check2 && check3 {
+        return ret;
+    } else {
+        return 0;
+    }
+}
+
+
+
 /// Assumes json is valid, and is the start of a value of some kind. If it is the start of an array, it
 /// finds the matching end of the array, and similarlly for start/end of objects, returning the number of
 /// bytes needed to get to the end. For true/false, strings and numbers it will return the byte count (by making
@@ -682,6 +709,30 @@ mod tests {
         assert_eq!(consume_punct::<b'}', true>(&u8p!(b"}\t } ")), 3);
     }
 
+    #[test]
+    fn test_consume_time(){
+        assert_eq!(consume_time(&u8p!(b"null")), 0);
+        assert_eq!(consume_time(&u8p!(b"true")), 0);
+        assert_eq!(consume_time(&u8p!(b"false")), 0);
+        assert_eq!(consume_time(&u8p!(b"1")), 0);
+        assert_eq!(consume_time(&u8p!(b"-1")), 0);
+        assert_eq!(consume_time(&u8p!(b"[false]")), 0);
+        assert_eq!(consume_time(&u8p!(b"{false}")), 0);
+        assert_eq!(consume_time(&u8p!(b"\"hello\"")), 0);
+
+
+        assert_eq!(consume_time(&u8p!(b"\"12:45\"   ")), 7);
+        assert_eq!(consume_time(&u8p!(b"\"12:45:08\"   ")), 10);
+        assert_eq!(consume_time(&u8p!(b"\"23:45:08\"   ")), 10);
+        assert_eq!(consume_time(&u8p!(b"\"33:45:08\"   ")), 0); // 33 hrs in a day..no
+        assert_eq!(consume_time(&u8p!(b"\"24:45:08\"   ")), 0); // 24 o'clock is the next day, so no
+        
+        assert_eq!(consume_time(&u8p!(b"\"12:45:08.0123\"   ")), 15);
+        assert_eq!(consume_time(&u8p!(b"\"12:45:08.012345\"   ")), 17);
+        assert_eq!(consume_time(&u8p!(b"\"12:45:08.0123456\"  ")), 0);
+        assert_eq!(consume_time(&u8p!(b"\"12:45:08x0123\"  ")), 0);
+    }
+
     #[test]
     fn test_consume_within_range() {
         pub const QUOTED_DATE_LOWER: &[u8; 12] = b"\"0000-00-00\"";

src/validate.rs

@@ -206,10 +206,7 @@ pub fn validate<'a, 'b>(root_schema: &'a AdaptivePrefixMap<Field>, max_field_idx
                             // missing: check = ((json[6] < '1') | (json[7] <= '2')) &  (json[9] < '3') | (json[10] <= '1') ...but still allows invalid dates (do we need to deal with months, what about leap years?!)
                         },
                         FieldType::TIME => {
-                            const QUOTED_TIME_LOWER: &[u8; 10] = b"\"00:00:00\"";
-                            const QUOTED_TIME_UPPER: &[u8; 10] = b"\"29:59:59\"";
-                            micro_util::consume_within_range(&json_offset, QUOTED_TIME_LOWER, QUOTED_TIME_UPPER) 
-                            // missing: check = (json[1] < '2') | (json[2] <= '3')
+                            micro_util::consume_time(&json_offset)
                         },
                         FieldType::DATETIME => {
                             const QUOTED_DATETIME_LOWER: &[u8; 21] = b"\"0000-00-00T00:00:00\"";