fix hostname tokenizing of newline, carriage return and tab

Author: edgul

src/testdata/urlpatterntestdata.json

@@ -2494,15 +2494,33 @@
   },
   {
     "pattern": [{ "hostname": "bad\nhostname" }],
-    "expected_obj": "error"
+    "inputs": [{ "hostname": "badhostname" }],
+    "expected_obj": {
+      "hostname": "badhostname"
+    },
+    "expected_match": {
+      "hostname": { "input": "badhostname", "groups": {} }
+    }
   },
   {
     "pattern": [{ "hostname": "bad\rhostname" }],
-    "expected_obj": "error"
+    "inputs": [{ "hostname": "badhostname" }],
+    "expected_obj": {
+      "hostname": "badhostname"
+    },
+    "expected_match": {
+      "hostname": { "input": "badhostname", "groups": {} }
+    }
   },
   {
     "pattern": [{ "hostname": "bad\thostname" }],
-    "expected_obj": "error"
+    "inputs": [{ "hostname": "badhostname" }],
+    "expected_obj": {
+      "hostname": "badhostname"
+    },
+    "expected_match": {
+      "hostname": { "input": "badhostname", "groups": {} }
+    }
   },
   {
     "pattern": [{}],

src/tokenizer.rs

@@ -160,6 +160,13 @@ pub fn tokenize(
       );
       continue;
     }
+    if tokenizer.code_point == Some('\n') ||
+       tokenizer.code_point == Some('\r') ||
+       tokenizer.code_point == Some('\t') {
+      // ignore newline, carriage return and tab
+      tokenizer.index = tokenizer.next_index;
+      continue;
+    }
     if tokenizer.code_point == Some('{') {
       tokenizer.add_token_with_default_pos_and_len(TokenType::Open);
       continue;