Use offset to traverse matching

Removes unnecessary string allocations when detecting a match

Author: Turnerj

src/TurnerSoftware.RobotsExclusionTools/Tokenization/TokenDefinition.cs

@@ -16,15 +16,15 @@ public TokenDefinition(TokenType tokenType, string regex)
 			TokenType = tokenType;
 		}
 
-		public TokenMatch Match(string input)
+		public TokenMatch Match(string input, int offset = 0)
 		{
-			var match = Regex.Match(input);
+			var match = Regex.Match(input, offset);
 			if (match.Success)
 			{
 				return new TokenMatch
 				{
 					IsMatch = true,
-					RemainingText = input.Substring(match.Length),
+					MatchLength = match.Length,
 					TokenType = TokenType,
 					Value = match.Value
 				};

src/TurnerSoftware.RobotsExclusionTools/Tokenization/TokenMatch.cs

@@ -9,6 +9,6 @@ public class TokenMatch
 		public bool IsMatch { get; set; }
 		public TokenType TokenType { get; set; }
 		public string Value { get; set; }
-		public string RemainingText { get; set; }
+		public int MatchLength { get; set; }
 	}
 }

src/TurnerSoftware.RobotsExclusionTools/Tokenization/TokenizerBase.cs

@@ -17,19 +17,20 @@ public abstract class TokenizerBase : ITokenizer
 		public IEnumerable<Token> Tokenize(string text)
 		{
 			var tokens = new List<Token>();
-			var remainingText = text;
+			var offset = 0;
+			var numberOfChars = text.Length;
 
-			while (!string.IsNullOrWhiteSpace(remainingText))
+			while (offset < numberOfChars)
 			{
-				var match = FindMatch(remainingText);
+				var match = FindMatch(text, offset);
 				if (match.IsMatch)
 				{
 					tokens.Add(new Token(match.TokenType, match.Value));
-					remainingText = match.RemainingText;
+					offset += match.MatchLength;
 				}
 				else
 				{
-					remainingText = remainingText.Substring(1);
+					offset++;
 				}
 			}
 
@@ -60,11 +61,11 @@ public async Task<IEnumerable<Token>> TokenizeAsync(TextReader reader)
 			return tokens;
 		}
 
-		private TokenMatch FindMatch(string text)
+		private TokenMatch FindMatch(string text, int offset)
 		{
 			foreach (var tokenDefinition in GetTokenDefinitions())
 			{
-				var match = tokenDefinition.Match(text);
+				var match = tokenDefinition.Match(text, offset);
 				if (match.IsMatch)
 				{
 					return match;

src/TurnerSoftware.RobotsExclusionTools/Tokenization/Tokenizers/RobotsFileTokenizer.cs

@@ -33,11 +33,11 @@ public class RobotsFileTokenizer : TokenizerBase
 			//This can be expressed as the following:
 			//\x21\x23-\x27\x2a\x2b\x2d\x2e\x41-\x5a\x5e-\x7a\x7c\x7e
 
-			new TokenDefinition(TokenType.Comment, @"^#[^\x0A\x0D]*"),
-			new TokenDefinition(TokenType.Field, @"^[\x21\x23-\x27\x2a\x2b\x2d\x2e\x41-\x5a\x5e-\x7a\x7c\x7e]+(?=:[ ])"),
-			new TokenDefinition(TokenType.FieldValueDelimiter, "^:[ ]"),
-			new TokenDefinition(TokenType.Value, @"^[^\x0A\x0D#]+"),
-			new TokenDefinition(TokenType.NewLine, @"^\x0D?\x0A")
+			new TokenDefinition(TokenType.Comment, @"\G#[^\x0A\x0D]*"),
+			new TokenDefinition(TokenType.Field, @"\G[\x21\x23-\x27\x2a\x2b\x2d\x2e\x41-\x5a\x5e-\x7a\x7c\x7e]+(?=:[ ])"),
+			new TokenDefinition(TokenType.FieldValueDelimiter, @"\G:[ ]"),
+			new TokenDefinition(TokenType.Value, @"\G[^\x0A\x0D#]+"),
+			new TokenDefinition(TokenType.NewLine, @"\G\x0D?\x0A")
 		};
 
 		protected override IEnumerable<TokenDefinition> GetTokenDefinitions()

src/TurnerSoftware.RobotsExclusionTools/Tokenization/Tokenizers/RobotsPageTokenizer.cs

@@ -9,11 +9,11 @@ public class RobotsPageTokenizer : TokenizerBase
 		private static readonly IEnumerable<TokenDefinition> TokenDefinitions = new []
 		{
 			//Based on the same character restriction rules as the RobotsFileTokenizer
-			new TokenDefinition(TokenType.Field, @"^[\x21\x23-\x27\x2a\x2b\x2d\x2e\x41-\x5a\x5e-\x7a\x7c\x7e]+(?=:[ ])"),
-			new TokenDefinition(TokenType.FieldValueDelimiter, "^:[ ]"),
-			new TokenDefinition(TokenType.Value, @"^[^\x0A\x0D#,]+"),
-			new TokenDefinition(TokenType.NewLine, @"^\x0D?\x0A"),
-			new TokenDefinition(TokenType.ValueDelimiter, "^,[ ]+")
+			new TokenDefinition(TokenType.Field, @"\G[\x21\x23-\x27\x2a\x2b\x2d\x2e\x41-\x5a\x5e-\x7a\x7c\x7e]+(?=:[ ])"),
+			new TokenDefinition(TokenType.FieldValueDelimiter, @"\G:[ ]"),
+			new TokenDefinition(TokenType.Value, @"\G[^\x0A\x0D#,]+"),
+			new TokenDefinition(TokenType.NewLine, @"\G\x0D?\x0A"),
+			new TokenDefinition(TokenType.ValueDelimiter, @"\G,[ ]+")
 		};
 
 		protected override IEnumerable<TokenDefinition> GetTokenDefinitions()