Skip to content

Commit 6bb830c

Browse files
TurnerjTurnerj
authored
Merge pull request #38 from TurnerSoftware/cancellation-token-support
Add cancellation token support
2 parents a26ecc6 + 03ecdf5 commit 6bb830c

File tree

7 files changed

+59
-9
lines changed

7 files changed

+59
-9
lines changed

‎src/TurnerSoftware.RobotsExclusionTools/IRobotsFileParser.cs

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,14 +2,15 @@
22
using System.Collections.Generic;
33
using System.IO;
44
using System.Text;
5+
using System.Threading;
56
using System.Threading.Tasks;
67

78
namespace TurnerSoftware.RobotsExclusionTools
89
{
910
public interface IRobotsFileParser
1011
{
1112
RobotsFile FromString(string robotsText, Uri baseUri);
12-
Task<RobotsFile> FromUriAsync(Uri robotsUri);
13-
Task<RobotsFile> FromStreamAsync(Stream stream, Uri baseUri);
13+
Task<RobotsFile> FromUriAsync(Uri robotsUri, CancellationToken cancellationToken = default);
14+
Task<RobotsFile> FromStreamAsync(Stream stream, Uri baseUri, CancellationToken cancellationToken = default);
1415
}
1516
}

‎src/TurnerSoftware.RobotsExclusionTools/RobotsFileParser.cs

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
using System.Collections.Generic;
99
using System.Net.Http;
1010
using System.Net;
11+
using System.Threading;
1112

1213
namespace TurnerSoftware.RobotsExclusionTools
1314
{
@@ -47,12 +48,12 @@ public RobotsFile FromString(string robotsText, Uri baseUri)
4748
}
4849
}
4950

50-
public async Task<RobotsFile> FromUriAsync(Uri robotsUri)
51+
public async Task<RobotsFile> FromUriAsync(Uri robotsUri, CancellationToken cancellationToken = default)
5152
{
5253
var baseUri = new Uri(robotsUri.GetLeftPart(UriPartial.Authority));
5354
robotsUri = new UriBuilder(robotsUri) { Path = "/robots.txt" }.Uri;
5455

55-
using (var response = await HttpClient.GetAsync(robotsUri))
56+
using (var response = await HttpClient.GetAsync(robotsUri, cancellationToken))
5657
{
5758
if (response.StatusCode == HttpStatusCode.NotFound)
5859
{
@@ -66,18 +67,18 @@ public async Task<RobotsFile> FromUriAsync(Uri robotsUri)
6667
{
6768
using (var stream = await response.Content.ReadAsStreamAsync())
6869
{
69-
return await FromStreamAsync(stream, baseUri);
70+
return await FromStreamAsync(stream, baseUri, cancellationToken);
7071
}
7172
}
7273
}
7374

7475
return RobotsFile.AllowAllRobots(baseUri);
7576
}
7677

77-
public async Task<RobotsFile> FromStreamAsync(Stream stream, Uri baseUri)
78+
public async Task<RobotsFile> FromStreamAsync(Stream stream, Uri baseUri, CancellationToken cancellationToken = default)
7879
{
7980
var streamReader = new StreamReader(stream);
80-
var tokens = await Tokenizer.TokenizeAsync(streamReader);
81+
var tokens = await Tokenizer.TokenizeAsync(streamReader, cancellationToken);
8182
return FromTokens(tokens, baseUri);
8283
}
8384

‎src/TurnerSoftware.RobotsExclusionTools/Tokenization/ITokenizer.cs

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
using System.Collections.Generic;
33
using System.IO;
44
using System.Text;
5+
using System.Threading;
56
using System.Threading.Tasks;
67

78
namespace TurnerSoftware.RobotsExclusionTools.Tokenization
@@ -10,6 +11,6 @@ public interface ITokenizer
1011
{
1112
IEnumerable<Token> Tokenize(string text);
1213
IEnumerable<Token> Tokenize(TextReader reader);
13-
Task<IEnumerable<Token>> TokenizeAsync(TextReader reader);
14+
Task<IEnumerable<Token>> TokenizeAsync(TextReader reader, CancellationToken cancellationToken = default);
1415
}
1516
}

‎src/TurnerSoftware.RobotsExclusionTools/Tokenization/TokenizerBase.cs

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
using System.IO;
44
using System.Text;
55
using System.Text.RegularExpressions;
6+
using System.Threading;
67
using System.Threading.Tasks;
78

89
namespace TurnerSoftware.RobotsExclusionTools.Tokenization
@@ -33,12 +34,13 @@ public IEnumerable<Token> Tokenize(TextReader reader)
3334
return tokens;
3435
}
3536

36-
public async Task<IEnumerable<Token>> TokenizeAsync(TextReader reader)
37+
public async Task<IEnumerable<Token>> TokenizeAsync(TextReader reader, CancellationToken cancellationToken = default)
3738
{
3839
var tokens = new List<Token>();
3940
string line;
4041
while ((line = await reader.ReadLineAsync()) != null)
4142
{
43+
cancellationToken.ThrowIfCancellationRequested();
4244
Tokenize(line, tokens);
4345
tokens.Add(Token.NewLineToken);
4446
}

‎tests/TurnerSoftware.RobotsExclusionTools.Tests/RobotsFile/RobotsFileParserTests.cs

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
using System.IO;
44
using System.Linq;
55
using System.Text;
6+
using System.Threading;
67
using System.Threading.Tasks;
78
using Microsoft.VisualStudio.TestTools.UnitTesting;
89
using TurnerSoftware.RobotsExclusionTools.Tests.TestSite;
@@ -88,5 +89,29 @@ public async Task FromStreamLoading()
8889
Assert.IsTrue(robotsFile.SiteAccessEntries.Any());
8990
}
9091
}
92+
93+
[TestMethod]
94+
public async Task FromUriLoading_Cancellation()
95+
{
96+
using (var fileStream = new FileStream("Resources/RobotsFile/NoRobots-RFC-Example.txt", FileMode.Open))
97+
{
98+
var cts = new CancellationToken(true);
99+
await Assert.ThrowsExceptionAsync<OperationCanceledException>(
100+
async () => await new RobotsFileParser().FromStreamAsync(fileStream, new Uri("http://www.example.org/"), cts)
101+
);
102+
}
103+
}
104+
105+
[TestMethod]
106+
public async Task FromStreamLoading_Cancellation()
107+
{
108+
using (var fileStream = new FileStream("Resources/RobotsFile/NoRobots-RFC-Example.txt", FileMode.Open))
109+
{
110+
var cts = new CancellationToken(true);
111+
await Assert.ThrowsExceptionAsync<OperationCanceledException>(
112+
async () => await new RobotsFileParser().FromStreamAsync(fileStream, new Uri("http://www.example.org/"), cts)
113+
);
114+
}
115+
}
91116
}
92117
}

‎tests/TurnerSoftware.RobotsExclusionTools.Tests/RobotsFile/RobotsFileTokenizationTests.cs

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,9 @@
33
using TurnerSoftware.RobotsExclusionTools.Tokenization.Tokenizers;
44
using System.IO;
55
using System.Linq;
6+
using System.Threading.Tasks;
7+
using System.Threading;
8+
using System;
69

710
namespace TurnerSoftware.RobotsExclusionTools.Tests.RobotsFile
811
{
@@ -65,5 +68,18 @@ public void InvalidFields()
6568
Assert.AreEqual(0, fieldTokens.Count());
6669
Assert.AreEqual(19, valueTokens.Count());
6770
}
71+
72+
[TestMethod]
73+
public async Task TokenizeAsyncCancellation()
74+
{
75+
using (var stream = LoadResourceStream("RobotsFile/InvalidField-Example.txt"))
76+
using (var reader = new StreamReader(stream))
77+
{
78+
var tokenizer = new RobotsFileTokenizer();
79+
await Assert.ThrowsExceptionAsync<OperationCanceledException>(
80+
async () => await tokenizer.TokenizeAsync(reader, new CancellationToken(true))
81+
);
82+
}
83+
}
6884
}
6985
}

‎tests/TurnerSoftware.RobotsExclusionTools.Tests/TestBase.cs

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,10 @@ protected string LoadResource(string name)
1313
{
1414
return File.ReadAllText($"Resources/{name}");
1515
}
16+
protected Stream LoadResourceStream(string name)
17+
{
18+
return File.OpenRead($"Resources/{name}");
19+
}
1620

1721
protected string LoadRobotsRfcFileExample()
1822
{

0 commit comments

Comments
 (0)