Programmatically agree to new ToS and GDPR

- Programmatically agree to the new ToS and GDPR. - Load ToS and GDPR cookie for regular tumblr blog site crawling (#234).
johanneszab · May 24, 2018 · e5719b2 · e5719b2
1 parent ad1f185
commit e5719b2
Show file tree

Hide file tree

Showing 3 changed files with 4 additions and 5 deletions.
diff --git a/src/TumblThree/SharedAssemblyInfo.cs b/src/TumblThree/SharedAssemblyInfo.cs
@@ -12,5 +12,5 @@
 
 [assembly: ComVisible(false)]
 [assembly: NeutralResourcesLanguage("en-US", UltimateResourceFallbackLocation.MainAssembly)]
-[assembly: AssemblyVersion("1.0.8.49")]
-[assembly: AssemblyFileVersion("1.0.8.49")]
+[assembly: AssemblyVersion("1.0.8.50")]
+[assembly: AssemblyFileVersion("1.0.8.50")]
diff --git a/src/TumblThree/TumblThree.Applications/Crawler/TumblrBlogCrawler.cs b/src/TumblThree/TumblThree.Applications/Crawler/TumblrBlogCrawler.cs
@@ -230,9 +230,9 @@ private async Task<string> GetApiPageAsync(int pageId)
             if (shellService.Settings.LimitConnections)
             {
                 crawlerService.Timeconstraint.Acquire();
-                return await RequestDataAsync(url).TimeoutAfter(shellService.Settings.TimeOut);
+                return await RequestDataAsync(url, "https://www.tumblr.com/", "https://" + blog.Name.Replace("+", "-") + ".tumblr.com").TimeoutAfter(shellService.Settings.TimeOut);
             }
-            return await RequestDataAsync(url).TimeoutAfter(shellService.Settings.TimeOut);
+            return await RequestDataAsync(url, "https://www.tumblr.com/", "https://" + blog.Name.Replace("+", "-") + ".tumblr.com").TimeoutAfter(shellService.Settings.TimeOut);
         }
 
         private async Task UpdateTotalPostCountAsync()

diff --git a/src/TumblThree/TumblThree.Applications/Crawler/TumblrTagSearchCrawler.cs b/src/TumblThree/TumblThree.Applications/Crawler/TumblrTagSearchCrawler.cs
@@ -1,5 +1,4 @@
 using System;
-using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.ComponentModel.Composition;
 using System.Globalization;