#273 Improves database loading

- Correctly handles SerializationExceptions in the IFiles databases. Previously, the exception wasn't handled at all and would stuck the crawler. - Displays a list of blogs that failed to deserialize at startup instead of stopping at the first blog. - Continues to load and add all remaining successfully deserialized blogs to the manager.
johanneszab · Sep 28, 2018 · 7c9120f · 7c9120f
1 parent 1be5a52
commit 7c9120f
Show file tree

Hide file tree

Showing 3 changed files with 67 additions and 50 deletions.
diff --git a/src/TumblThree/TumblThree.Applications/Controllers/ManagerController.cs b/src/TumblThree/TumblThree.Applications/Controllers/ManagerController.cs
@@ -5,6 +5,7 @@
 using System.ComponentModel.Composition;
 using System.IO;
 using System.Linq;
+using System.Runtime.Serialization;
 using System.Threading;
 using System.Threading.Tasks;
 using System.Waf.Applications;
@@ -166,26 +167,19 @@ private async Task LoadLibraryAsync()
             managerService.BlogFiles.Clear();
             string path = Path.Combine(shellService.Settings.DownloadLocation, "Index");
 
-            try
+            if (Directory.Exists(path))
             {
-                if (Directory.Exists(path))
                 {
+                    IReadOnlyList<IBlog> files = await GetIBlogsAsync(path);
+                    foreach (IBlog file in files)
                     {
-                        IReadOnlyList<IBlog> files = await GetIBlogsAsync(path);
-                        foreach (IBlog file in files)
-                        {
-                            managerService.BlogFiles.Add(file);
-                        }
-
-                        BlogManagerFinishedLoadingLibrary?.Invoke(this, EventArgs.Empty);
+                        managerService.BlogFiles.Add(file);
                     }
+
+                    BlogManagerFinishedLoadingLibrary?.Invoke(this, EventArgs.Empty);
                 }
             }
-            catch (Exception ex)
-            {
-                Logger.Verbose("ManagerController:LoadLibrary: {0}", ex);
-                shellService.ShowError(ex, Resources.CouldNotLoadLibrary, ex.Data["Filename"]);
-            }
+
             Logger.Verbose("ManagerController.LoadLibrary:End");
         }
 
@@ -221,6 +215,7 @@ private IReadOnlyList<IBlog> GetIBlogsCore(string directory)
             Logger.Verbose("ManagerController:GetFilesCore Start");
 
             var blogs = new List<IBlog>();
+            var failedToLoadBlogs = new List<string>();
 
             string[] supportedFileTypes = Enum.GetNames(typeof(BlogTypes)).ToArray();
 
@@ -229,17 +224,31 @@ private IReadOnlyList<IBlog> GetIBlogsCore(string directory)
                             !fileName.Contains("_files")))
             {
                 //TODO: Refactor
-                if (filename.EndsWith(BlogTypes.tumblr.ToString()))
-                    blogs.Add(new TumblrBlog().Load(filename));
-                if (filename.EndsWith(BlogTypes.tmblrpriv.ToString()))
-                    blogs.Add(new TumblrHiddenBlog().Load(filename));
-                if (filename.EndsWith(BlogTypes.tlb.ToString()))
-                    blogs.Add(new TumblrLikedByBlog().Load(filename));
-                if (filename.EndsWith(BlogTypes.tumblrsearch.ToString()))
-                    blogs.Add(new TumblrSearchBlog().Load(filename));
-                if (filename.EndsWith(BlogTypes.tumblrtagsearch.ToString()))
-                    blogs.Add(new TumblrTagSearchBlog().Load(filename));
+                try
+                {
+                    if (filename.EndsWith(BlogTypes.tumblr.ToString()))
+                        blogs.Add(new TumblrBlog().Load(filename));
+                    if (filename.EndsWith(BlogTypes.tmblrpriv.ToString()))
+                        blogs.Add(new TumblrHiddenBlog().Load(filename));
+                    if (filename.EndsWith(BlogTypes.tlb.ToString()))
+                        blogs.Add(new TumblrLikedByBlog().Load(filename));
+                    if (filename.EndsWith(BlogTypes.tumblrsearch.ToString()))
+                        blogs.Add(new TumblrSearchBlog().Load(filename));
+                    if (filename.EndsWith(BlogTypes.tumblrtagsearch.ToString()))
+                        blogs.Add(new TumblrTagSearchBlog().Load(filename));
+                }
+                catch (SerializationException ex)
+                {
+                    failedToLoadBlogs.Add(ex.Data["Filename"].ToString());
+                }
+            }
+            if (failedToLoadBlogs.Any())
+            {
+                string failedBlogNames = failedToLoadBlogs.Aggregate((a, b) => a + ", " + b);
+                Logger.Verbose("ManagerController:LoadLibrary: {0}", failedBlogNames);
+                shellService.ShowError(new SerializationException(), Resources.CouldNotLoadLibrary, failedBlogNames);
             }
+
             Logger.Verbose("ManagerController.GetFilesCore End");
 
             return blogs;
@@ -255,6 +264,7 @@ private IReadOnlyList<IFiles> GetIFilesCore(string directory)
             Logger.Verbose("ManagerController:GetFilesCore Start");
 
             var blogs = new List<IFiles>();
+            var failedToLoadBlogs = new List<string>();
 
             string[] supportedFileTypes = Enum.GetNames(typeof(BlogTypes)).ToArray();
 
@@ -263,8 +273,26 @@ private IReadOnlyList<IFiles> GetIFilesCore(string directory)
                             fileName.Contains("_files")))
             {
                 //TODO: Refactor
-                blogs.Add(new Files().Load(filename));
+                try
+                {
+                    IFiles blog = new Files().Load(filename);
+                    if (shellService.Settings.LoadAllDatabases)
+                    {
+                        blogs.Add(blog);
+                    }
+                }
+                catch (SerializationException ex)
+                {
+                    failedToLoadBlogs.Add(ex.Data["Filename"].ToString());
+                }
             }
+            if (failedToLoadBlogs.Any())
+            {
+                string failedBlogNames = failedToLoadBlogs.Aggregate((a, b) => a + ", " + b);
+                Logger.Verbose("ManagerController:LoadLibrary: {0}", failedBlogNames);
+                shellService.ShowError(new SerializationException(), Resources.CouldNotLoadLibrary, failedBlogNames);
+            }
+
             Logger.Verbose("ManagerController.GetFilesCore End");
 
             return blogs;
@@ -274,35 +302,31 @@ private async Task LoadAllDatabasesAsync()
         {
             Logger.Verbose("ManagerController.LoadDatabasesGloballyAsync:Start");
             managerService.ClearDatabases();
-            if (shellService.Settings.LoadAllDatabases)
-            {
-                string path = Path.Combine(shellService.Settings.DownloadLocation, "Index");
+            string path = Path.Combine(shellService.Settings.DownloadLocation, "Index");
 
-                try
+            try
+            {
+                if (Directory.Exists(path))
                 {
-                    if (Directory.Exists(path))
                     {
+                        IReadOnlyList<IFiles> databases = await GetIFilesAsync(path);
+                        foreach (IFiles database in databases)
                         {
-                            IReadOnlyList<IFiles> databases = await GetIFilesAsync(path);
-                            foreach (IFiles database in databases)
-                            {
-                                managerService.AddDatabase(database);
-                            }
+                            managerService.AddDatabase(database);
                         }
                     }
                 }
-                catch (Exception ex)
-                {
-                    Logger.Verbose("ManagerController:LoadDatabasesGloballyAsync: {0}", ex);
-                    shellService.ShowError(ex, Resources.CouldNotLoadLibrary, ex.Data["Filename"]);
-                }
             }
+            catch (Exception ex)
+            {
+                Logger.Verbose("ManagerController:LoadDatabasesGloballyAsync: {0}", ex);
+                shellService.ShowError(ex, Resources.CouldNotLoadLibrary, ex.Data["Filename"]);
+            }
+
             BlogManaerFinishedLoadingDatabases?.Invoke(this, EventArgs.Empty);
             Logger.Verbose("ManagerController.LoadDatabasesGloballyAsync:End");
         }
 
-
-
         private bool CanLoadLibrary()
         {
             return !crawlerService.IsCrawl;
@@ -486,12 +510,6 @@ private bool CanCopyUrl()
 
         private async Task CheckStatusAsync()
         {
-            //        foreach (IBlog blog in selectionService.SelectedBlogFiles.ToArray())
-            //        {
-            //            ICrawler crawler = CrawlerFactory.GetCrawler(blog, new CancellationToken(), new PauseToken(),
-            //new Progress<DownloadProgress>(), shellService, crawlerService, managerService);
-            //            await crawler.IsBlogOnlineAsync();
-            //        }
             await Task.Run(async () =>
             {
                 var semaphoreSlim = new SemaphoreSlim(25);

diff --git a/src/TumblThree/TumblThree.Applications/Crawler/ICrawler.cs b/src/TumblThree/TumblThree.Applications/Crawler/ICrawler.cs
@@ -9,7 +9,5 @@ public interface ICrawler
         Task IsBlogOnlineAsync();
 
         Task UpdateMetaInformationAsync();
-
-        void UpdateProgressQueueInformation(string format, params object[] args);
     }
 }
diff --git a/src/TumblThree/TumblThree.Domain/Models/Files/Files.cs b/src/TumblThree/TumblThree.Domain/Models/Files/Files.cs
@@ -22,6 +22,7 @@ public class Files : Model, IFiles
 
         public Files()
         {
+            // DO NOT USE
         }
 
         public Files(string name, string location)
-Original file line number
+Diff line change
@@ Expand Up / @@ -22,6 +22,7 @@ public class Files : Model, IFiles @@
             public Files()
             {
+                // DO NOT USE
             }
             public Files(string name, string location)
@@ Expand Down @@