diff --git a/AppInit.cs b/AppInit.cs index a98f209..0c1766a 100644 --- a/AppInit.cs +++ b/AppInit.cs @@ -74,6 +74,8 @@ public static AppInit conf public bool syncsport = true; + public bool syncspidr = false; + public int maxreadfile = 200; public Evercache evercache = new Evercache() { enable = true, validHour = 1, maxOpenWriteTask = 2000 }; diff --git a/Controllers/SyncController.cs b/Controllers/SyncController.cs index 958910f..5465036 100644 --- a/Controllers/SyncController.cs +++ b/Controllers/SyncController.cs @@ -20,6 +20,7 @@ public JsonResult Configuration() return Json(new { fbd = true, + spidr = true, version = 2 }); } @@ -41,7 +42,7 @@ public ActionResult FdbKey(string key) } [Route("/sync/fdb/torrents")] - public ActionResult FdbTorrents(long time, long start = -1) + public ActionResult FdbTorrents(long time, long start = -1, bool spidr = false) { if (!AppInit.conf.opensync || time == 0) return Json(new { nextread = false, collections = new List() }); @@ -62,7 +63,7 @@ public ActionResult FdbTorrents(long time, long start = -1) foreach (var t in FileDB.OpenRead(item.Key)) { - if (start != -1 && start > t.Value.updateTime.ToFileTimeUtc()) + if (spidr || (start != -1 && start > t.Value.updateTime.ToFileTimeUtc())) { torrent.TryAdd(t.Key, new TorrentDetails() { @@ -115,7 +116,7 @@ public ActionResult FdbTorrents(long time, long start = -1) } } - return Json(new { nextread, collections }); + return Json(new { nextread, countread, take, collections }); } diff --git a/Engine/SyncCron.cs b/Engine/SyncCron.cs index f9937be..1ff60f5 100644 --- a/Engine/SyncCron.cs +++ b/Engine/SyncCron.cs @@ -14,7 +14,8 @@ public static class SyncCron { static long lastsync = -1, starsync = -1; - async public static Task Run() + #region Torrents + async public static Task Torrents() { await Task.Delay(20_000); @@ -55,19 +56,16 @@ async public static Task Run() else if (root.collections.Count > 0) { reset = true; - var torrents = new List(); + var torrents = new List(root.countread); foreach (var collection in root.collections) { foreach (var torrent in collection.Value.torrents) { - if (torrent.Value.types == null) + if (AppInit.conf.synctrackers != null && torrent.Value.trackerName != null && !AppInit.conf.synctrackers.Contains(torrent.Value.trackerName)) continue; - if (AppInit.conf.synctrackers != null && !AppInit.conf.synctrackers.Contains(torrent.Value.trackerName)) - continue; - - if (!AppInit.conf.syncsport && torrent.Value.types.Contains("sport")) + if (!AppInit.conf.syncsport && torrent.Value.types != null && torrent.Value.types.Contains("sport")) continue; torrents.Add(torrent.Value); @@ -118,6 +116,8 @@ async public static Task Run() FileDB.SaveChangesToFile(); File.WriteAllText("lastsync.txt", lastsync.ToString()); + + Console.WriteLine("sync: end"); } else { @@ -125,7 +125,7 @@ async public static Task Run() continue; } } - catch + catch (Exception ex) { try { @@ -136,12 +136,62 @@ async public static Task Run() } } catch { } + + Console.WriteLine("sync: error / " + ex.Message); } - Console.WriteLine("sync: end"); await Task.Delay(1000 * Random.Shared.Next(60, 300)); await Task.Delay(1000 * 60 * (20 > AppInit.conf.timeSync ? 20 : AppInit.conf.timeSync)); } } + #endregion + + #region Spidr + async public static Task Spidr() + { + while (true) + { + await Task.Delay(1000 * Random.Shared.Next(60, 300)); + await Task.Delay(1000 * 60 * 120); + + try + { + if (!string.IsNullOrWhiteSpace(AppInit.conf.syncapi) && AppInit.conf.syncspidr) + { + long lastsync_spidr = -1; + + var conf = await HttpClient.Get($"{AppInit.conf.syncapi}/sync/conf"); + if (conf != null && conf.ContainsKey("spidr") && conf.Value("spidr")) + { + Console.WriteLine($"\n\nsync_spidr: start / {DateTime.Now}"); + + next: var root = await HttpClient.Get($"{AppInit.conf.syncapi}/sync/fdb/torrents?time={lastsync_spidr}&spidr=true", timeoutSeconds: 300, MaxResponseContentBufferSize: 100_000_000); + + Console.WriteLine($"sync_spidr: time={lastsync_spidr}"); + + if (root?.collections != null && root.collections.Count > 0) + { + foreach (var collection in root.collections) + FileDB.AddOrUpdate(collection.Value.torrents.Values); + + lastsync_spidr = root.collections.Last().Value.fileTime; + + if (root.nextread) + goto next; + } + + Console.WriteLine("sync_spidr: end"); + } + } + else + { + await Task.Delay(TimeSpan.FromMinutes(1)); + continue; + } + } + catch (Exception ex) { Console.WriteLine("sync_spidr: error / " + ex.Message); } + } + } + #endregion } } \ No newline at end of file diff --git a/Models/Sync/v2/RootObject.cs b/Models/Sync/v2/RootObject.cs index 7c83c22..52c7a99 100644 --- a/Models/Sync/v2/RootObject.cs +++ b/Models/Sync/v2/RootObject.cs @@ -6,6 +6,10 @@ public class RootObject { public bool nextread { get; set; } + public int take { get; set; } + + public int countread { get; set; } + public List collections { get; set; } } } diff --git a/Program.cs b/Program.cs index e5a9efc..bbe431b 100644 --- a/Program.cs +++ b/Program.cs @@ -12,7 +12,9 @@ public class Program { public static void Main(string[] args) { - ThreadPool.QueueUserWorkItem(async _ => await SyncCron.Run()); + ThreadPool.QueueUserWorkItem(async _ => await SyncCron.Torrents()); + ThreadPool.QueueUserWorkItem(async _ => await SyncCron.Spidr()); + ThreadPool.QueueUserWorkItem(async _ => await StatsCron.Run()); ThreadPool.QueueUserWorkItem(async _ => await FileDB.Cron());