Skip to content

Commit

Permalink
spidr
Browse files Browse the repository at this point in the history
  • Loading branch information
@immisterio
immisterio committed Jan 19, 2024
1 parent e844e05 commit 5434549
Show file tree
Hide file tree
Showing 5 changed files with 72 additions and 13 deletions.
2 changes: 2 additions & 0 deletions AppInit.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,8 @@ public static AppInit conf

public bool syncsport = true;

public bool syncspidr = false;

public int maxreadfile = 200;

public Evercache evercache = new Evercache() { enable = true, validHour = 1, maxOpenWriteTask = 2000 };
Expand Down
7 changes: 4 additions & 3 deletions Controllers/SyncController.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ public JsonResult Configuration()
return Json(new
{
fbd = true,
spidr = true,
version = 2
});
}
Expand All @@ -41,7 +42,7 @@ public ActionResult FdbKey(string key)
}

[Route("/sync/fdb/torrents")]
public ActionResult FdbTorrents(long time, long start = -1)
public ActionResult FdbTorrents(long time, long start = -1, bool spidr = false)
{
if (!AppInit.conf.opensync || time == 0)
return Json(new { nextread = false, collections = new List<Collection>() });
Expand All @@ -62,7 +63,7 @@ public ActionResult FdbTorrents(long time, long start = -1)

foreach (var t in FileDB.OpenRead(item.Key))
{
if (start != -1 && start > t.Value.updateTime.ToFileTimeUtc())
if (spidr || (start != -1 && start > t.Value.updateTime.ToFileTimeUtc()))
{
torrent.TryAdd(t.Key, new TorrentDetails()
{
Expand Down Expand Up @@ -115,7 +116,7 @@ public ActionResult FdbTorrents(long time, long start = -1)
}
}

return Json(new { nextread, collections });
return Json(new { nextread, countread, take, collections });
}


Expand Down
68 changes: 59 additions & 9 deletions Engine/SyncCron.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,8 @@ public static class SyncCron
{
static long lastsync = -1, starsync = -1;

async public static Task Run()
#region Torrents
async public static Task Torrents()
{
await Task.Delay(20_000);

Expand Down Expand Up @@ -55,19 +56,16 @@ async public static Task Run()
else if (root.collections.Count > 0)
{
reset = true;
var torrents = new List<TorrentBaseDetails>();
var torrents = new List<TorrentBaseDetails>(root.countread);

foreach (var collection in root.collections)
{
foreach (var torrent in collection.Value.torrents)
{
if (torrent.Value.types == null)
if (AppInit.conf.synctrackers != null && torrent.Value.trackerName != null && !AppInit.conf.synctrackers.Contains(torrent.Value.trackerName))
continue;

if (AppInit.conf.synctrackers != null && !AppInit.conf.synctrackers.Contains(torrent.Value.trackerName))
continue;

if (!AppInit.conf.syncsport && torrent.Value.types.Contains("sport"))
if (!AppInit.conf.syncsport && torrent.Value.types != null && torrent.Value.types.Contains("sport"))
continue;

torrents.Add(torrent.Value);
Expand Down Expand Up @@ -118,14 +116,16 @@ async public static Task Run()

FileDB.SaveChangesToFile();
File.WriteAllText("lastsync.txt", lastsync.ToString());

Console.WriteLine("sync: end");
}
else
{
await Task.Delay(TimeSpan.FromMinutes(1));
continue;
}
}
catch
catch (Exception ex)
{
try
{
Expand All @@ -136,12 +136,62 @@ async public static Task Run()
}
}
catch { }

Console.WriteLine("sync: error / " + ex.Message);
}

Console.WriteLine("sync: end");
await Task.Delay(1000 * Random.Shared.Next(60, 300));
await Task.Delay(1000 * 60 * (20 > AppInit.conf.timeSync ? 20 : AppInit.conf.timeSync));
}
}
#endregion

#region Spidr
async public static Task Spidr()
{
while (true)
{
await Task.Delay(1000 * Random.Shared.Next(60, 300));
await Task.Delay(1000 * 60 * 120);

try
{
if (!string.IsNullOrWhiteSpace(AppInit.conf.syncapi) && AppInit.conf.syncspidr)
{
long lastsync_spidr = -1;

var conf = await HttpClient.Get<JObject>($"{AppInit.conf.syncapi}/sync/conf");
if (conf != null && conf.ContainsKey("spidr") && conf.Value<bool>("spidr"))
{
Console.WriteLine($"\n\nsync_spidr: start / {DateTime.Now}");

next: var root = await HttpClient.Get<Models.Sync.v2.RootObject>($"{AppInit.conf.syncapi}/sync/fdb/torrents?time={lastsync_spidr}&spidr=true", timeoutSeconds: 300, MaxResponseContentBufferSize: 100_000_000);

Console.WriteLine($"sync_spidr: time={lastsync_spidr}");

if (root?.collections != null && root.collections.Count > 0)
{
foreach (var collection in root.collections)
FileDB.AddOrUpdate(collection.Value.torrents.Values);

lastsync_spidr = root.collections.Last().Value.fileTime;

if (root.nextread)
goto next;
}

Console.WriteLine("sync_spidr: end");
}
}
else
{
await Task.Delay(TimeSpan.FromMinutes(1));
continue;
}
}
catch (Exception ex) { Console.WriteLine("sync_spidr: error / " + ex.Message); }
}
}
#endregion
}
}
4 changes: 4 additions & 0 deletions Models/Sync/v2/RootObject.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,10 @@ public class RootObject
{
public bool nextread { get; set; }

public int take { get; set; }

public int countread { get; set; }

public List<Collection> collections { get; set; }
}
}
4 changes: 3 additions & 1 deletion Program.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,9 @@ public class Program
{
public static void Main(string[] args)
{
ThreadPool.QueueUserWorkItem(async _ => await SyncCron.Run());
ThreadPool.QueueUserWorkItem(async _ => await SyncCron.Torrents());
ThreadPool.QueueUserWorkItem(async _ => await SyncCron.Spidr());

ThreadPool.QueueUserWorkItem(async _ => await StatsCron.Run());

ThreadPool.QueueUserWorkItem(async _ => await FileDB.Cron());
Expand Down

0 comments on commit 5434549

Please sign in to comment.