Skip to content

Commit

Permalink
Parallel fetching of torrents
Browse files Browse the repository at this point in the history
  • Loading branch information
Ben Sarmiento committed May 15, 2024
1 parent 93961f1 commit 4ccbb64
Show file tree
Hide file tree
Showing 7 changed files with 162 additions and 244 deletions.
53 changes: 40 additions & 13 deletions src/pages/library.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -366,8 +366,8 @@ function TorrentsPage() {
);
const hashesArr = Array.from(hashes);
hashesArr.sort();
checkForUncachedInRd(rdKey, userTorrentsList, setUncachedRdHashes, torrentDB).then(
(nonVideoHashes) => {
checkForUncachedInRd(rdKey, userTorrentsList, setUncachedRdHashes, torrentDB)
.then((nonVideoHashes) => {
setUserTorrentsList((prev) => {
return prev.map((t) => {
if (t.id.startsWith('rd:') && nonVideoHashes.has(t.hash)) {
Expand All @@ -378,8 +378,11 @@ function TorrentsPage() {
return t;
});
});
}
);
return userTorrentsList.filter((t) => nonVideoHashes.has(t.hash));
})
.then((nonVideoTorrents) => {
return Promise.all(nonVideoTorrents.map((t) => torrentDB.add(t)));
});
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [rdKey, rdSyncing]);

Expand Down Expand Up @@ -1019,31 +1022,55 @@ function TorrentsPage() {
info.files.filter((f) => f.selected).length !== info.links.length &&
info.links.length === 1))
) {
t.mediaType = 'other';
t.title = t.filename;
t.info = undefined;
setUserTorrentsList((prev) => {
const idx = prev.findIndex((i) => i.id === t.id);
if (idx >= 0) {
const newList = [...prev];
newList[idx].mediaType = 'other';
newList[idx].title = newList[idx].filename;
newList[idx].info = undefined;
return newList;
}
return prev;
});
await torrentDB.add(t);
} else if (
t.mediaType !== 'tv' &&
t.mediaType === 'movie' &&
(hasEpisodes ||
some(torrentAndFiles, (f) => /s\d\d\d?.?e\d\d\d?/i.test(f)) ||
some(torrentAndFiles, (f) => /season.?\d+/i.test(f)) ||
some(torrentAndFiles, (f) => /episodes?\s?\d+/i.test(f)) ||
some(torrentAndFiles, (f) => /\b[a-fA-F0-9]{8}\b/.test(f)))
) {
t.mediaType = 'tv';
t.info = filenameParse(t.filename, true);
setUserTorrentsList((prev) => {
const idx = prev.findIndex((i) => i.id === t.id);
if (idx >= 0) {
const newList = [...prev];
newList[idx].mediaType = 'tv';
newList[idx].info = filenameParse(t.filename, true);
return newList;
}
return prev;
});
await torrentDB.add(t);
} else if (
t.mediaType !== 'movie' &&
t.mediaType === 'tv' &&
!hasEpisodes &&
every(torrentAndFiles, (f) => !/s\d\d\d?.?e\d\d\d?/i.test(f)) &&
every(torrentAndFiles, (f) => !/season.?\d+/i.test(f)) &&
every(torrentAndFiles, (f) => !/episodes?\s?\d+/i.test(f)) &&
every(torrentAndFiles, (f) => !/\b[a-fA-F0-9]{8}\b/.test(f))
) {
t.mediaType = 'movie';
t.info = filenameParse(t.filename);
setUserTorrentsList((prev) => {
const idx = prev.findIndex((i) => i.id === t.id);
if (idx >= 0) {
const newList = [...prev];
newList[idx].mediaType = 'movie';
newList[idx].info = filenameParse(t.filename);
return newList;
}
return prev;
});
await torrentDB.add(t);
}

Expand Down
53 changes: 25 additions & 28 deletions src/services/realDebrid.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -214,43 +214,40 @@ export const getCurrentUser = async (accessToken: string) => {
}
};

export async function* getUserTorrentsList(accessToken: string, limit: number = 0) {
interface UserTorrentsResult {
data: UserTorrentResponse[];
totalCount: number | null;
}

export async function getUserTorrentsList(
accessToken: string,
limit: number = 0,
page: number = 1
): Promise<UserTorrentsResult> {
const headers = {
Authorization: `Bearer ${accessToken}`,
};

let page = 1;
const limitSet = limit || Infinity;
if (!limit) limit = 1000;

while (true) {
const response = await axios.get<UserTorrentResponse[]>(
`${config.realDebridHostname}/rest/1.0/torrents`,
{ headers, params: { page, limit } }
);

const {
data,
headers: { 'x-total-count': totalCount },
} = response;

yield data; // Yield the current page of torrents
const response = await axios.get<UserTorrentResponse[]>(
`${config.realDebridHostname}/rest/1.0/torrents`,
{ headers, params: { page, limit } }
);

if (data.length < limit || !totalCount) {
break;
}
const {
data,
headers: { 'x-total-count': totalCount },
} = response;

const totalCountValue = parseInt(totalCount, 10);
// Parse the totalCount from the headers
let totalCountValue: number | null = null;
if (totalCount) {
totalCountValue = parseInt(totalCount, 10);
if (isNaN(totalCountValue)) {
break;
}

if (data.length >= limitSet || data.length >= totalCountValue) {
break;
totalCountValue = null;
}

page++;
}

return { data, totalCount: totalCountValue };
}

export const getDownloads = async (accessToken: string): Promise<DownloadResponse[]> => {
Expand Down
1 change: 1 addition & 0 deletions src/torrent/db.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,7 @@ class UserTorrentDB {

private async insertToDB(torrent: UserTorrent) {
const db = await this.getDB();
await db.delete(this.torrentsTbl, torrent.id);
await db.put(this.torrentsTbl, torrent);
}

Expand Down
150 changes: 94 additions & 56 deletions src/utils/fetchTorrents.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,71 +15,109 @@ export const fetchRealDebrid = async (
customLimit?: number
) => {
try {
for await (let pageOfTorrents of getUserTorrentsList(rdKey, customLimit)) {
const torrents = pageOfTorrents.map((torrentInfo) => {
let mediaType = getTypeByNameAndFileCount(
torrentInfo.filename,
torrentInfo.links.length
);
const serviceStatus = torrentInfo.status;
let status: UserTorrentStatus;
switch (torrentInfo.status) {
case 'magnet_conversion':
case 'waiting_files_selection':
case 'queued':
status = UserTorrentStatus.waiting;
break;
case 'downloading':
case 'compressing':
status = UserTorrentStatus.downloading;
break;
case 'uploading':
case 'downloaded':
status = UserTorrentStatus.finished;
break;
default:
status = UserTorrentStatus.error;
break;
}
let info = {} as ParsedFilename;
try {
info =
mediaType === 'movie'
? filenameParse(torrentInfo.filename)
: filenameParse(torrentInfo.filename, true);
} catch (error) {
// flip the condition if error is thrown
mediaType = mediaType === 'movie' ? 'tv' : 'movie';
mediaType === 'movie'
? filenameParse(torrentInfo.filename)
: filenameParse(torrentInfo.filename, true);
}
return {
...torrentInfo,
// score: getReleaseTags(torrentInfo.filename, torrentInfo.bytes / ONE_GIGABYTE).score,
info,
status,
serviceStatus,
mediaType,
added: new Date(torrentInfo.added.replace('Z', '+01:00')),
id: `rd:${torrentInfo.id}`,
links: torrentInfo.links.map((l) => l.replaceAll('/', '/')),
seeders: torrentInfo.seeders || 0,
speed: torrentInfo.speed || 0,
title: getMediaId(info, mediaType, false) || torrentInfo.filename,
cached: true,
selectedFiles: [],
};
}) as UserTorrent[];
// Step 1: Initial request to get the first item and total count of items
const { data: initialData, totalCount } = await getUserTorrentsList(
rdKey,
customLimit ?? 1,
1
);

if (!initialData.length) {
await callback([]);
return;
}

// Step 2: If limit input is set, convert and call callback
if (customLimit && customLimit === 2) {
const torrents = await processTorrents(initialData);
await callback(torrents);
return;
}

// Step 3: Send requests in parallel to fetch the other items. Limit count should be 1000
const limit = 1000;
const maxPages = Math.ceil((totalCount ?? 0) / limit);
const allPagesPromises = [];

for (let page = 1; page <= maxPages; page++) {
allPagesPromises.push(getUserTorrentsList(rdKey, limit, page));
// if multiple of 5, wait for 1 second
if (page % 5 === 0) {
await new Promise((resolve) => setTimeout(resolve, 1000));
}
}

const pagesOfTorrents = await Promise.all(allPagesPromises);
const allData = pagesOfTorrents.flatMap((pageResult) => pageResult.data);

const torrents = await processTorrents(allData);
await callback(torrents);
} catch (error) {
await callback([]);
toast.error('Error fetching Real-Debrid torrents list', genericToastOptions);
console.error(error);
}
};

async function processTorrents(torrentData: UserTorrentResponse[]): Promise<UserTorrent[]> {
return Promise.all(
torrentData.map((torrentInfo) => {
let mediaType = getTypeByNameAndFileCount(
torrentInfo.filename,
torrentInfo.links.length
);
const serviceStatus = torrentInfo.status;
let status: UserTorrentStatus;
switch (torrentInfo.status) {
case 'magnet_conversion':
case 'waiting_files_selection':
case 'queued':
status = UserTorrentStatus.waiting;
break;
case 'downloading':
case 'compressing':
status = UserTorrentStatus.downloading;
break;
case 'uploading':
case 'downloaded':
status = UserTorrentStatus.finished;
break;
default:
status = UserTorrentStatus.error;
break;
}
let info = {} as ParsedFilename;
try {
info =
mediaType === 'movie'
? filenameParse(torrentInfo.filename)
: filenameParse(torrentInfo.filename, true);
} catch (error) {
// flip the condition if error is thrown
mediaType = mediaType === 'movie' ? 'tv' : 'movie';
mediaType === 'movie'
? filenameParse(torrentInfo.filename)
: filenameParse(torrentInfo.filename, true);
}
return {
...torrentInfo,
info,
status,
serviceStatus,
mediaType,
added: new Date(torrentInfo.added.replace('Z', '+01:00')),
id: `rd:${torrentInfo.id}`,
links: torrentInfo.links.map((l) => l.replaceAll('/', '/')),
seeders: torrentInfo.seeders || 0,
speed: torrentInfo.speed || 0,
title: getMediaId(info, mediaType, false) || torrentInfo.filename,
cached: true,
selectedFiles: [],
};
})
);
}

export const fetchAllDebrid = async (
adKey: string,
callback: (torrents: UserTorrent[]) => Promise<void>
Expand Down
1 change: 0 additions & 1 deletion src/utils/instantChecks.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -134,7 +134,6 @@ export const checkForUncachedInRd = async (
}
}
const hashes = Array.from(hashesToCheck);
console.log('Checking RD for uncached hashes:', hashes.length);

const funcs = [];
for (const hashGroup of groupBy(100, hashes)) {
Expand Down
Loading

0 comments on commit 4ccbb64

Please sign in to comment.