Merge pull request #1429 from ebkr/chunky-pt2

Skip unnecessary steps when package list hasn't changed

src/components/mixins/SplashMixin.vue

@@ -4,6 +4,8 @@ import Component from 'vue-class-component';
 
 import R2Error from '../../model/errors/R2Error';
 import RequestItem from '../../model/requests/RequestItem';
+import type { PackageListChunks, PackageListIndex } from '../../store/modules/TsModsModule';
+
 
 @Component
 export default class SplashMixin extends Vue {
@@ -49,27 +51,83 @@ export default class SplashMixin extends Vue {
 
     // Get the list of Thunderstore mods from API or local cache.
     async getThunderstoreMods() {
-        this.loadingText = 'Connecting to Thunderstore';
-        let packageListChunks: {full_name: string}[][]|undefined = undefined;
+        const packageListIndex = await this.fetchPackageListIndex();
+        const packageListChunks = await this.fetchPackageListChunksIfUpdated(packageListIndex);
+        this.getRequestItem('ThunderstoreDownload').setProgress(100);
+        await this.writeModsToPersistentCacheIfUpdated(packageListIndex, packageListChunks);
+        const isModListLoaded = await this.readModsToVuex();
 
-        const showProgress = (progress: number) => {
-            this.loadingText = 'Getting mod list from Thunderstore';
+        // To proceed, the loading of the mod list should result in a non-empty list.
+        // Empty list is allowed if that's actually what the API returned.
+        // API wasn't queried at all if we already had the latest index chunk.
+        const modListHasMods = this.$store.state.tsMods.mods.length;
+        const apiReturnedEmptyList = packageListChunks && packageListChunks[0].length === 0;
+        const apiWasNotQueried = packageListIndex && packageListIndex.isLatest;
+        if (isModListLoaded && (modListHasMods || apiReturnedEmptyList || apiWasNotQueried)) {
+            await this.moveToNextScreen();
+        } else {
+            this.heroTitle = 'Failed to get the list of online mods';
+            this.loadingText = 'You may still use the manager offline, but some features might be unavailable.';
+            this.isOffline = true;
+        }
+    }
+
+    /***
+     * Query Thunderstore API for the URLs pointing to parts of the package list.
+     * Fails silently to fallback reading the old values from the IndexedDB cache.
+     */
+    async fetchPackageListIndex(): Promise<PackageListIndex|undefined> {
+        this.loadingText = 'Checking for mod list updates from Thunderstore';
+
+        try {
+            return await this.$store.dispatch('tsMods/fetchPackageListIndex');
+        } catch (e) {
+            console.error('SplashMixin failed to fetch mod list index from API.', e);
+            return undefined;
+        }
+    }
+
+    /***
+     * Load the package list in chunks pointed out by the packageListIndex.
+     * This step is skipped if we can't or don't need to load the chunks.
+     * Fails silently to fallback reading the old values from the IndexedDB cache.
+     */
+    async fetchPackageListChunksIfUpdated(packageListIndex?: PackageListIndex): Promise<PackageListChunks|undefined> {
+        // Skip loading chunks if loading index failed, or if we already have the latest data.
+        if (!packageListIndex || packageListIndex.isLatest) {
+            return undefined;
+        }
+
+        const progressCallback = (progress: number) => {
+            this.loadingText = 'Loading latest mod list from Thunderstore';
             this.getRequestItem('ThunderstoreDownload').setProgress(progress);
         };
 
         try {
-            packageListChunks = await this.$store.dispatch('tsMods/fetchPackageListChunks', showProgress);
+            return await this.$store.dispatch(
+                'tsMods/fetchPackageListChunks',
+                {chunkUrls: packageListIndex.content, progressCallback}
+            );
         } catch (e) {
             console.error('SplashMixin failed to fetch mod list from API.', e);
-        } finally {
-            this.getRequestItem('ThunderstoreDownload').setProgress(100);
+            return undefined;
         }
+    }
 
-        if (packageListChunks) {
+    /***
+     * Update a fresh package list to the IndexedDB cache.
+     * Done only if there was a fresh list to load and it was loaded successfully.
+     * Fails silently to fallback reading the old values from the IndexedDB cache.
+     */
+    async writeModsToPersistentCacheIfUpdated(packageListIndex?: PackageListIndex, packageListChunks?: PackageListChunks) {
+        if (packageListIndex && packageListChunks) {
             this.loadingText = 'Storing the mod list into local cache';
 
             try {
-                await this.$store.dispatch('tsMods/updatePersistentCache', packageListChunks);
+                await this.$store.dispatch(
+                    'tsMods/updatePersistentCache',
+                    {indexHash: packageListIndex.hash, chunks: packageListChunks}
+                );
             } catch (e) {
                 console.error('SplashMixin failed to cache mod list locally.', e);
             }
@@ -80,6 +138,22 @@ export default class SplashMixin extends Vue {
         }
 
         this.getRequestItem('CacheOperations').setProgress(50);
+    }
+
+    /***
+     * Read mod list from the IndexedDB cache to Vuex so it's kept in memory.
+     * Always read from the IndexedDB since we don't know if the mod list was
+     * queried from the API successfully or not. This also handles the type
+     * casting, since mod manager expects the data to be formatted into objects.
+     *
+     * Failure at this point is no longer silently ignored, instead an error
+     * modal is shown.
+     *
+     * Return value is used to tell whether Vuex might contain an empty list
+     * after calling this because there was an error, or because the package
+     * list is actually empty.
+     */
+    async readModsToVuex(): Promise<boolean> {
         let isModListLoaded = false;
 
         try {
@@ -92,18 +166,7 @@ export default class SplashMixin extends Vue {
             );
         } finally {
             this.getRequestItem('CacheOperations').setProgress(100);
-        }
-
-        // To proceed, the loading of the mod list should result in a non-empty list.
-        // Empty list is allowed if that's actually what the API returned.
-        const modListHasMods = this.$store.state.tsMods.mods.length;
-        const apiReturnedEmptyList = packageListChunks && packageListChunks[0].length === 0;
-        if (isModListLoaded && (modListHasMods || apiReturnedEmptyList)) {
-            await this.moveToNextScreen();
-        } else {
-            this.heroTitle = 'Failed to get the list of online mods';
-            this.loadingText = 'You may still use the manager offline, but some features might be unavailable.';
-            this.isOffline = true;
+            return isModListLoaded;
         }
     }
 

src/components/mixins/UtilityMixin.vue

@@ -4,6 +4,7 @@ import Component from 'vue-class-component';
 
 import R2Error from '../../model/errors/R2Error';
 import CdnProvider from '../../providers/generic/connection/CdnProvider';
+import { PackageListIndex } from '../../store/modules/TsModsModule';
 
 @Component
 export default class UtilityMixin extends Vue {
@@ -15,8 +16,21 @@ export default class UtilityMixin extends Vue {
     }
 
     async refreshThunderstoreModList() {
-        const packageListChunks = await this.$store.dispatch('tsMods/fetchPackageListChunks');
-        await this.$store.dispatch("tsMods/updatePersistentCache", packageListChunks);
+        const packageListIndex: PackageListIndex = await this.$store.dispatch("tsMods/fetchPackageListIndex");
+
+        if (packageListIndex.isLatest) {
+            await this.$store.dispatch("tsMods/updateModsLastUpdated");
+            return;
+        }
+
+        const packageListChunks = await this.$store.dispatch(
+            "tsMods/fetchPackageListChunks",
+            {chunkUrls: packageListIndex.content},
+        );
+        await this.$store.dispatch(
+            "tsMods/updatePersistentCache",
+            {chunks: packageListChunks, indexHash: packageListIndex.hash},
+        );
         await this.$store.dispatch("tsMods/updateMods");
         await this.$store.dispatch("profile/tryLoadModListFromDisk");
         await this.$store.dispatch("tsMods/prewarmCache");

src/r2mm/manager/PackageDexieStore.ts

@@ -40,15 +40,27 @@ interface DexiePackage {
     default_order: number; // Entry's index when received from the API
 }
 
+// For keeping track of seen package list index files so we can
+// skip processing chunks if there's no changes.
+interface IndexChunkHash {
+    community: string;
+    hash: string;
+    date_updated: Date;
+}
+
 class PackageDexieStore extends Dexie {
     packages!: Table<DexiePackage, string>;
+    indexHashes!: Table<IndexChunkHash, string>;
 
     constructor() {
         super('tsPackages');
 
         this.version(1).stores({
             packages: '[community+full_name], [community+date_fetched]'
         });
+        this.version(2).stores({
+            indexHashes: '&community, [community+hash]'
+        });
     }
 }
 
@@ -111,17 +123,17 @@ export async function getPackagesByNames(community: string, packageNames: string
     return packages.map(ThunderstoreMod.parseFromThunderstoreData);
 }
 
-// TODO: Dexie v3 doesn't support combining .where() and .orderBy() in a
-// way that would utilize the DB indexes. The current implementation
-// bypasses this by assuming that outside the updateFromApiResponse
-// transaction, all the packages have the same date_fetched value.
-// Moving to Dexie v4 might improve things, but if that doesn't turn out
-// to be true, filter or order the result set on JS side instead.
 export async function getLastPackageListUpdateTime(community: string) {
-    const fetched = await db.packages
-        .where('[community+date_fetched]')
-        .between([community, Dexie.minKey], [community, Dexie.maxKey])
-        .first();
+    const hash = await db.indexHashes.where({community}).first();
+    return hash ? hash.date_updated : undefined;
+}
+
+export async function isLatestPackageListIndex(community: string, hash: string) {
+    return Boolean(
+        await db.indexHashes.where({community, hash}).count()
+    );
+}
 
-    return fetched ? fetched.date_fetched : undefined;
+export async function setLatestPackageListIndex(community: string, hash: string) {
+    await db.indexHashes.put({community, hash, date_updated: new Date()});
 }

src/store/modules/TsModsModule.ts

@@ -28,7 +28,8 @@ interface State {
 
 type ProgressCallback = (progress: number) => void;
 type PackageListChunk = {full_name: string}[];
-type ChunkedPackageList = PackageListChunk[];
+export type PackageListChunks = PackageListChunk[];
+export type PackageListIndex = {content: string[], hash: string, isLatest: boolean};
 
 function isPackageListChunk(value: unknown): value is PackageListChunk {
     return Array.isArray(value) && (
@@ -169,39 +170,47 @@ export const TsModsModule = {
     },
 
     actions: <ActionTree<State, RootState>>{
-        async _fetchPackageListIndex({rootState}): Promise<string[]> {
+        async fetchPackageListIndex({dispatch, rootState}): Promise<PackageListIndex> {
             const indexUrl = rootState.activeGame.thunderstoreUrl;
-            const chunkIndex: string[] = await retry(() => fetchAndProcessBlobFile(indexUrl));
+            const index = await retry(() => fetchAndProcessBlobFile(indexUrl));
 
-            if (!isStringArray(chunkIndex)) {
+            if (!isStringArray(index.content)) {
                 throw new Error('Received invalid chunk index from API');
             }
-            if (isEmptyArray(chunkIndex)) {
+            if (isEmptyArray(index.content)) {
                 throw new Error('Received empty chunk index from API');
             }
 
-            return chunkIndex;
+            const community = rootState.activeGame.internalFolderName;
+            const isLatest = await PackageDb.isLatestPackageListIndex(community, index.hash);
+
+            // Normally the hash would be updated after the mod list is successfully
+            // fetched and written to IndexedDB, but if the list hasn't changed,
+            // those step are skipped, so update the "last seen" timestamp now.
+            if (isLatest) {
+                await dispatch('updateIndexHash', index.hash);
+            }
+
+            return {...index, isLatest};
         },
 
         async fetchPackageListChunks(
-            {dispatch},
-            progressCallback?: ProgressCallback
-        ): Promise<ChunkedPackageList> {
-            const chunkIndex: string[] = await dispatch('_fetchPackageListIndex');
-
+            {},
+            {chunkUrls, progressCallback}: {chunkUrls: string[], progressCallback?: ProgressCallback},
+        ): Promise<PackageListChunks> {
             // Count index as a chunk for progress bar purposes.
-            const chunkCount = chunkIndex.length + 1;
+            const chunkCount = chunkUrls.length + 1;
             let completed = 1;
             const updateProgress = () => progressCallback && progressCallback((completed / chunkCount) * 100);
             updateProgress();
 
             // Download chunks serially to avoid slow connections timing
             // out due to concurrent requests competing for the bandwidth.
             const chunks = [];
-            for (const [i, chunkUrl] of chunkIndex.entries()) {
-                const chunk = await retry(() => fetchAndProcessBlobFile(chunkUrl))
+            for (const [i, chunkUrl] of chunkUrls.entries()) {
+                const {content: chunk} = await retry(() => fetchAndProcessBlobFile(chunkUrl))
 
-                if (chunkIndex.length > 1 && isEmptyArray(chunkIndex)) {
+                if (chunkUrls.length > 1 && isEmptyArray(chunk)) {
                     throw new Error(`Chunk #${i} in multichunk response was empty`);
                 } else if (!isPackageListChunk(chunk)) {
                     throw new Error(`Chunk #${i} was invalid format`);
@@ -228,29 +237,39 @@ export const TsModsModule = {
             commit('setExclusions', exclusions);
         },
 
-        async updateMods({commit, rootState}) {
+        async updateMods({commit, dispatch, rootState}) {
             const modList = await PackageDb.getPackagesAsThunderstoreMods(rootState.activeGame.internalFolderName);
-            const updated = await PackageDb.getLastPackageListUpdateTime(rootState.activeGame.internalFolderName);
             commit('setMods', modList);
-            commit('setModsLastUpdated', updated);
             commit('updateDeprecated', modList);
             commit('clearModCache');
+            await dispatch('updateModsLastUpdated');
+        },
+
+        async updateModsLastUpdated({commit, rootState}) {
+            const updated = await PackageDb.getLastPackageListUpdateTime(rootState.activeGame.internalFolderName);
+            commit('setModsLastUpdated', updated);
         },
 
         /*** Save a mod list received from the Thunderstore API to IndexedDB */
         async updatePersistentCache(
             {dispatch, rootState, state},
-            packages: ChunkedPackageList
+            {chunks, indexHash}: {chunks: PackageListChunks, indexHash: string}
         ) {
             if (state.exclusions === undefined) {
                 await dispatch('updateExclusions');
             }
 
-            const filtered = packages.map((chunk) => chunk.filter(
+            const filtered = chunks.map((chunk) => chunk.filter(
                 (pkg) => !state.exclusions!.includes(pkg.full_name)
             ));
             const community = rootState.activeGame.internalFolderName;
             await PackageDb.updateFromApiResponse(community, filtered);
-        }
+            await dispatch('updateIndexHash', indexHash);
+        },
+
+        async updateIndexHash({rootState}, indexHash: string) {
+            const community = rootState.activeGame.internalFolderName;
+            await PackageDb.setLatestPackageListIndex(community, indexHash);
+        },
     }
 }

src/utils/HttpUtils.ts

@@ -112,8 +112,17 @@ export const makeLongRunningGetRequest = async (
 export const fetchAndProcessBlobFile = async (url: string) => {
     const response = await makeLongRunningGetRequest(url, {axiosConfig: {responseType: 'arraybuffer'}});
     const buffer = Buffer.from(response.data);
+    const hash = await getSha256Hash(buffer);
     const jsonString = await decompressArrayBuffer(buffer);
-    return JSON.parse(jsonString);
+    const content = JSON.parse(jsonString);
+    return {content, hash};
+}
+
+async function getSha256Hash(arrayBuffer: ArrayBuffer): Promise<string> {
+    const hashBuffer = await crypto.subtle.digest('SHA-256', arrayBuffer);
+    const hashByteArray = Array.from(new Uint8Array(hashBuffer));
+    const hexHash = hashByteArray.map(b => b.toString(16).padStart(2, '0')).join('');
+    return hexHash;
 }
 
 export const isNetworkError = (responseOrError: unknown) =>