Merge remote-tracking branch 'origin' into feat/xxhash

.gitignore

@@ -21,3 +21,5 @@ mobile/openapi/.openapi-generator/FILES
 open-api/typescript-sdk/build
 mobile/android/fastlane/report.xml
 mobile/ios/fastlane/report.xml
+
+vite.config.js.timestamp-*

e2e/src/api/specs/library.e2e-spec.ts

@@ -500,13 +500,13 @@ describe('/libraries', () => {
     });
 
     it('should set an asset offline its file is not in any import path', async () => {
+      utils.createImageFile(`${testAssetDir}/temp/offline/offline.png`);
+
       const library = await utils.createLibrary(admin.accessToken, {
         ownerId: admin.userId,
         importPaths: [`${testAssetDirInternal}/temp/offline`],
       });
 
-      utils.createImageFile(`${testAssetDir}/temp/offline/offline.png`);
-
       await scan(admin.accessToken, library.id);
       await utils.waitForQueueFinish(admin.accessToken, 'library');
 

e2e/src/utils.ts

@@ -374,8 +374,8 @@ export const utils = {
   },
 
   createDirectory: (path: string) => {
-    if (!existsSync(dirname(path))) {
-      mkdirSync(dirname(path), { recursive: true });
+    if (!existsSync(path)) {
+      mkdirSync(path, { recursive: true });
     }
   },
 
@@ -392,7 +392,7 @@ export const utils = {
       return;
     }
 
-    rmSync(path);
+    rmSync(path, { recursive: true });
   },
 
   getAssetInfo: (accessToken: string, id: string) => getAssetInfo({ id }, { headers: asBearerAuth(accessToken) }),

server/src/entities/asset-files.entity.ts

@@ -36,7 +36,7 @@ export class AssetFileEntity {
   @Column()
   path!: string;
 
-  @Column({ type: 'bytea' })
+  @Column({ type: 'bytea', nullable: true, default: null })
   @Index()
   checksum!: Buffer | null;
 }

server/src/interfaces/asset.interface.ts

@@ -195,6 +195,7 @@ export interface IAssetRepository {
   getDuplicates(options: AssetBuilderOptions): Promise<AssetEntity[]>;
   getAllForUserFullSync(options: AssetFullSyncOptions): Promise<AssetEntity[]>;
   getChangedDeltaSync(options: AssetDeltaSyncOptions): Promise<AssetEntity[]>;
+  removeFile(assetId: string, type: AssetFileType): Promise<void>;
   upsertFile(file: UpsertFileOptions): Promise<void>;
   upsertFiles(files: UpsertFileOptions[]): Promise<void>;
 }

server/src/migrations/1728632095015-AddAssetFileChecksum.ts

@@ -4,7 +4,7 @@ export class AssetFileChecksum1728632095015 implements MigrationInterface {
   name = 'AssetFileChecksum1728632095015';
 
   public async up(queryRunner: QueryRunner): Promise<void> {
-    await queryRunner.query(`ALTER TABLE "asset_files" ADD "checksum" bytea NULL`);
+    await queryRunner.query(`ALTER TABLE "asset_files" ADD "checksum" bytea`);
     await queryRunner.query(`CREATE INDEX "IDX_c946066edd16cfa5c25a26aa8e" ON "asset_files" ("checksum")`);
   }
 

server/src/queries/asset.repository.sql

@@ -1134,7 +1134,8 @@ SET
 RETURNING
   "id",
   "createdAt",
-  "updatedAt"
+  "updatedAt",
+  "checksum"
 
 -- AssetRepository.upsertFiles
 INSERT INTO
@@ -1159,4 +1160,5 @@ SET
 RETURNING
   "id",
   "createdAt",
-  "updatedAt"
+  "updatedAt",
+  "checksum"

server/src/repositories/asset.repository.ts

@@ -767,6 +767,10 @@ export class AssetRepository implements IAssetRepository {
     return builder.getMany();
   }
 
+  async removeFile(assetId: string, type: AssetFileType): Promise<void> {
+    await this.fileRepository.delete({ assetId, type });
+  }
+
   @GenerateSql({ params: [{ assetId: DummyValue.UUID, type: AssetFileType.PREVIEW, path: '/path/to/file' }] })
   async upsertFile(file: { assetId: string; type: AssetFileType; path: string; checksum?: Buffer }): Promise<void> {
     await this.fileRepository.upsert(file, { conflictPaths: ['assetId', 'type'] });

server/src/services/library.service.spec.ts

@@ -119,6 +119,64 @@ describe(LibraryService.name, () => {
     });
   });
 
+  describe('onConfigUpdateEvent', () => {
+    beforeEach(async () => {
+      systemMock.get.mockResolvedValue(defaults);
+      databaseMock.tryLock.mockResolvedValue(true);
+      await sut.onBootstrap();
+    });
+
+    it('should do nothing if oldConfig is not provided', async () => {
+      await sut.onConfigUpdate({ newConfig: systemConfigStub.libraryScan as SystemConfig });
+      expect(jobMock.updateCronJob).not.toHaveBeenCalled();
+    });
+
+    it('should do nothing if instance does not have the watch lock', async () => {
+      databaseMock.tryLock.mockResolvedValue(false);
+      await sut.onBootstrap();
+      await sut.onConfigUpdate({ newConfig: systemConfigStub.libraryScan as SystemConfig, oldConfig: defaults });
+      expect(jobMock.updateCronJob).not.toHaveBeenCalled();
+    });
+
+    it('should update cron job and enable watching', async () => {
+      libraryMock.getAll.mockResolvedValue([]);
+      await sut.onConfigUpdate({
+        newConfig: {
+          library: { ...systemConfigStub.libraryScan.library, ...systemConfigStub.libraryWatchEnabled.library },
+        } as SystemConfig,
+        oldConfig: defaults,
+      });
+
+      expect(jobMock.updateCronJob).toHaveBeenCalledWith(
+        'libraryScan',
+        systemConfigStub.libraryScan.library.scan.cronExpression,
+        systemConfigStub.libraryScan.library.scan.enabled,
+      );
+    });
+
+    it('should update cron job and disable watching', async () => {
+      libraryMock.getAll.mockResolvedValue([]);
+      await sut.onConfigUpdate({
+        newConfig: {
+          library: { ...systemConfigStub.libraryScan.library, ...systemConfigStub.libraryWatchEnabled.library },
+        } as SystemConfig,
+        oldConfig: defaults,
+      });
+      await sut.onConfigUpdate({
+        newConfig: {
+          library: { ...systemConfigStub.libraryScan.library, ...systemConfigStub.libraryWatchDisabled.library },
+        } as SystemConfig,
+        oldConfig: defaults,
+      });
+
+      expect(jobMock.updateCronJob).toHaveBeenCalledWith(
+        'libraryScan',
+        systemConfigStub.libraryScan.library.scan.cronExpression,
+        systemConfigStub.libraryScan.library.scan.enabled,
+      );
+    });
+  });
+
   describe('onConfigValidateEvent', () => {
     it('should allow a valid cron expression', () => {
       expect(() =>
@@ -139,7 +197,7 @@ describe(LibraryService.name, () => {
     });
   });
 
-  describe('handleQueueAssetRefresh', () => {
+  describe('handleQueueSyncFiles', () => {
     it('should queue refresh of a new asset', async () => {
       libraryMock.get.mockResolvedValue(libraryStub.externalLibrary1);
       storageMock.walk.mockImplementation(mockWalk);
@@ -559,8 +617,8 @@ describe(LibraryService.name, () => {
       expect(jobMock.queueAll).not.toHaveBeenCalled();
     });
 
-    it('should throw BadRequestException when asset does not exist', async () => {
-      storageMock.stat.mockRejectedValue(new Error("ENOENT, no such file or directory '/data/user1/photo.jpg'"));
+    it('should fail when the file could not be read', async () => {
+      storageMock.stat.mockRejectedValue(new Error('Could not read file'));
 
       const mockLibraryJob: ILibraryFileJob = {
         id: libraryStub.externalLibrary1.id,
@@ -572,6 +630,27 @@ describe(LibraryService.name, () => {
       assetMock.create.mockResolvedValue(assetStub.image);
 
       await expect(sut.handleSyncFile(mockLibraryJob)).resolves.toBe(JobStatus.FAILED);
+      expect(libraryMock.get).not.toHaveBeenCalled();
+      expect(assetMock.create).not.toHaveBeenCalled();
+    });
+
+    it('should skip if the file could not be found', async () => {
+      const error = new Error('File not found') as any;
+      error.code = 'ENOENT';
+      storageMock.stat.mockRejectedValue(error);
+
+      const mockLibraryJob: ILibraryFileJob = {
+        id: libraryStub.externalLibrary1.id,
+        ownerId: userStub.admin.id,
+        assetPath: '/data/user1/photo.jpg',
+      };
+
+      assetMock.getByLibraryIdAndOriginalPath.mockResolvedValue(null);
+      assetMock.create.mockResolvedValue(assetStub.image);
+
+      await expect(sut.handleSyncFile(mockLibraryJob)).resolves.toBe(JobStatus.SKIPPED);
+      expect(libraryMock.get).not.toHaveBeenCalled();
+      expect(assetMock.create).not.toHaveBeenCalled();
     });
   });
 
@@ -654,6 +733,10 @@ describe(LibraryService.name, () => {
 
       expect(libraryMock.getStatistics).toHaveBeenCalledWith(libraryStub.externalLibrary1.id);
     });
+
+    it('should throw an error if the library could not be found', async () => {
+      await expect(sut.getStatistics('foo')).rejects.toBeInstanceOf(BadRequestException);
+    });
   });
 
   describe('create', () => {
@@ -783,6 +866,13 @@ describe(LibraryService.name, () => {
     });
   });
 
+  describe('getAll', () => {
+    it('should get all libraries', async () => {
+      libraryMock.getAll.mockResolvedValue([libraryStub.externalLibrary1]);
+      await expect(sut.getAll()).resolves.toEqual([expect.objectContaining({ id: libraryStub.externalLibrary1.id })]);
+    });
+  });
+
   describe('handleQueueCleanup', () => {
     it('should queue cleanup jobs', async () => {
       libraryMock.getAllDeleted.mockResolvedValue([libraryStub.externalLibrary1, libraryStub.externalLibrary2]);
@@ -803,15 +893,38 @@ describe(LibraryService.name, () => {
       await sut.onBootstrap();
     });
 
+    it('should throw an error if an import path is invalid', async () => {
+      libraryMock.update.mockResolvedValue(libraryStub.externalLibrary1);
+      libraryMock.get.mockResolvedValue(libraryStub.externalLibrary1);
+
+      await expect(sut.update('library-id', { importPaths: ['foo/bar'] })).rejects.toBeInstanceOf(BadRequestException);
+      expect(libraryMock.update).not.toHaveBeenCalled();
+    });
+
     it('should update library', async () => {
       libraryMock.update.mockResolvedValue(libraryStub.externalLibrary1);
       libraryMock.get.mockResolvedValue(libraryStub.externalLibrary1);
-      await expect(sut.update('library-id', {})).resolves.toEqual(mapLibrary(libraryStub.externalLibrary1));
+      storageMock.stat.mockResolvedValue({ isDirectory: () => true } as Stats);
+      storageMock.checkFileExists.mockResolvedValue(true);
+
+      await expect(sut.update('library-id', { importPaths: ['foo/bar'] })).resolves.toEqual(
+        mapLibrary(libraryStub.externalLibrary1),
+      );
       expect(libraryMock.update).toHaveBeenCalledWith(expect.objectContaining({ id: 'library-id' }));
     });
   });
 
+  describe('onShutdown', () => {
+    it('should do nothing if instance does not have the watch lock', async () => {
+      await sut.onShutdown();
+    });
+  });
+
   describe('watchAll', () => {
+    it('should return false if instance does not have the watch lock', async () => {
+      await expect(sut.watchAll()).resolves.toBe(false);
+    });
+
     describe('watching disabled', () => {
       beforeEach(async () => {
         systemMock.get.mockResolvedValue(systemConfigStub.libraryWatchDisabled);
@@ -872,6 +985,7 @@ describe(LibraryService.name, () => {
       it('should handle a new file event', async () => {
         libraryMock.get.mockResolvedValue(libraryStub.externalLibraryWithImportPaths1);
         libraryMock.getAll.mockResolvedValue([libraryStub.externalLibraryWithImportPaths1]);
+        assetMock.getByLibraryIdAndOriginalPath.mockResolvedValue(assetStub.image);
         storageMock.watch.mockImplementation(makeMockWatcher({ items: [{ event: 'add', value: '/foo/photo.jpg' }] }));
 
         await sut.watchAll();
@@ -886,11 +1000,15 @@ describe(LibraryService.name, () => {
             },
           },
         ]);
+        expect(jobMock.queueAll).toHaveBeenCalledWith([
+          { name: JobName.LIBRARY_SYNC_ASSET, data: expect.objectContaining({ id: assetStub.image.id }) },
+        ]);
       });
 
       it('should handle a file change event', async () => {
         libraryMock.get.mockResolvedValue(libraryStub.externalLibraryWithImportPaths1);
         libraryMock.getAll.mockResolvedValue([libraryStub.externalLibraryWithImportPaths1]);
+        assetMock.getByLibraryIdAndOriginalPath.mockResolvedValue(assetStub.image);
         storageMock.watch.mockImplementation(
           makeMockWatcher({ items: [{ event: 'change', value: '/foo/photo.jpg' }] }),
         );
@@ -907,6 +1025,24 @@ describe(LibraryService.name, () => {
             },
           },
         ]);
+        expect(jobMock.queueAll).toHaveBeenCalledWith([
+          { name: JobName.LIBRARY_SYNC_ASSET, data: expect.objectContaining({ id: assetStub.image.id }) },
+        ]);
+      });
+
+      it('should handle a file unlink event', async () => {
+        libraryMock.get.mockResolvedValue(libraryStub.externalLibraryWithImportPaths1);
+        libraryMock.getAll.mockResolvedValue([libraryStub.externalLibraryWithImportPaths1]);
+        assetMock.getByLibraryIdAndOriginalPath.mockResolvedValue(assetStub.image);
+        storageMock.watch.mockImplementation(
+          makeMockWatcher({ items: [{ event: 'unlink', value: '/foo/photo.jpg' }] }),
+        );
+
+        await sut.watchAll();
+
+        expect(jobMock.queueAll).toHaveBeenCalledWith([
+          { name: JobName.LIBRARY_SYNC_ASSET, data: expect.objectContaining({ id: assetStub.image.id }) },
+        ]);
       });
 
       it('should handle an error event', async () => {
@@ -986,15 +1122,14 @@ describe(LibraryService.name, () => {
     it('should delete an empty library', async () => {
       libraryMock.get.mockResolvedValue(libraryStub.externalLibrary1);
       assetMock.getAll.mockResolvedValue({ items: [], hasNextPage: false });
-      libraryMock.delete.mockImplementation(async () => {});
 
       await expect(sut.handleDeleteLibrary({ id: libraryStub.externalLibrary1.id })).resolves.toBe(JobStatus.SUCCESS);
+      expect(libraryMock.delete).toHaveBeenCalled();
     });
 
-    it('should delete a library with assets', async () => {
+    it('should delete all assets in a library', async () => {
       libraryMock.get.mockResolvedValue(libraryStub.externalLibrary1);
       assetMock.getAll.mockResolvedValue({ items: [assetStub.image1], hasNextPage: false });
-      libraryMock.delete.mockImplementation(async () => {});
 
       assetMock.getById.mockResolvedValue(assetStub.image1);
 
@@ -1076,6 +1211,10 @@ describe(LibraryService.name, () => {
   });
 
   describe('validate', () => {
+    it('should not require import paths', async () => {
+      await expect(sut.validate('library-id', {})).resolves.toEqual({ importPaths: [] });
+    });
+
     it('should validate directory', async () => {
       storageMock.stat.mockResolvedValue({
         isDirectory: () => true,