diff --git a/.gitignore b/.gitignore index 537e048be2837..e0544ad8d5925 100644 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,5 @@ mobile/openapi/.openapi-generator/FILES open-api/typescript-sdk/build mobile/android/fastlane/report.xml mobile/ios/fastlane/report.xml + +vite.config.js.timestamp-* diff --git a/e2e/src/api/specs/library.e2e-spec.ts b/e2e/src/api/specs/library.e2e-spec.ts index 9f5adc4e27d92..fe0b4f2bd44bc 100644 --- a/e2e/src/api/specs/library.e2e-spec.ts +++ b/e2e/src/api/specs/library.e2e-spec.ts @@ -500,13 +500,13 @@ describe('/libraries', () => { }); it('should set an asset offline its file is not in any import path', async () => { + utils.createImageFile(`${testAssetDir}/temp/offline/offline.png`); + const library = await utils.createLibrary(admin.accessToken, { ownerId: admin.userId, importPaths: [`${testAssetDirInternal}/temp/offline`], }); - utils.createImageFile(`${testAssetDir}/temp/offline/offline.png`); - await scan(admin.accessToken, library.id); await utils.waitForQueueFinish(admin.accessToken, 'library'); diff --git a/e2e/src/utils.ts b/e2e/src/utils.ts index 52acd35a5c4b1..3af44b50b8330 100644 --- a/e2e/src/utils.ts +++ b/e2e/src/utils.ts @@ -374,8 +374,8 @@ export const utils = { }, createDirectory: (path: string) => { - if (!existsSync(dirname(path))) { - mkdirSync(dirname(path), { recursive: true }); + if (!existsSync(path)) { + mkdirSync(path, { recursive: true }); } }, @@ -392,7 +392,7 @@ export const utils = { return; } - rmSync(path); + rmSync(path, { recursive: true }); }, getAssetInfo: (accessToken: string, id: string) => getAssetInfo({ id }, { headers: asBearerAuth(accessToken) }), diff --git a/server/src/entities/asset-files.entity.ts b/server/src/entities/asset-files.entity.ts index dbed61136ce57..0488abd4b61dd 100644 --- a/server/src/entities/asset-files.entity.ts +++ b/server/src/entities/asset-files.entity.ts @@ -36,7 +36,7 @@ export class AssetFileEntity { @Column() path!: string; - @Column({ type: 'bytea' }) + @Column({ type: 'bytea', nullable: true, default: null }) @Index() checksum!: Buffer | null; } diff --git a/server/src/interfaces/asset.interface.ts b/server/src/interfaces/asset.interface.ts index bafef02c8c0fb..41b02f6487f35 100644 --- a/server/src/interfaces/asset.interface.ts +++ b/server/src/interfaces/asset.interface.ts @@ -195,6 +195,7 @@ export interface IAssetRepository { getDuplicates(options: AssetBuilderOptions): Promise; getAllForUserFullSync(options: AssetFullSyncOptions): Promise; getChangedDeltaSync(options: AssetDeltaSyncOptions): Promise; + removeFile(assetId: string, type: AssetFileType): Promise; upsertFile(file: UpsertFileOptions): Promise; upsertFiles(files: UpsertFileOptions[]): Promise; } diff --git a/server/src/migrations/1728632095015-AddAssetFileChecksum.ts b/server/src/migrations/1728632095015-AddAssetFileChecksum.ts index eeb3e4740f591..aa405dd53b867 100644 --- a/server/src/migrations/1728632095015-AddAssetFileChecksum.ts +++ b/server/src/migrations/1728632095015-AddAssetFileChecksum.ts @@ -4,7 +4,7 @@ export class AssetFileChecksum1728632095015 implements MigrationInterface { name = 'AssetFileChecksum1728632095015'; public async up(queryRunner: QueryRunner): Promise { - await queryRunner.query(`ALTER TABLE "asset_files" ADD "checksum" bytea NULL`); + await queryRunner.query(`ALTER TABLE "asset_files" ADD "checksum" bytea`); await queryRunner.query(`CREATE INDEX "IDX_c946066edd16cfa5c25a26aa8e" ON "asset_files" ("checksum")`); } diff --git a/server/src/queries/asset.repository.sql b/server/src/queries/asset.repository.sql index ca5acf5de4082..767ff45067f91 100644 --- a/server/src/queries/asset.repository.sql +++ b/server/src/queries/asset.repository.sql @@ -1134,7 +1134,8 @@ SET RETURNING "id", "createdAt", - "updatedAt" + "updatedAt", + "checksum" -- AssetRepository.upsertFiles INSERT INTO @@ -1159,4 +1160,5 @@ SET RETURNING "id", "createdAt", - "updatedAt" + "updatedAt", + "checksum" diff --git a/server/src/repositories/asset.repository.ts b/server/src/repositories/asset.repository.ts index f6a1be504a93d..078228df78984 100644 --- a/server/src/repositories/asset.repository.ts +++ b/server/src/repositories/asset.repository.ts @@ -767,6 +767,10 @@ export class AssetRepository implements IAssetRepository { return builder.getMany(); } + async removeFile(assetId: string, type: AssetFileType): Promise { + await this.fileRepository.delete({ assetId, type }); + } + @GenerateSql({ params: [{ assetId: DummyValue.UUID, type: AssetFileType.PREVIEW, path: '/path/to/file' }] }) async upsertFile(file: { assetId: string; type: AssetFileType; path: string; checksum?: Buffer }): Promise { await this.fileRepository.upsert(file, { conflictPaths: ['assetId', 'type'] }); diff --git a/server/src/services/library.service.spec.ts b/server/src/services/library.service.spec.ts index 7993c7daccd3c..b021eedbe901b 100644 --- a/server/src/services/library.service.spec.ts +++ b/server/src/services/library.service.spec.ts @@ -119,6 +119,64 @@ describe(LibraryService.name, () => { }); }); + describe('onConfigUpdateEvent', () => { + beforeEach(async () => { + systemMock.get.mockResolvedValue(defaults); + databaseMock.tryLock.mockResolvedValue(true); + await sut.onBootstrap(); + }); + + it('should do nothing if oldConfig is not provided', async () => { + await sut.onConfigUpdate({ newConfig: systemConfigStub.libraryScan as SystemConfig }); + expect(jobMock.updateCronJob).not.toHaveBeenCalled(); + }); + + it('should do nothing if instance does not have the watch lock', async () => { + databaseMock.tryLock.mockResolvedValue(false); + await sut.onBootstrap(); + await sut.onConfigUpdate({ newConfig: systemConfigStub.libraryScan as SystemConfig, oldConfig: defaults }); + expect(jobMock.updateCronJob).not.toHaveBeenCalled(); + }); + + it('should update cron job and enable watching', async () => { + libraryMock.getAll.mockResolvedValue([]); + await sut.onConfigUpdate({ + newConfig: { + library: { ...systemConfigStub.libraryScan.library, ...systemConfigStub.libraryWatchEnabled.library }, + } as SystemConfig, + oldConfig: defaults, + }); + + expect(jobMock.updateCronJob).toHaveBeenCalledWith( + 'libraryScan', + systemConfigStub.libraryScan.library.scan.cronExpression, + systemConfigStub.libraryScan.library.scan.enabled, + ); + }); + + it('should update cron job and disable watching', async () => { + libraryMock.getAll.mockResolvedValue([]); + await sut.onConfigUpdate({ + newConfig: { + library: { ...systemConfigStub.libraryScan.library, ...systemConfigStub.libraryWatchEnabled.library }, + } as SystemConfig, + oldConfig: defaults, + }); + await sut.onConfigUpdate({ + newConfig: { + library: { ...systemConfigStub.libraryScan.library, ...systemConfigStub.libraryWatchDisabled.library }, + } as SystemConfig, + oldConfig: defaults, + }); + + expect(jobMock.updateCronJob).toHaveBeenCalledWith( + 'libraryScan', + systemConfigStub.libraryScan.library.scan.cronExpression, + systemConfigStub.libraryScan.library.scan.enabled, + ); + }); + }); + describe('onConfigValidateEvent', () => { it('should allow a valid cron expression', () => { expect(() => @@ -139,7 +197,7 @@ describe(LibraryService.name, () => { }); }); - describe('handleQueueAssetRefresh', () => { + describe('handleQueueSyncFiles', () => { it('should queue refresh of a new asset', async () => { libraryMock.get.mockResolvedValue(libraryStub.externalLibrary1); storageMock.walk.mockImplementation(mockWalk); @@ -559,8 +617,8 @@ describe(LibraryService.name, () => { expect(jobMock.queueAll).not.toHaveBeenCalled(); }); - it('should throw BadRequestException when asset does not exist', async () => { - storageMock.stat.mockRejectedValue(new Error("ENOENT, no such file or directory '/data/user1/photo.jpg'")); + it('should fail when the file could not be read', async () => { + storageMock.stat.mockRejectedValue(new Error('Could not read file')); const mockLibraryJob: ILibraryFileJob = { id: libraryStub.externalLibrary1.id, @@ -572,6 +630,27 @@ describe(LibraryService.name, () => { assetMock.create.mockResolvedValue(assetStub.image); await expect(sut.handleSyncFile(mockLibraryJob)).resolves.toBe(JobStatus.FAILED); + expect(libraryMock.get).not.toHaveBeenCalled(); + expect(assetMock.create).not.toHaveBeenCalled(); + }); + + it('should skip if the file could not be found', async () => { + const error = new Error('File not found') as any; + error.code = 'ENOENT'; + storageMock.stat.mockRejectedValue(error); + + const mockLibraryJob: ILibraryFileJob = { + id: libraryStub.externalLibrary1.id, + ownerId: userStub.admin.id, + assetPath: '/data/user1/photo.jpg', + }; + + assetMock.getByLibraryIdAndOriginalPath.mockResolvedValue(null); + assetMock.create.mockResolvedValue(assetStub.image); + + await expect(sut.handleSyncFile(mockLibraryJob)).resolves.toBe(JobStatus.SKIPPED); + expect(libraryMock.get).not.toHaveBeenCalled(); + expect(assetMock.create).not.toHaveBeenCalled(); }); }); @@ -654,6 +733,10 @@ describe(LibraryService.name, () => { expect(libraryMock.getStatistics).toHaveBeenCalledWith(libraryStub.externalLibrary1.id); }); + + it('should throw an error if the library could not be found', async () => { + await expect(sut.getStatistics('foo')).rejects.toBeInstanceOf(BadRequestException); + }); }); describe('create', () => { @@ -783,6 +866,13 @@ describe(LibraryService.name, () => { }); }); + describe('getAll', () => { + it('should get all libraries', async () => { + libraryMock.getAll.mockResolvedValue([libraryStub.externalLibrary1]); + await expect(sut.getAll()).resolves.toEqual([expect.objectContaining({ id: libraryStub.externalLibrary1.id })]); + }); + }); + describe('handleQueueCleanup', () => { it('should queue cleanup jobs', async () => { libraryMock.getAllDeleted.mockResolvedValue([libraryStub.externalLibrary1, libraryStub.externalLibrary2]); @@ -803,15 +893,38 @@ describe(LibraryService.name, () => { await sut.onBootstrap(); }); + it('should throw an error if an import path is invalid', async () => { + libraryMock.update.mockResolvedValue(libraryStub.externalLibrary1); + libraryMock.get.mockResolvedValue(libraryStub.externalLibrary1); + + await expect(sut.update('library-id', { importPaths: ['foo/bar'] })).rejects.toBeInstanceOf(BadRequestException); + expect(libraryMock.update).not.toHaveBeenCalled(); + }); + it('should update library', async () => { libraryMock.update.mockResolvedValue(libraryStub.externalLibrary1); libraryMock.get.mockResolvedValue(libraryStub.externalLibrary1); - await expect(sut.update('library-id', {})).resolves.toEqual(mapLibrary(libraryStub.externalLibrary1)); + storageMock.stat.mockResolvedValue({ isDirectory: () => true } as Stats); + storageMock.checkFileExists.mockResolvedValue(true); + + await expect(sut.update('library-id', { importPaths: ['foo/bar'] })).resolves.toEqual( + mapLibrary(libraryStub.externalLibrary1), + ); expect(libraryMock.update).toHaveBeenCalledWith(expect.objectContaining({ id: 'library-id' })); }); }); + describe('onShutdown', () => { + it('should do nothing if instance does not have the watch lock', async () => { + await sut.onShutdown(); + }); + }); + describe('watchAll', () => { + it('should return false if instance does not have the watch lock', async () => { + await expect(sut.watchAll()).resolves.toBe(false); + }); + describe('watching disabled', () => { beforeEach(async () => { systemMock.get.mockResolvedValue(systemConfigStub.libraryWatchDisabled); @@ -872,6 +985,7 @@ describe(LibraryService.name, () => { it('should handle a new file event', async () => { libraryMock.get.mockResolvedValue(libraryStub.externalLibraryWithImportPaths1); libraryMock.getAll.mockResolvedValue([libraryStub.externalLibraryWithImportPaths1]); + assetMock.getByLibraryIdAndOriginalPath.mockResolvedValue(assetStub.image); storageMock.watch.mockImplementation(makeMockWatcher({ items: [{ event: 'add', value: '/foo/photo.jpg' }] })); await sut.watchAll(); @@ -886,11 +1000,15 @@ describe(LibraryService.name, () => { }, }, ]); + expect(jobMock.queueAll).toHaveBeenCalledWith([ + { name: JobName.LIBRARY_SYNC_ASSET, data: expect.objectContaining({ id: assetStub.image.id }) }, + ]); }); it('should handle a file change event', async () => { libraryMock.get.mockResolvedValue(libraryStub.externalLibraryWithImportPaths1); libraryMock.getAll.mockResolvedValue([libraryStub.externalLibraryWithImportPaths1]); + assetMock.getByLibraryIdAndOriginalPath.mockResolvedValue(assetStub.image); storageMock.watch.mockImplementation( makeMockWatcher({ items: [{ event: 'change', value: '/foo/photo.jpg' }] }), ); @@ -907,6 +1025,24 @@ describe(LibraryService.name, () => { }, }, ]); + expect(jobMock.queueAll).toHaveBeenCalledWith([ + { name: JobName.LIBRARY_SYNC_ASSET, data: expect.objectContaining({ id: assetStub.image.id }) }, + ]); + }); + + it('should handle a file unlink event', async () => { + libraryMock.get.mockResolvedValue(libraryStub.externalLibraryWithImportPaths1); + libraryMock.getAll.mockResolvedValue([libraryStub.externalLibraryWithImportPaths1]); + assetMock.getByLibraryIdAndOriginalPath.mockResolvedValue(assetStub.image); + storageMock.watch.mockImplementation( + makeMockWatcher({ items: [{ event: 'unlink', value: '/foo/photo.jpg' }] }), + ); + + await sut.watchAll(); + + expect(jobMock.queueAll).toHaveBeenCalledWith([ + { name: JobName.LIBRARY_SYNC_ASSET, data: expect.objectContaining({ id: assetStub.image.id }) }, + ]); }); it('should handle an error event', async () => { @@ -986,15 +1122,14 @@ describe(LibraryService.name, () => { it('should delete an empty library', async () => { libraryMock.get.mockResolvedValue(libraryStub.externalLibrary1); assetMock.getAll.mockResolvedValue({ items: [], hasNextPage: false }); - libraryMock.delete.mockImplementation(async () => {}); await expect(sut.handleDeleteLibrary({ id: libraryStub.externalLibrary1.id })).resolves.toBe(JobStatus.SUCCESS); + expect(libraryMock.delete).toHaveBeenCalled(); }); - it('should delete a library with assets', async () => { + it('should delete all assets in a library', async () => { libraryMock.get.mockResolvedValue(libraryStub.externalLibrary1); assetMock.getAll.mockResolvedValue({ items: [assetStub.image1], hasNextPage: false }); - libraryMock.delete.mockImplementation(async () => {}); assetMock.getById.mockResolvedValue(assetStub.image1); @@ -1076,6 +1211,10 @@ describe(LibraryService.name, () => { }); describe('validate', () => { + it('should not require import paths', async () => { + await expect(sut.validate('library-id', {})).resolves.toEqual({ importPaths: [] }); + }); + it('should validate directory', async () => { storageMock.stat.mockResolvedValue({ isDirectory: () => true, diff --git a/server/src/services/library.service.ts b/server/src/services/library.service.ts index 0cd939dd36967..0a85750768ff1 100644 --- a/server/src/services/library.service.ts +++ b/server/src/services/library.service.ts @@ -16,7 +16,7 @@ import { } from 'src/dtos/library.dto'; import { AssetEntity } from 'src/entities/asset.entity'; import { LibraryEntity } from 'src/entities/library.entity'; -import { AssetType } from 'src/enum'; +import { AssetFileType, AssetType } from 'src/enum'; import { DatabaseLock } from 'src/interfaces/database.interface'; import { ArgOf } from 'src/interfaces/event.interface'; import { @@ -303,7 +303,6 @@ export class LibraryService extends BaseService { async update(id: string, dto: UpdateLibraryDto): Promise { await this.findOrFail(id); - const library = await this.libraryRepository.update({ id, ...dto }); if (dto.importPaths) { const validation = await this.validate(id, { importPaths: dto.importPaths }); @@ -316,6 +315,7 @@ export class LibraryService extends BaseService { } } + const library = await this.libraryRepository.update({ id, ...dto }); return mapLibrary(library); } @@ -424,6 +424,8 @@ export class LibraryService extends BaseService { isExternal: true, }); + await this.assetRepository.upsertFile({ assetId: asset.id, type: AssetFileType.ORIGINAL, path: assetPath }); + await this.queuePostSyncJobs(asset); return JobStatus.SUCCESS; @@ -482,6 +484,7 @@ export class LibraryService extends BaseService { if (!asset.isOffline) { this.logger.debug(`${explanation}, removing: ${asset.originalPath}`); await this.assetRepository.updateAll([asset.id], { isOffline: true, deletedAt: new Date() }); + await this.assetRepository.removeFile(asset.id, AssetFileType.ORIGINAL); } }; @@ -518,6 +521,12 @@ export class LibraryService extends BaseService { fileModifiedAt: mtime, originalFileName: parse(asset.originalPath).base, }); + + await this.assetRepository.upsertFile({ + assetId: asset.id, + type: AssetFileType.ORIGINAL, + path: asset.originalPath, + }); } if (isAssetModified) {