Skip to content

Commit

Permalink
refactor(server): db env (immich-app#13167)
Browse files Browse the repository at this point in the history
  • Loading branch information
jrasm91 authored Oct 3, 2024
1 parent e2bf680 commit 3ac00b0
Show file tree
Hide file tree
Showing 11 changed files with 84 additions and 31 deletions.
18 changes: 8 additions & 10 deletions server/src/database.config.ts
Original file line number Diff line number Diff line change
@@ -1,16 +1,17 @@
import { DatabaseExtension } from 'src/interfaces/database.interface';
import { ConfigRepository } from 'src/repositories/config.repository';
import { DataSource } from 'typeorm';
import { PostgresConnectionOptions } from 'typeorm/driver/postgres/PostgresConnectionOptions.js';

const url = process.env.DB_URL;
const { database } = new ConfigRepository().getEnv();
const { url, host, port, username, password, name } = database;
const urlOrParts = url
? { url }
: {
host: process.env.DB_HOSTNAME || 'database',
port: Number.parseInt(process.env.DB_PORT || '5432'),
username: process.env.DB_USERNAME || 'postgres',
password: process.env.DB_PASSWORD || 'postgres',
database: process.env.DB_DATABASE_NAME || 'immich',
host,
port,
username,
password,
database: name,
};

/* eslint unicorn/prefer-module: "off" -- We can fix this when migrating to ESM*/
Expand All @@ -32,6 +33,3 @@ export const databaseConfig: PostgresConnectionOptions = {
* this export is ONLY to be used for TypeORM commands in package.json#scripts
*/
export const dataSource = new DataSource({ ...databaseConfig, host: 'localhost' });

export const getVectorExtension = () =>
process.env.DB_VECTOR_EXTENSION === 'pgvector' ? DatabaseExtension.VECTOR : DatabaseExtension.VECTORS;
6 changes: 6 additions & 0 deletions server/src/interfaces/config.interface.ts
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,12 @@ export interface EnvData {
};

database: {
url?: string;
host: string;
port: number;
username: string;
password: string;
name: string;
skipMigrations: boolean;
vectorExtension: VectorExtension;
};
Expand Down
6 changes: 4 additions & 2 deletions server/src/migrations/1700713871511-UsePgVectors.ts
Original file line number Diff line number Diff line change
@@ -1,13 +1,15 @@
import { getVectorExtension } from 'src/database.config';
import { ConfigRepository } from 'src/repositories/config.repository';
import { getCLIPModelInfo } from 'src/utils/misc';
import { MigrationInterface, QueryRunner } from 'typeorm';

const vectorExtension = new ConfigRepository().getEnv().database.vectorExtension;

export class UsePgVectors1700713871511 implements MigrationInterface {
name = 'UsePgVectors1700713871511';

public async up(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(`SET search_path TO "$user", public, vectors`);
await queryRunner.query(`CREATE EXTENSION IF NOT EXISTS ${getVectorExtension()}`);
await queryRunner.query(`CREATE EXTENSION IF NOT EXISTS ${vectorExtension}`);
const faceDimQuery = await queryRunner.query(`
SELECT CARDINALITY(embedding::real[]) as dimsize
FROM asset_faces
Expand Down
6 changes: 4 additions & 2 deletions server/src/migrations/1700713994428-AddCLIPEmbeddingIndex.ts
Original file line number Diff line number Diff line change
@@ -1,12 +1,14 @@
import { getVectorExtension } from 'src/database.config';
import { DatabaseExtension } from 'src/interfaces/database.interface';
import { ConfigRepository } from 'src/repositories/config.repository';
import { MigrationInterface, QueryRunner } from 'typeorm';

const vectorExtension = new ConfigRepository().getEnv().database.vectorExtension;

export class AddCLIPEmbeddingIndex1700713994428 implements MigrationInterface {
name = 'AddCLIPEmbeddingIndex1700713994428';

public async up(queryRunner: QueryRunner): Promise<void> {
if (getVectorExtension() === DatabaseExtension.VECTORS) {
if (vectorExtension === DatabaseExtension.VECTORS) {
await queryRunner.query(`SET vectors.pgvector_compatibility=on`);
}
await queryRunner.query(`SET search_path TO "$user", public, vectors`);
Expand Down
6 changes: 4 additions & 2 deletions server/src/migrations/1700714033632-AddFaceEmbeddingIndex.ts
Original file line number Diff line number Diff line change
@@ -1,12 +1,14 @@
import { getVectorExtension } from 'src/database.config';
import { DatabaseExtension } from 'src/interfaces/database.interface';
import { ConfigRepository } from 'src/repositories/config.repository';
import { MigrationInterface, QueryRunner } from 'typeorm';

const vectorExtension = new ConfigRepository().getEnv().database.vectorExtension;

export class AddFaceEmbeddingIndex1700714033632 implements MigrationInterface {
name = 'AddFaceEmbeddingIndex1700714033632';

public async up(queryRunner: QueryRunner): Promise<void> {
if (getVectorExtension() === DatabaseExtension.VECTORS) {
if (vectorExtension === DatabaseExtension.VECTORS) {
await queryRunner.query(`SET vectors.pgvector_compatibility=on`);
}
await queryRunner.query(`SET search_path TO "$user", public, vectors`);
Expand Down
15 changes: 9 additions & 6 deletions server/src/migrations/1718486162779-AddFaceSearchRelation.ts
Original file line number Diff line number Diff line change
@@ -1,10 +1,12 @@
import { getVectorExtension } from 'src/database.config';
import { DatabaseExtension } from 'src/interfaces/database.interface';
import { ConfigRepository } from 'src/repositories/config.repository';
import { MigrationInterface, QueryRunner } from 'typeorm';

const vectorExtension = new ConfigRepository().getEnv().database.vectorExtension;

export class AddFaceSearchRelation1718486162779 implements MigrationInterface {
public async up(queryRunner: QueryRunner): Promise<void> {
if (getVectorExtension() === DatabaseExtension.VECTORS) {
if (vectorExtension === DatabaseExtension.VECTORS) {
await queryRunner.query(`SET search_path TO "$user", public, vectors`);
await queryRunner.query(`SET vectors.pgvector_compatibility=on`);
}
Expand All @@ -13,9 +15,10 @@ export class AddFaceSearchRelation1718486162779 implements MigrationInterface {
const columns = await queryRunner.query(
`SELECT column_name as name
FROM information_schema.columns
WHERE table_name = '${tableName}'`);
WHERE table_name = '${tableName}'`,
);
return columns.some((column: { name: string }) => column.name === 'embedding');
}
};

const hasAssetEmbeddings = await hasEmbeddings('smart_search');
if (!hasAssetEmbeddings) {
Expand All @@ -31,7 +34,7 @@ export class AddFaceSearchRelation1718486162779 implements MigrationInterface {
await queryRunner.query(`ALTER TABLE face_search ALTER COLUMN embedding SET STORAGE EXTERNAL`);
await queryRunner.query(`ALTER TABLE smart_search ALTER COLUMN embedding SET STORAGE EXTERNAL`);

const hasFaceEmbeddings = await hasEmbeddings('asset_faces')
const hasFaceEmbeddings = await hasEmbeddings('asset_faces');
if (hasFaceEmbeddings) {
await queryRunner.query(`
INSERT INTO face_search("faceId", embedding)
Expand All @@ -56,7 +59,7 @@ export class AddFaceSearchRelation1718486162779 implements MigrationInterface {
}

public async down(queryRunner: QueryRunner): Promise<void> {
if (getVectorExtension() === DatabaseExtension.VECTORS) {
if (vectorExtension === DatabaseExtension.VECTORS) {
await queryRunner.query(`SET search_path TO "$user", public, vectors`);
await queryRunner.query(`SET vectors.pgvector_compatibility=on`);
}
Expand Down
12 changes: 10 additions & 2 deletions server/src/repositories/config.repository.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import { Injectable } from '@nestjs/common';
import { getVectorExtension } from 'src/database.config';
import { ImmichEnvironment, ImmichWorker, LogLevel } from 'src/enum';
import { EnvData, IConfigRepository } from 'src/interfaces/config.interface';
import { DatabaseExtension } from 'src/interfaces/database.interface';
import { setDifference } from 'src/utils/set';

// TODO replace src/config validation with class-validator, here
Expand Down Expand Up @@ -65,8 +65,16 @@ export class ConfigRepository implements IConfigRepository {
},

database: {
url: process.env.DB_URL,
host: process.env.DB_HOSTNAME || 'database',
port: Number(process.env.DB_PORT) || 5432,
username: process.env.DB_USERNAME || 'postgres',
password: process.env.DB_PASSWORD || 'postgres',
name: process.env.DB_DATABASE_NAME || 'immich',

skipMigrations: process.env.DB_SKIP_MIGRATIONS === 'true',
vectorExtension: getVectorExtension(),
vectorExtension:
process.env.DB_VECTOR_EXTENSION === 'pgvector' ? DatabaseExtension.VECTOR : DatabaseExtension.VECTORS,
},

licensePublicKey: isProd ? productionKeys : stagingKeys,
Expand Down
9 changes: 6 additions & 3 deletions server/src/repositories/database.repository.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import { InjectDataSource } from '@nestjs/typeorm';
import AsyncLock from 'async-lock';
import semver from 'semver';
import { POSTGRES_VERSION_RANGE, VECTOR_VERSION_RANGE, VECTORS_VERSION_RANGE } from 'src/constants';
import { getVectorExtension } from 'src/database.config';
import { IConfigRepository } from 'src/interfaces/config.interface';
import {
DatabaseExtension,
DatabaseLock,
Expand All @@ -22,12 +22,15 @@ import { DataSource, EntityManager, QueryRunner } from 'typeorm';
@Instrumentation()
@Injectable()
export class DatabaseRepository implements IDatabaseRepository {
private vectorExtension: VectorExtension;
readonly asyncLock = new AsyncLock();

constructor(
@InjectDataSource() private dataSource: DataSource,
@Inject(ILoggerRepository) private logger: ILoggerRepository,
@Inject(IConfigRepository) configRepository: IConfigRepository,
) {
this.vectorExtension = configRepository.getEnv().database.vectorExtension;
this.logger.setContext(DatabaseRepository.name);
}

Expand Down Expand Up @@ -119,7 +122,7 @@ export class DatabaseRepository implements IDatabaseRepository {
try {
await this.dataSource.query(`REINDEX INDEX ${index}`);
} catch (error) {
if (getVectorExtension() !== DatabaseExtension.VECTORS) {
if (this.vectorExtension !== DatabaseExtension.VECTORS) {
throw error;
}
this.logger.warn(`Could not reindex index ${index}. Attempting to auto-fix.`);
Expand All @@ -141,7 +144,7 @@ export class DatabaseRepository implements IDatabaseRepository {
}

async shouldReindex(name: VectorIndex): Promise<boolean> {
if (getVectorExtension() !== DatabaseExtension.VECTORS) {
if (this.vectorExtension !== DatabaseExtension.VECTORS) {
return false;
}

Expand Down
9 changes: 6 additions & 3 deletions server/src/repositories/search.repository.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
import { Inject, Injectable } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { randomUUID } from 'node:crypto';
import { getVectorExtension } from 'src/database.config';
import { DummyValue, GenerateSql } from 'src/decorators';
import { AssetFaceEntity } from 'src/entities/asset-face.entity';
import { AssetEntity } from 'src/entities/asset.entity';
Expand All @@ -10,7 +9,8 @@ import { GeodataPlacesEntity } from 'src/entities/geodata-places.entity';
import { SmartInfoEntity } from 'src/entities/smart-info.entity';
import { SmartSearchEntity } from 'src/entities/smart-search.entity';
import { AssetType, PaginationMode } from 'src/enum';
import { DatabaseExtension } from 'src/interfaces/database.interface';
import { IConfigRepository } from 'src/interfaces/config.interface';
import { DatabaseExtension, VectorExtension } from 'src/interfaces/database.interface';
import { ILoggerRepository } from 'src/interfaces/logger.interface';
import {
AssetDuplicateResult,
Expand All @@ -31,6 +31,7 @@ import { Repository, SelectQueryBuilder } from 'typeorm';
@Instrumentation()
@Injectable()
export class SearchRepository implements ISearchRepository {
private vectorExtension: VectorExtension;
private faceColumns: string[];
private assetsByCityQuery: string;

Expand All @@ -42,7 +43,9 @@ export class SearchRepository implements ISearchRepository {
@InjectRepository(SmartSearchEntity) private smartSearchRepository: Repository<SmartSearchEntity>,
@InjectRepository(GeodataPlacesEntity) private geodataPlacesRepository: Repository<GeodataPlacesEntity>,
@Inject(ILoggerRepository) private logger: ILoggerRepository,
@Inject(IConfigRepository) configRepository: IConfigRepository,
) {
this.vectorExtension = configRepository.getEnv().database.vectorExtension;
this.logger.setContext(SearchRepository.name);
this.faceColumns = this.assetFaceRepository.manager.connection
.getMetadata(AssetFaceEntity)
Expand Down Expand Up @@ -440,7 +443,7 @@ export class SearchRepository implements ISearchRepository {
}

private getRuntimeConfig(numResults?: number): string {
if (getVectorExtension() === DatabaseExtension.VECTOR) {
if (this.vectorExtension === DatabaseExtension.VECTOR) {
return 'SET LOCAL hnsw.ef_search = 1000;'; // mitigate post-filter recall
}

Expand Down
22 changes: 21 additions & 1 deletion server/src/services/database.service.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,17 @@ describe(DatabaseService.name, () => {
])('should work with $extensionName', ({ extension, extensionName }) => {
beforeEach(() => {
configMock.getEnv.mockReturnValue(
mockEnvData({ database: { skipMigrations: false, vectorExtension: extension } }),
mockEnvData({
database: {
host: 'database',
port: 5432,
username: 'postgres',
password: 'postgres',
name: 'immich',
skipMigrations: false,
vectorExtension: extension,
},
}),
);
});

Expand Down Expand Up @@ -245,6 +255,11 @@ describe(DatabaseService.name, () => {
configMock.getEnv.mockReturnValue(
mockEnvData({
database: {
host: 'database',
port: 5432,
username: 'postgres',
password: 'postgres',
name: 'immich',
skipMigrations: true,
vectorExtension: DatabaseExtension.VECTORS,
},
Expand All @@ -260,6 +275,11 @@ describe(DatabaseService.name, () => {
configMock.getEnv.mockReturnValue(
mockEnvData({
database: {
host: 'database',
port: 5432,
username: 'postgres',
password: 'postgres',
name: 'immich',
skipMigrations: true,
vectorExtension: DatabaseExtension.VECTOR,
},
Expand Down
6 changes: 6 additions & 0 deletions server/test/repositories/config.repository.mock.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,12 @@ const envData: EnvData = {
buildMetadata: {},

database: {
host: 'database',
port: 5432,
username: 'postgres',
password: 'postgres',
name: 'immich',

skipMigrations: false,
vectorExtension: DatabaseExtension.VECTORS,
},
Expand Down

0 comments on commit 3ac00b0

Please sign in to comment.