Skip to content

Commit

Permalink
move to a separate package instead and address comments
Browse files Browse the repository at this point in the history
Signed-off-by: Fredrik Adelöw <freben@gmail.com>
  • Loading branch information
freben committed Nov 11, 2021
1 parent 222793c commit e09bf60
Show file tree
Hide file tree
Showing 25 changed files with 229 additions and 101 deletions.
2 changes: 1 addition & 1 deletion .changeset/happy-rice-tickle.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,4 +2,4 @@
'@backstage/backend-common': patch
---

Add support for distributed mutexes and scheduled tasks, through the `TaskManager` class. This class can be particularly useful for coordinating things across many deployed instances of a given backend plugin. An example of this is catalog entity providers - with this facility you can register tasks similar to a cron job, and make sure that only one host at a time tries to execute the job, and that the timing (call frequency, timeouts etc) are retained as a global concern, letting you scale your workload safely without affecting the task behavior.
Added the `isDatabaseConflictError` function.
50 changes: 0 additions & 50 deletions packages/backend-common/api-report.md
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@ import { BitbucketIntegration } from '@backstage/integration';
import { Config } from '@backstage/config';
import cors from 'cors';
import Docker from 'dockerode';
import { Duration } from 'luxon';
import { ErrorRequestHandler } from 'express';
import express from 'express';
import { GithubCredentialsProvider } from '@backstage/integration';
Expand Down Expand Up @@ -384,11 +383,6 @@ export function loadBackendConfig(options: {
argv: string[];
}): Promise<Config>;

// @public
export interface LockOptions {
timeout: Duration;
}

// @public
export function notFoundHandler(): RequestHandler;

Expand All @@ -408,29 +402,6 @@ export type PluginEndpointDiscovery = {
getExternalBaseUrl(pluginId: string): Promise<string>;
};

// @public
export interface PluginTaskManager {
acquireLock(
id: string,
options: LockOptions,
): Promise<
| {
acquired: false;
}
| {
acquired: true;
release(): Promise<void>;
}
>;
scheduleTask(
id: string,
options: TaskOptions,
fn: () => void | Promise<void>,
): Promise<{
unschedule: () => Promise<void>;
}>;
}

// @public
export type ReaderFactory = (options: {
config: Config;
Expand Down Expand Up @@ -608,27 +579,6 @@ export interface StatusCheckHandlerOptions {
statusCheck?: StatusCheck;
}

// @public
export class TaskManager {
constructor(databaseManager: DatabaseManager, logger: Logger_2);
forPlugin(pluginId: string): PluginTaskManager;
// (undocumented)
static fromConfig(
config: Config,
options?: {
databaseManager?: DatabaseManager;
logger?: Logger_2;
},
): TaskManager;
}

// @public
export interface TaskOptions {
frequency?: Duration;
initialDelay?: Duration;
timeout?: Duration;
}

// @public
export type UrlReader = {
read(url: string): Promise<Buffer>;
Expand Down
11 changes: 2 additions & 9 deletions packages/backend-common/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,6 @@
"@types/cors": "^2.8.6",
"@types/dockerode": "^3.2.1",
"@types/express": "^4.17.6",
"@types/luxon": "^2.0.4",
"archiver": "^5.0.2",
"aws-sdk": "^2.840.0",
"compression": "^1.7.4",
Expand All @@ -60,7 +59,6 @@
"knex": "^0.95.1",
"lodash": "^4.17.21",
"logform": "^2.1.1",
"luxon": "^2.0.2",
"minimatch": "^3.0.4",
"minimist": "^1.2.5",
"morgan": "^1.10.0",
Expand All @@ -69,10 +67,8 @@
"stoppable": "^1.1.0",
"tar": "^6.1.2",
"unzipper": "^0.10.11",
"uuid": "^8.0.0",
"winston": "^3.2.1",
"yn": "^4.0.0",
"zod": "^3.9.5"
"yn": "^4.0.0"
},
"peerDependencies": {
"pg-connection-string": "^2.3.0"
Expand All @@ -85,7 +81,6 @@
"devDependencies": {
"@backstage/backend-test-utils": "^0.1.8",
"@backstage/cli": "^0.8.2",
"@backstage/test-utils": "^0.1.21",
"@types/archiver": "^5.1.0",
"@types/compression": "^1.7.0",
"@types/concat-stream": "^1.6.0",
Expand All @@ -108,12 +103,10 @@
"msw": "^0.35.0",
"mysql2": "^2.2.5",
"recursive-readdir": "^2.2.2",
"supertest": "^6.1.3",
"wait-for-expect": "^3.0.2"
"supertest": "^6.1.3"
},
"files": [
"dist",
"migrations/**/*.{js,d.ts}",
"config.d.ts"
],
"configSchema": "config.d.ts"
Expand Down
1 change: 0 additions & 1 deletion packages/backend-common/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -31,5 +31,4 @@ export * from './paths';
export * from './reading';
export * from './scm';
export * from './service';
export * from './tasks';
export * from './util';
3 changes: 3 additions & 0 deletions packages/backend-tasks/.eslintrc.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
module.exports = {
extends: [require.resolve('@backstage/cli/config/eslint.backend')],
};
36 changes: 36 additions & 0 deletions packages/backend-tasks/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
# @backstage/backend-tasks

Common distributed task management / locking library for Backstage backends.

## Usage

Add the library to your backend package:

```sh
# From your Backstage root directory
cd packages/backend
yarn add @backstage/backend-tasks
```

then make use of its facilities as necessary:

```typescript
import { TaskManager } from '@backstage/backend-tasks';

const manager = TaskManager.fromConfig(rootConfig).forPlugin('my-plugin');

const { unschedule } = await manager.scheduleTask(
'refresh-things',
{
frequency: Duration.fromObject({ minutes: 10 }),
},
async () => {
await entityProvider.run();
},
);
```

## Documentation

- [Backstage Readme](https://github.com/backstage/backstage/blob/master/README.md)
- [Backstage Documentation](https://github.com/backstage/backstage/blob/master/docs/README.md)
59 changes: 59 additions & 0 deletions packages/backend-tasks/api-report.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
## API Report File for "@backstage/backend-tasks"

> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
```ts
import { Config } from '@backstage/config';
import { DatabaseManager } from '@backstage/backend-common';
import { Duration } from 'luxon';
import { Logger as Logger_2 } from 'winston';

// @public
export interface LockOptions {
timeout: Duration;
}

// @public
export interface PluginTaskManager {
acquireLock(
id: string,
options: LockOptions,
): Promise<
| {
acquired: false;
}
| {
acquired: true;
release(): Promise<void>;
}
>;
scheduleTask(
id: string,
options: TaskOptions,
fn: () => void | Promise<void>,
): Promise<{
unschedule: () => Promise<void>;
}>;
}

// @public
export class TaskManager {
constructor(databaseManager: DatabaseManager, logger: Logger_2);
forPlugin(pluginId: string): PluginTaskManager;
// (undocumented)
static fromConfig(
config: Config,
options?: {
databaseManager?: DatabaseManager;
logger?: Logger_2;
},
): TaskManager;
}

// @public
export interface TaskOptions {
frequency?: Duration;
initialDelay?: Duration;
timeout?: Duration;
}
```
File renamed without changes.
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ exports.up = async function up(knex) {
//
// mutexes
//
await knex.schema.createTable('backstage_backend_common__mutexes', table => {
await knex.schema.createTable('backstage_backend_tasks__mutexes', table => {
table.comment('Locks used for mutual exclusion among multiple workers');
table
.text('id')
Expand All @@ -32,7 +32,7 @@ exports.up = async function up(knex) {
.comment('The unique ID of this particular mutex');
table
.text('current_lock_ticket')
.nullable()
.notNullable()
.comment('A unique ticket for the current mutex lock');
table
.dateTime('current_lock_acquired_at')
Expand All @@ -42,12 +42,12 @@ exports.up = async function up(knex) {
.dateTime('current_lock_expires_at')
.nullable()
.comment('The time when a locked mutex will time out and auto-release');
table.index(['id'], 'backstage_backend_common__mutexes__id_idx');
table.index(['id'], 'backstage_backend_tasks__mutexes__id_idx');
});
//
// tasks
//
await knex.schema.createTable('backstage_backend_common__tasks', table => {
await knex.schema.createTable('backstage_backend_tasks__tasks', table => {
table.comment('Tasks used for scheduling work on multiple workers');
table
.text('id')
Expand All @@ -74,7 +74,7 @@ exports.up = async function up(knex) {
.dateTime('current_run_expires_at')
.nullable()
.comment('The time that the current task run will time out');
table.index(['id'], 'backstage_backend_common__tasks__id_idx');
table.index(['id'], 'backstage_backend_tasks__tasks__id_idx');
});
};

Expand All @@ -85,18 +85,15 @@ exports.down = async function down(knex) {
//
// tasks
//
await knex.schema.alterTable('backstage_backend_common__tasks', table => {
table.dropIndex([], 'backstage_backend_common__tasks__id_idx');
await knex.schema.alterTable('backstage_backend_tasks__tasks', table => {
table.dropIndex([], 'backstage_backend_tasks__tasks__id_idx');
});
await knex.schema.dropTable('backstage_backend_common__tasks');
await knex.schema.dropTable('backstage_backend_tasks__tasks');
//
// locks
//
await knex.schema.alterTable(
'backstage_backend_common__task_locks',
table => {
table.dropIndex([], 'backstage_backend_common__task_locks__id_idx');
},
);
await knex.schema.dropTable('backstage_backend_common__task_locks');
await knex.schema.alterTable('backstage_backend_tasks__task_locks', table => {
table.dropIndex([], 'backstage_backend_tasks__task_locks__id_idx');
});
await knex.schema.dropTable('backstage_backend_tasks__task_locks');
};
54 changes: 54 additions & 0 deletions packages/backend-tasks/package.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
{
"name": "@backstage/backend-tasks",
"description": "Common distributed task management / locking library for Backstage backends",
"version": "0.1.0",
"main": "src/index.ts",
"types": "src/index.ts",
"private": false,
"publishConfig": {
"access": "public",
"main": "dist/index.cjs.js",
"types": "dist/index.d.ts"
},
"homepage": "https://backstage.io",
"repository": {
"type": "git",
"url": "https://github.com/backstage/backstage",
"directory": "packages/backend-tasks"
},
"keywords": [
"backstage"
],
"license": "Apache-2.0",
"scripts": {
"build": "backstage-cli build --outputs cjs,types",
"lint": "backstage-cli lint",
"test": "backstage-cli test",
"prepack": "backstage-cli prepack",
"postpack": "backstage-cli postpack",
"clean": "backstage-cli clean"
},
"dependencies": {
"@backstage/backend-common": "^0.9.7",
"@backstage/config": "^0.1.10",
"@backstage/errors": "^0.1.3",
"@backstage/types": "^0.1.1",
"@types/luxon": "^2.0.4",
"knex": "^0.95.1",
"lodash": "^4.17.21",
"luxon": "^2.0.2",
"uuid": "^8.0.0",
"winston": "^3.2.1",
"zod": "^3.9.5"
},
"devDependencies": {
"@backstage/backend-test-utils": "^0.1.7",
"@backstage/cli": "^0.8.0",
"jest": "^26.0.1",
"wait-for-expect": "^3.0.2"
},
"files": [
"dist",
"migrations/**/*.{js,d.ts}"
]
}
Original file line number Diff line number Diff line change
Expand Up @@ -14,16 +14,16 @@
* limitations under the License.
*/

import { resolvePackagePath } from '@backstage/backend-common';
import { Knex } from 'knex';
import { resolvePackagePath } from '../paths';
import { DB_MIGRATIONS_TABLE } from './tables';

const migrationsDir = resolvePackagePath(
'@backstage/backend-common',
'@backstage/backend-tasks',
'migrations',
);

export async function migrateBackendCommon(knex: Knex): Promise<void> {
export async function migrateBackendTasks(knex: Knex): Promise<void> {
await knex.migrate.latest({
directory: migrationsDir,
tableName: DB_MIGRATIONS_TABLE,
Expand Down
Loading

0 comments on commit e09bf60

Please sign in to comment.