Merge pull request backstage#7612 from backstage/freben/recurring-tasks

Add support for distributed tasks

.changeset/early-cobras-explode.md

@@ -0,0 +1,5 @@
+---
+'@backstage/catalog-client': patch
+---
+
+Update to the right version of @backstage/errors

.changeset/forty-ligers-protect.md

@@ -0,0 +1,5 @@
+---
+'@backstage/cli': patch
+---
+
+remove double config dep

.changeset/happy-rice-tickle.md

@@ -0,0 +1,5 @@
+---
+'@backstage/backend-common': patch
+---
+
+Added the `isDatabaseConflictError` function.

.github/styles/vocab.txt

@@ -52,6 +52,7 @@ configmaps
 configs
 const
 cookiecutter
+cron
 css
 Datadog
 dataflow
@@ -162,6 +163,8 @@ Monorepo
 monorepos
 msgraph
 msw
+mutex
+mutexes
 mysql
 namespace
 namespaced

packages/backend-common/api-report.md

@@ -374,6 +374,9 @@ export class GitlabUrlReader implements UrlReader {
 
 export { isChildPath };
 
+// @public
+export function isDatabaseConflictError(e: unknown): boolean;
+
 // @public
 export function loadBackendConfig(options: {
   logger: Logger_2;

packages/backend-common/src/database/DatabaseManager.ts

@@ -13,14 +13,15 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 import { Knex } from 'knex';
 import { omit } from 'lodash';
 import { Config, ConfigReader } from '@backstage/config';
 import { JsonObject } from '@backstage/types';
 import {
   createDatabaseClient,
-  ensureDatabaseExists,
   createNameOverride,
+  ensureDatabaseExists,
   normalizeConnection,
 } from './connection';
 import { PluginDatabaseManager } from './types';
@@ -165,7 +166,7 @@ export class DatabaseManager {
     );
 
     return {
-      // include base connection if client type has not been overriden
+      // include base connection if client type has not been overridden
       ...(overridden ? {} : baseConnection),
       ...connection,
     };

packages/backend-common/src/database/index.ts

@@ -28,3 +28,4 @@ export {
 } from './connection';
 
 export type { PluginDatabaseManager } from './types';
+export { isDatabaseConflictError } from './util';

packages/backend-common/src/database/util.ts

@@ -0,0 +1,33 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Tries to deduce whether a thrown error is a database conflict.
+ *
+ * @public
+ * @param e - A thrown error
+ * @returns True if the error looks like it was a conflict error thrown by a
+ *          known database engine
+ */
+export function isDatabaseConflictError(e: unknown) {
+  const message = (e as any)?.message;
+
+  return (
+    typeof message === 'string' &&
+    (/SQLITE_CONSTRAINT: UNIQUE/.test(message) ||
+      /unique constraint/.test(message))
+  );
+}

packages/backend-tasks/.eslintrc.js

@@ -0,0 +1,3 @@
+module.exports = {
+  extends: [require.resolve('@backstage/cli/config/eslint.backend')],
+};

packages/backend-tasks/README.md

@@ -0,0 +1,35 @@
+# @backstage/backend-tasks
+
+Common distributed task management for Backstage backends.
+
+## Usage
+
+Add the library to your backend package:
+
+```sh
+# From your Backstage root directory
+cd packages/backend
+yarn add @backstage/backend-tasks
+```
+
+then make use of its facilities as necessary:
+
+```typescript
+import { TaskScheduler } from '@backstage/backend-tasks';
+import { Duration } from 'luxon';
+
+const scheduler = TaskScheduler.fromConfig(rootConfig).forPlugin('my-plugin');
+
+await scheduler.scheduleTask({
+  id: 'refresh-things',
+  frequency: Duration.fromObject({ minutes: 10 }),
+  fn: async () => {
+    await entityProvider.run();
+  },
+});
+```
+
+## Documentation
+
+- [Backstage Readme](https://github.com/backstage/backstage/blob/master/README.md)
+- [Backstage Documentation](https://github.com/backstage/backstage/blob/master/docs/README.md)

packages/backend-tasks/api-report.md

@@ -0,0 +1,45 @@
+## API Report File for "@backstage/backend-tasks"
+
+> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
+
+```ts
+import { AbortSignal as AbortSignal_2 } from 'node-abort-controller';
+import { Config } from '@backstage/config';
+import { DatabaseManager } from '@backstage/backend-common';
+import { Duration } from 'luxon';
+import { Logger as Logger_2 } from 'winston';
+
+// @public
+export interface PluginTaskScheduler {
+  scheduleTask(task: TaskDefinition): Promise<void>;
+}
+
+// @public
+export interface TaskDefinition {
+  fn: TaskFunction;
+  frequency: Duration;
+  id: string;
+  initialDelay?: Duration;
+  signal?: AbortSignal_2;
+  timeout: Duration;
+}
+
+// @public
+export type TaskFunction =
+  | ((abortSignal: AbortSignal_2) => void | Promise<void>)
+  | (() => void | Promise<void>);
+
+// @public
+export class TaskScheduler {
+  constructor(databaseManager: DatabaseManager, logger: Logger_2);
+  forPlugin(pluginId: string): PluginTaskScheduler;
+  // (undocumented)
+  static fromConfig(
+    config: Config,
+    options?: {
+      databaseManager?: DatabaseManager;
+      logger?: Logger_2;
+    },
+  ): TaskScheduler;
+}
+```

packages/backend-tasks/migrations/20210928160613_init.js

@@ -0,0 +1,64 @@
+/*
+ * Copyright 2020 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// @ts-check
+
+/**
+ * @param {import('knex').Knex} knex
+ */
+exports.up = async function up(knex) {
+  //
+  // tasks
+  //
+  await knex.schema.createTable('backstage_backend_tasks__tasks', table => {
+    table.comment('Tasks used for scheduling work on multiple workers');
+    table
+      .text('id')
+      .primary()
+      .notNullable()
+      .comment('The unique ID of this particular task');
+    table
+      .text('settings_json')
+      .notNullable()
+      .comment('JSON serialized object with properties for this task');
+    table
+      .dateTime('next_run_start_at')
+      .notNullable()
+      .comment('The next time that the task should be started');
+    table
+      .text('current_run_ticket')
+      .nullable()
+      .comment('A unique ticket for the current task run');
+    table
+      .dateTime('current_run_started_at')
+      .nullable()
+      .comment('The time that the current task run started');
+    table
+      .dateTime('current_run_expires_at')
+      .nullable()
+      .comment('The time that the current task run will time out');
+  });
+};
+
+/**
+ * @param {import('knex').Knex} knex
+ */
+exports.down = async function down(knex) {
+  //
+  // tasks
+  //
+  await knex.schema.dropTable('backstage_backend_tasks__tasks');
+};

packages/backend-tasks/package.json

@@ -0,0 +1,55 @@
+{
+  "name": "@backstage/backend-tasks",
+  "description": "Common distributed task management library for Backstage backends",
+  "version": "0.1.0",
+  "main": "src/index.ts",
+  "types": "src/index.ts",
+  "private": false,
+  "publishConfig": {
+    "access": "public",
+    "main": "dist/index.cjs.js",
+    "types": "dist/index.d.ts"
+  },
+  "homepage": "https://backstage.io",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/backstage/backstage",
+    "directory": "packages/backend-tasks"
+  },
+  "keywords": [
+    "backstage"
+  ],
+  "license": "Apache-2.0",
+  "scripts": {
+    "build": "backstage-cli build --outputs cjs,types",
+    "lint": "backstage-cli lint",
+    "test": "backstage-cli test",
+    "prepack": "backstage-cli prepack",
+    "postpack": "backstage-cli postpack",
+    "clean": "backstage-cli clean"
+  },
+  "dependencies": {
+    "@backstage/backend-common": "^0.9.8",
+    "@backstage/config": "^0.1.11",
+    "@backstage/errors": "^0.1.4",
+    "@backstage/types": "^0.1.1",
+    "@types/luxon": "^2.0.4",
+    "knex": "^0.95.1",
+    "lodash": "^4.17.21",
+    "luxon": "^2.0.2",
+    "node-abort-controller": "^3.0.1",
+    "uuid": "^8.0.0",
+    "winston": "^3.2.1",
+    "zod": "^3.9.5"
+  },
+  "devDependencies": {
+    "@backstage/backend-test-utils": "^0.1.8",
+    "@backstage/cli": "^0.8.1",
+    "jest": "^26.0.1",
+    "wait-for-expect": "^3.0.2"
+  },
+  "files": [
+    "dist",
+    "migrations/**/*.{js,d.ts}"
+  ]
+}

packages/backend-tasks/src/database/migrateBackendTasks.ts

@@ -0,0 +1,31 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { resolvePackagePath } from '@backstage/backend-common';
+import { Knex } from 'knex';
+import { DB_MIGRATIONS_TABLE } from './tables';
+
+const migrationsDir = resolvePackagePath(
+  '@backstage/backend-tasks',
+  'migrations',
+);
+
+export async function migrateBackendTasks(knex: Knex): Promise<void> {
+  await knex.migrate.latest({
+    directory: migrationsDir,
+    tableName: DB_MIGRATIONS_TABLE,
+  });
+}

packages/backend-tasks/src/database/tables.ts

@@ -0,0 +1,27 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export const DB_MIGRATIONS_TABLE = 'backstage_backend_tasks__knex_migrations';
+export const DB_TASKS_TABLE = 'backstage_backend_tasks__tasks';
+
+export type DbTasksRow = {
+  id: string;
+  settings_json: string;
+  next_run_start_at: Date;
+  current_run_ticket?: string;
+  current_run_started_at?: Date | string;
+  current_run_expires_at?: Date | string;
+};