Implement the locks part of the task manager

Signed-off-by: Fredrik Adelöw <freben@gmail.com>

packages/backend-common/knexfile.js

@@ -0,0 +1,26 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// This file makes it possible to run "yarn knex migrate:make some_file_name"
+// to assist in making new migrations
+module.exports = {
+  client: 'sqlite3',
+  connection: ':memory:',
+  useNullAsDefault: true,
+  migrations: {
+    directory: './migrations',
+  },
+};

packages/backend-common/migrations/20210928160613_init.js

@@ -0,0 +1,71 @@
+/*
+ * Copyright 2020 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// @ts-check
+
+/**
+ * @param {import('knex').Knex} knex
+ */
+exports.up = async function up(knex) {
+  //
+  // locking
+  //
+  await knex.schema.createTable(
+    'backstage_backend_common__task_locks',
+    table => {
+      table.comment('Locks used for mutual exclusion among multiple workers');
+      table
+        .text('id')
+        .primary()
+        .notNullable()
+        .comment('The unique id of this particular lock');
+      table
+        .text('acquired_ticket')
+        .nullable()
+        .comment('A unique ticket for the current lock acquiral, if any');
+      table
+        .dateTime('acquired_at')
+        .nullable()
+        .comment('The time when the lock was acquired, if locked');
+      table
+        .dateTime('expires_at')
+        .nullable()
+        .comment('The time when an acquired lock will time out and expire');
+      table.index('id', 'task_locks_id_idx');
+    },
+  );
+  //
+  // tasks
+  //
+  await knex.schema.createTable('backstage_backend_common__tasks', table => {
+    table.comment('Tasks used for scheduling work on multiple workers');
+    table
+      .text('id')
+      .primary()
+      .notNullable()
+      .comment('The unique id of this particular task');
+  });
+};
+
+/**
+ * @param {import('knex').Knex} knex
+ */
+exports.down = async function down(knex) {
+  await knex.schema.alterTable('task_locks', table => {
+    table.dropIndex([], 'task_locks_id_idx');
+  });
+  await knex.schema.dropTable('task_locks');
+};

packages/backend-common/package.json

@@ -59,14 +59,17 @@
     "knex": "^0.95.1",
     "lodash": "^4.17.21",
     "logform": "^2.1.1",
+    "luxon": "^2.0.2",
     "minimatch": "^3.0.4",
     "minimist": "^1.2.5",
     "morgan": "^1.10.0",
+    "node-abort-controller": "^3.0.0",
     "raw-body": "^2.4.1",
     "selfsigned": "^1.10.7",
     "stoppable": "^1.1.0",
     "tar": "^6.1.2",
     "unzipper": "^0.10.11",
+    "uuid": "^8.0.0",
     "winston": "^3.2.1",
     "yn": "^4.0.0"
   },
@@ -79,6 +82,7 @@
     }
   },
   "devDependencies": {
+    "@backstage/backend-test-utils": "^0.1.8",
     "@backstage/cli": "^0.8.2",
     "@backstage/test-utils": "^0.1.21",
     "@types/archiver": "^5.1.0",
@@ -107,6 +111,7 @@
   },
   "files": [
     "dist",
+    "migrations/**/*.{js,d.ts}",
     "config.d.ts"
   ],
   "configSchema": "config.d.ts"

packages/backend-common/src/database/DatabaseManager.ts

@@ -13,14 +13,15 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 import { Knex } from 'knex';
 import { omit } from 'lodash';
 import { Config, ConfigReader } from '@backstage/config';
 import { JsonObject } from '@backstage/types';
 import {
   createDatabaseClient,
-  ensureDatabaseExists,
   createNameOverride,
+  ensureDatabaseExists,
   normalizeConnection,
 } from './connection';
 import { PluginDatabaseManager } from './types';
@@ -165,7 +166,7 @@ export class DatabaseManager {
     );
 
     return {
-      // include base connection if client type has not been overriden
+      // include base connection if client type has not been overridden
       ...(overridden ? {} : baseConnection),
       ...connection,
     };

packages/backend-common/src/database/migrateBackendCommon.ts

@@ -0,0 +1,30 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { Knex } from 'knex';
+import { resolvePackagePath } from '../paths';
+
+const migrationsDir = resolvePackagePath(
+  '@backstage/backend-common',
+  'migrations',
+);
+
+export async function migrateBackendCommon(knex: Knex): Promise<void> {
+  await knex.migrate.latest({
+    directory: migrationsDir,
+    tableName: 'knex_migrations_backstage_backend_common',
+  });
+}

packages/backend-common/src/database/tables.ts

@@ -0,0 +1,22 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export type DbTaskLocksRow = {
+  id: string;
+  acquired_ticket?: string;
+  acquired_at?: Date;
+  expires_at?: Date;
+};

packages/backend-common/src/index.ts

@@ -31,4 +31,5 @@ export * from './paths';
 export * from './reading';
 export * from './scm';
 export * from './service';
+export * from './tasks';
 export * from './util';

packages/backend-common/src/tasks/TaskManager.test.ts

@@ -0,0 +1,73 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { TestDatabaseId, TestDatabases } from '@backstage/backend-test-utils';
+import { Duration } from 'luxon';
+import { DatabaseManager } from '../database';
+import { TaskManager } from './TaskManager';
+
+describe('TaskManager', () => {
+  const databases = TestDatabases.create({
+    ids: ['POSTGRES_13', 'POSTGRES_9', 'SQLITE_3'],
+  });
+
+  async function createDatabase(
+    databaseId: TestDatabaseId,
+  ): Promise<DatabaseManager> {
+    const knex = await databases.init(databaseId);
+    const databaseManager: Partial<DatabaseManager> = {
+      forPlugin: () => ({
+        getClient: async () => knex,
+      }),
+    };
+    return databaseManager as DatabaseManager;
+  }
+
+  describe('locking', () => {
+    it.each(databases.eachSupportedId())(
+      'can run the happy path, %p',
+      async databaseId => {
+        const database = await createDatabase(databaseId);
+        const manager = new TaskManager(database).forPlugin('test');
+
+        const lock1 = await manager.acquireLock('lock1', {
+          timeout: Duration.fromMillis(5000),
+        });
+        const lock2 = await manager.acquireLock('lock2', {
+          timeout: Duration.fromMillis(5000),
+        });
+
+        expect(lock1.acquired).toBe(true);
+        expect(lock2.acquired).toBe(true);
+
+        await expect(
+          manager.acquireLock('lock1', {
+            timeout: Duration.fromMillis(5000),
+          }),
+        ).resolves.toEqual({ acquired: false });
+
+        await (lock1 as any).release();
+        await (lock2 as any).release();
+
+        const lock1Again = await manager.acquireLock('lock1', {
+          timeout: Duration.fromMillis(5000),
+        });
+        expect(lock1Again.acquired).toBe(true);
+        await (lock1Again as any).release();
+      },
+    );
+  });
+});