Merge pull request backstage#7606 from SDA-SE/feat/msgraphorgentitypr…

…ovider

Add `MicrosoftGraphOrgEntityProvider`

.changeset/wise-hats-remember.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-catalog-backend-module-msgraph': patch
+---
+
+Add `MicrosoftGraphOrgEntityProvider` as an alternative to `MicrosoftGraphOrgReaderProcessor` that automatically handles user and group deletions.

docs/integrations/azure/org.md

@@ -7,7 +7,7 @@ description: Importing users and groups from a Microsoft Azure Active Directory
 ---
 
 The Backstage catalog can be set up to ingest organizational data - users and
-teams - directly from an tenant in Microsoft Azure Active Directory via the
+teams - directly from a tenant in Microsoft Azure Active Directory via the
 Microsoft Graph API.
 
 More details on this are available in the

plugins/catalog-backend-module-msgraph/README.md

@@ -1,41 +1,24 @@
 # Catalog Backend Module for Microsoft Graph
 
 This is an extension module to the `plugin-catalog-backend` plugin, providing a
-`MicrosoftGraphOrgReaderProcessor` that can be used to ingest organization data
-from the Microsoft Graph API. This processor is useful if you want to import
-users and groups from Azure Active Directory or Office 365.
+`MicrosoftGraphOrgReaderProcessor` and a `MicrosoftGraphOrgEntityProvider` that
+can be used to ingest organization data from the Microsoft Graph API. This
+processor is useful if you want to import users and groups from Azure Active
+Directory or Office 365.
 
 ## Getting Started
 
-1. The processor is not installed by default, therefore you have to add a
-   dependency to `@backstage/plugin-catalog-backend-module-msgraph` to your
-   backend package.
+First you need to decide whether you want to use an [entity provider or a processor](https://backstage.io/docs/features/software-catalog/life-of-an-entity#stitching) to ingest the organization data.
+If you want groups and users deleted from the source to be automatically deleted
+from Backstage, choose the entity provider.
 
-```bash
-# From your Backstage root directory
-cd packages/backend
-yarn add @backstage/plugin-catalog-backend-module-msgraph
-```
-
-2. The `MicrosoftGraphOrgReaderProcessor` is not registered by default, so you
-   have to register it in the catalog plugin:
-
-```typescript
-// packages/backend/src/plugins/catalog.ts
-builder.addProcessor(
-  MicrosoftGraphOrgReaderProcessor.fromConfig(config, {
-    logger,
-  }),
-);
-```
-
-3. Create or use an existing App registration in the [Microsoft Azure Portal](https://portal.azure.com/).
+1. Create or use an existing App registration in the [Microsoft Azure Portal](https://portal.azure.com/).
    The App registration requires at least the API permissions `Group.Read.All`,
    `GroupMember.Read.All`, `User.Read` and `User.Read.All` for Microsoft Graph
    (if you still run into errors about insufficient privileges, add
    `Team.ReadBasic.All` and `TeamMember.Read.All` too).
 
-4. Configure the processor:
+2. Configure the processor or entity provider:
 
 ```yaml
 # app-config.yaml
@@ -69,6 +52,56 @@ catalog:
 
 By default, all users are loaded. If you want to filter users based on their attributes, use `userFilter`. `userGroupMemberFilter` can be used if you want to load users based on their group membership.
 
+3. The package is not installed by default, therefore you have to add a
+   dependency to `@backstage/plugin-catalog-backend-module-msgraph` to your
+   backend package.
+
+```bash
+# From your Backstage root directory
+cd packages/backend
+yarn add @backstage/plugin-catalog-backend-module-msgraph
+```
+
+### Using the Entity Provider
+
+4. The `MicrosoftGraphOrgEntityProvider` is not registered by default, so you
+   have to register it in the catalog plugin. Pass the target to reference a
+   provider from the configuration. As entity providers are not part of the
+   entity refresh loop, you have to run them manually.
+
+```typescript
+// packages/backend/src/plugins/catalog.ts
+const msGraphOrgEntityProvider = MicrosoftGraphOrgEntityProvider.fromConfig(
+  env.config,
+  {
+    id: 'https://graph.microsoft.com/v1.0',
+    target: 'https://graph.microsoft.com/v1.0',
+    logger: env.logger,
+  },
+);
+builder.addEntityProvider(msGraphOrgEntityProvider);
+
+// Trigger a read every 5 minutes
+useHotCleanup(
+  module,
+  runPeriodically(() => msGraphOrgEntityProvider.read(), 5 * 60 * 1000),
+);
+```
+
+### Using the Processor
+
+4. The `MicrosoftGraphOrgReaderProcessor` is not registered by default, so you
+   have to register it in the catalog plugin:
+
+```typescript
+// packages/backend/src/plugins/catalog.ts
+builder.addProcessor(
+  MicrosoftGraphOrgReaderProcessor.fromConfig(config, {
+    logger,
+  }),
+);
+```
+
 5. Add a location that ingests from Microsoft Graph:
 
 ```yaml
@@ -84,10 +117,11 @@ catalog:
     …
 ```
 
-## Customize the Processor
+## Customize the Processor or Entity Provider
 
-In case you want to customize the ingested entities, the `MicrosoftGraphOrgReaderProcessor`
-allows to pass transformers for users, groups and the organization.
+In case you want to customize the ingested entities, both the `MicrosoftGraphOrgReaderProcessor`
+and the `MicrosoftGraphOrgEntityProvider` allows to pass transformers for users,
+groups and the organization.
 
 1. Create a transformer:
 

plugins/catalog-backend-module-msgraph/api-report.md

@@ -6,6 +6,8 @@
 import { CatalogProcessor } from '@backstage/plugin-catalog-backend';
 import { CatalogProcessorEmit } from '@backstage/plugin-catalog-backend';
 import { Config } from '@backstage/config';
+import { EntityProvider } from '@backstage/plugin-catalog-backend';
+import { EntityProviderConnection } from '@backstage/plugin-catalog-backend';
 import { GroupEntity } from '@backstage/catalog-model';
 import { LocationSpec } from '@backstage/catalog-model';
 import { Logger as Logger_2 } from 'winston';
@@ -93,6 +95,38 @@ export class MicrosoftGraphClient {
   requestRaw(url: string): Promise<Response>;
 }
 
+// Warning: (ae-missing-release-tag) "MicrosoftGraphOrgEntityProvider" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
+//
+// @public
+export class MicrosoftGraphOrgEntityProvider implements EntityProvider {
+  constructor(options: {
+    id: string;
+    provider: MicrosoftGraphProviderConfig;
+    logger: Logger_2;
+    userTransformer?: UserTransformer;
+    groupTransformer?: GroupTransformer;
+    organizationTransformer?: OrganizationTransformer;
+  });
+  // (undocumented)
+  connect(connection: EntityProviderConnection): Promise<void>;
+  // (undocumented)
+  static fromConfig(
+    config: Config,
+    options: {
+      id: string;
+      target: string;
+      logger: Logger_2;
+      userTransformer?: UserTransformer;
+      groupTransformer?: GroupTransformer;
+      organizationTransformer?: OrganizationTransformer;
+    },
+  ): MicrosoftGraphOrgEntityProvider;
+  // (undocumented)
+  getProviderName(): string;
+  // (undocumented)
+  read(): Promise<void>;
+}
+
 // Warning: (ae-missing-release-tag) "MicrosoftGraphOrgReaderProcessor" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
 //
 // @public

plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.test.ts

@@ -72,4 +72,21 @@ describe('readMicrosoftGraphConfig', () => {
     ];
     expect(actual).toEqual(expected);
   });
+
+  it('should fail if both userFilter and userGroupMemberFilter are set', () => {
+    const config = {
+      providers: [
+        {
+          target: 'target',
+          tenantId: 'tenantId',
+          clientId: 'clientId',
+          clientSecret: 'clientSecret',
+          authority: 'https://login.example.com/',
+          userFilter: 'accountEnabled eq true',
+          userGroupMemberFilter: 'any',
+        },
+      ],
+    };
+    expect(() => readMicrosoftGraphConfig(new ConfigReader(config))).toThrow();
+  });
 });

plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.ts

@@ -87,6 +87,12 @@ export function readMicrosoftGraphConfig(
     );
     const groupFilter = providerConfig.getOptionalString('groupFilter');
 
+    if (userFilter && userGroupMemberFilter) {
+      throw new Error(
+        `userFilter and userGroupMemberFilter are mutually exclusive, only one can be specified.`,
+      );
+    }
+
     providers.push({
       target,
       authority,

plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgEntityProvider.test.ts

@@ -0,0 +1,174 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { getVoidLogger } from '@backstage/backend-common';
+import {
+  GroupEntity,
+  LOCATION_ANNOTATION,
+  ORIGIN_LOCATION_ANNOTATION,
+  UserEntity,
+} from '@backstage/catalog-model';
+import { EntityProviderConnection } from '@backstage/plugin-catalog-backend';
+import {
+  MicrosoftGraphClient,
+  MICROSOFT_GRAPH_USER_ID_ANNOTATION,
+  readMicrosoftGraphOrg,
+} from '../microsoftGraph';
+import {
+  MicrosoftGraphOrgEntityProvider,
+  withLocations,
+} from './MicrosoftGraphOrgEntityProvider';
+
+jest.mock('../microsoftGraph', () => {
+  return {
+    ...jest.requireActual('../microsoftGraph'),
+    readMicrosoftGraphOrg: jest.fn(),
+  };
+});
+
+const readMicrosoftGraphOrgMocked = readMicrosoftGraphOrg as jest.Mock<
+  Promise<{ users: UserEntity[]; groups: GroupEntity[] }>
+>;
+
+describe('MicrosoftGraphOrgEntityProvider', () => {
+  afterEach(() => jest.resetAllMocks());
+
+  it('should apply mutation', async () => {
+    jest
+      .spyOn(MicrosoftGraphClient, 'create')
+      .mockReturnValue({} as unknown as MicrosoftGraphClient);
+
+    readMicrosoftGraphOrgMocked.mockResolvedValue({
+      users: [
+        {
+          apiVersion: 'backstage.io/v1alpha1',
+          kind: 'User',
+          metadata: {
+            name: 'u1',
+          },
+          spec: {
+            memberOf: [],
+          },
+        },
+      ],
+      groups: [
+        {
+          apiVersion: 'backstage.io/v1alpha1',
+          kind: 'Group',
+          metadata: {
+            name: 'g1',
+          },
+          spec: {
+            type: 'team',
+            children: [],
+          },
+        },
+      ],
+    });
+
+    const entityProviderConnection: EntityProviderConnection = {
+      applyMutation: jest.fn(),
+    };
+    const provider = new MicrosoftGraphOrgEntityProvider({
+      id: 'test',
+      logger: getVoidLogger(),
+      provider: {
+        target: 'https://example.com',
+        tenantId: 'tenant',
+        clientId: 'clientid',
+        clientSecret: 'clientsecret',
+      },
+    });
+
+    provider.connect(entityProviderConnection);
+
+    await provider.read();
+
+    expect(entityProviderConnection.applyMutation).toBeCalledWith({
+      entities: [
+        {
+          entity: {
+            apiVersion: 'backstage.io/v1alpha1',
+            kind: 'User',
+            metadata: {
+              annotations: {
+                'backstage.io/managed-by-location': 'msgraph:test/u1',
+                'backstage.io/managed-by-origin-location': 'msgraph:test/u1',
+              },
+              name: 'u1',
+            },
+            spec: {
+              memberOf: [],
+            },
+          },
+          locationKey: 'msgraph-org-provider:test',
+        },
+        {
+          entity: {
+            apiVersion: 'backstage.io/v1alpha1',
+            kind: 'Group',
+            metadata: {
+              annotations: {
+                'backstage.io/managed-by-location': 'msgraph:test/g1',
+                'backstage.io/managed-by-origin-location': 'msgraph:test/g1',
+              },
+              name: 'g1',
+            },
+            spec: {
+              children: [],
+              type: 'team',
+            },
+          },
+          locationKey: 'msgraph-org-provider:test',
+        },
+      ],
+      type: 'full',
+    });
+  });
+});
+
+describe('withLocations', () => {
+  it('should set location annotations', () => {
+    expect(
+      withLocations('test', {
+        apiVersion: 'backstage.io/v1alpha1',
+        kind: 'User',
+        metadata: {
+          name: 'u1',
+          annotations: {
+            [MICROSOFT_GRAPH_USER_ID_ANNOTATION]: 'uid',
+          },
+        },
+        spec: {
+          memberOf: [],
+        },
+      }),
+    ).toEqual({
+      apiVersion: 'backstage.io/v1alpha1',
+      kind: 'User',
+      metadata: {
+        name: 'u1',
+        annotations: {
+          [MICROSOFT_GRAPH_USER_ID_ANNOTATION]: 'uid',
+          [LOCATION_ANNOTATION]: 'msgraph:test/uid',
+          [ORIGIN_LOCATION_ANNOTATION]: 'msgraph:test/uid',
+        },
+      },
+      spec: {
+        memberOf: [],
+      },
+    });
+  });
+});