feat: add conformance test for authorization server metadata

src/index.ts

@@ -10,6 +10,10 @@ import {
   printServerSummary,
   runInteractiveMode
 } from './runner';
+import {
+  printAuthorizationServerSummary,
+  runAuthorizationServerConformanceTest
+} from './runner/authorization-server';
 import {
   listScenarios,
   listClientScenarios,
@@ -23,11 +27,17 @@ import {
   listScenariosForSpec,
   listClientScenariosForSpec,
   getScenarioSpecVersions,
+  listClientScenariosForAuthorizationServer,
+  listClientScenariosForAuthorizationServerForSpec,
   ALL_SPEC_VERSIONS
 } from './scenarios';
 import type { SpecVersion } from './scenarios';
 import { ConformanceCheck } from './types';
-import { ClientOptionsSchema, ServerOptionsSchema } from './schemas';
+import {
+  AuthorizationServerOptionsSchema,
+  ClientOptionsSchema,
+  ServerOptionsSchema
+} from './schemas';
 import {
   loadExpectedFailures,
   evaluateBaseline,
@@ -52,12 +62,19 @@ function resolveSpecVersion(value: string): SpecVersion {
 function filterScenariosBySpecVersion(
   allScenarios: string[],
   version: SpecVersion,
-  command: 'client' | 'server'
+  command: 'client' | 'server' | 'authorization'
 ): string[] {
-  const versionScenarios =
-    command === 'client'
-      ? listScenariosForSpec(version)
-      : listClientScenariosForSpec(version);
+  let versionScenarios: string[];
+  if (command === 'client') {
+    versionScenarios = listScenariosForSpec(version);
+  } else if (command === 'server') {
+    versionScenarios = listClientScenariosForSpec(version);
+  } else if (command === 'authorization') {
+    versionScenarios =
+      listClientScenariosForAuthorizationServerForSpec(version);
+  } else {
+    versionScenarios = [];
+  }
   const allowed = new Set(versionScenarios);
   return allScenarios.filter((s) => allowed.has(s));
 }
@@ -444,6 +461,87 @@ program
     }
   });
 
+// Authorization command - tests an authorization server implementation
+program
+  .command('authorization')
+  .description(
+    'Run conformance tests against an authorization server implementation'
+  )
+  .requiredOption('--url <url>', 'URL of the authorization server to test')
+  .option('-o, --output-dir <path>', 'Save results to this directory')
+  .option(
+    '--spec-version <version>',
+    'Filter scenarios by spec version (cumulative for date versions)'
+  )
+  .action(async (options) => {
+    try {
+      // Validate options with Zod
+      const validated = AuthorizationServerOptionsSchema.parse(options);
+      const outputDir = options.outputDir;
+      const specVersionFilter = options.specVersion
+        ? resolveSpecVersion(options.specVersion)
+        : undefined;
+
+      let scenarios: string[];
+      scenarios = listClientScenariosForAuthorizationServer();
+      if (specVersionFilter) {
+        scenarios = filterScenariosBySpecVersion(
+          scenarios,
+          specVersionFilter,
+          'authorization'
+        );
+      }
+      console.log(
+        `Running test (${scenarios.length} scenarios) against ${validated.url}\n`
+      );
+
+      const allResults: { scenario: string; checks: ConformanceCheck[] }[] = [];
+      for (const scenarioName of scenarios) {
+        console.log(`\n=== Running scenario: ${scenarioName} ===`);
+        try {
+          const result = await runAuthorizationServerConformanceTest(
+            validated.url,
+            scenarioName,
+            outputDir
+          );
+          allResults.push({ scenario: scenarioName, checks: result.checks });
+        } catch (error) {
+          console.error(`Failed to run scenario ${scenarioName}:`, error);
+          allResults.push({
+            scenario: scenarioName,
+            checks: [
+              {
+                id: scenarioName,
+                name: scenarioName,
+                description: 'Failed to run scenario',
+                status: 'FAILURE',
+                timestamp: new Date().toISOString(),
+                errorMessage:
+                  error instanceof Error ? error.message : String(error)
+              }
+            ]
+          });
+        }
+      }
+      const { totalFailed } = printAuthorizationServerSummary(allResults);
+      process.exit(totalFailed > 0 ? 1 : 0);
+    } catch (error) {
+      if (error instanceof ZodError) {
+        console.error('Validation error:');
+        error.errors.forEach((err) => {
+          console.error(`  ${err.path.join('.')}: ${err.message}`);
+        });
+        console.error('\nAvailable authorization server scenarios:');
+        listClientScenariosForAuthorizationServer().forEach((s) =>
+          console.error(`  - ${s}`)
+        );
+        process.exit(1);
+      }
+      console.error('Authorization server test error:', error);
+      process.exit(1);
+    }
+  });
+
 // Tier check command
 program.addCommand(createTierCheckCommand());
 
@@ -453,6 +551,7 @@ program
   .description('List available test scenarios')
   .option('--client', 'List client scenarios')
   .option('--server', 'List server scenarios')
+  .option('--authorization', 'List authorization server scenarios')
   .option(
     '--spec-version <version>',
     'Filter scenarios by spec version (cumulative for date versions)'
@@ -462,7 +561,10 @@ program
       ? resolveSpecVersion(options.specVersion)
       : undefined;
 
-    if (options.server || (!options.client && !options.server)) {
+    if (
+      options.server ||
+      (!options.client && !options.server && !options.authorization)
+    ) {
       console.log('Server scenarios (test against a server):');
       let serverScenarios = listClientScenarios();
       if (specVersionFilter) {
@@ -478,7 +580,10 @@ program
       });
     }
 
-    if (options.client || (!options.client && !options.server)) {
+    if (
+      options.client ||
+      (!options.client && !options.server && !options.authorization)
+    ) {
       if (options.server || (!options.client && !options.server)) {
         console.log('');
       }
@@ -496,6 +601,31 @@ program
         console.log(`  - ${s}${v ? ` [${v}]` : ''}`);
       });
     }
+
+    if (
+      options.authorization ||
+      (!options.authorization && !options.server && !options.client)
+    ) {
+      if (!(options.authorization && !options.server && !options.client)) {
+        console.log('');
+      }
+      console.log(
+        'Authorization server scenarios (test against an authorization server):'
+      );
+      let authorizationServerScenarios =
+        listClientScenariosForAuthorizationServer();
+      if (specVersionFilter) {
+        authorizationServerScenarios = filterScenariosBySpecVersion(
+          authorizationServerScenarios,
+          specVersionFilter,
+          'authorization'
+        );
+      }
+      authorizationServerScenarios.forEach((s) => {
+        const v = getScenarioSpecVersions(s);
+        console.log(`  - ${s}${v ? ` [${v}]` : ''}`);
+      });
+    }
   });
 
 program.parse();

src/runner/authorization-server.ts

@@ -0,0 +1,74 @@
+import { promises as fs } from 'fs';
+import path from 'path';
+import { ConformanceCheck } from '../types';
+import { getClientScenarioForAuthorizationServer } from '../scenarios';
+import { createResultDir } from './utils';
+
+export async function runAuthorizationServerConformanceTest(
+  serverUrl: string,
+  scenarioName: string,
+  outputDir?: string
+): Promise<{
+  checks: ConformanceCheck[];
+  resultDir?: string;
+  scenarioDescription: string;
+}> {
+  let resultDir: string | undefined;
+
+  if (outputDir) {
+    resultDir = createResultDir(
+      outputDir,
+      scenarioName,
+      'authorization-server'
+    );
+    await fs.mkdir(resultDir, { recursive: true });
+  }
+
+  // Scenario is guaranteed to exist by CLI validation
+  const scenario = getClientScenarioForAuthorizationServer(scenarioName)!;
+
+  console.log(
+    `Running client scenario for authorization server '${scenarioName}' against server: ${serverUrl}`
+  );
+
+  const checks = await scenario.run(serverUrl);
+
+  if (resultDir) {
+    await fs.writeFile(
+      path.join(resultDir, 'checks.json'),
+      JSON.stringify(checks, null, 2)
+    );
+
+    console.log(`Results saved to ${resultDir}`);
+  }
+
+  return {
+    checks,
+    resultDir,
+    scenarioDescription: scenario.description
+  };
+}
+
+export function printAuthorizationServerSummary(
+  allResults: { scenario: string; checks: ConformanceCheck[] }[]
+): { totalPassed: number; totalFailed: number } {
+  console.log('\n\n=== SUMMARY ===');
+  let totalPassed = 0;
+  let totalFailed = 0;
+
+  for (const result of allResults) {
+    const passed = result.checks.filter((c) => c.status === 'SUCCESS').length;
+    const failed = result.checks.filter((c) => c.status === 'FAILURE').length;
+    totalPassed += passed;
+    totalFailed += failed;
+
+    const status = failed === 0 ? '✓' : '✗';
+    console.log(
+      `${status} ${result.scenario}: ${passed} passed, ${failed} failed`
+    );
+  }
+
+  console.log(`\nTotal: ${totalPassed} passed, ${totalFailed} failed`);
+
+  return { totalPassed, totalFailed };
+}

src/scenarios/authorization-server/authorization-server-metadata.test.ts

@@ -0,0 +1,52 @@
+import { describe, it, expect, vi } from 'vitest';
+import { AuthorizationServerMetadataEndpointScenario } from './authorization-server-metadata.js';
+import { request } from 'undici';
+
+vi.mock('undici', () => ({
+  request: vi.fn()
+}));
+
+const mockedRequest = vi.mocked(request);
+
+describe('AuthorizationServerMetadataEndpointScenario (SUCCESS only)', () => {
+  it('returns SUCCESS for valid authorization server metadata', async () => {
+    const scenario = new AuthorizationServerMetadataEndpointScenario();
+    const serverUrl =
+      'https://example.com/.well-known/oauth-authorization-server';
+
+    mockedRequest.mockResolvedValue({
+      statusCode: 200,
+      headers: {
+        'content-type': 'application/json'
+      },
+      body: {
+        json: async () => ({
+          issuer: 'https://example.com',
+          authorization_endpoint: 'https://example.com/auth',
+          token_endpoint: 'https://example.com/token',
+          response_types_supported: ['code']
+        })
+      }
+    } as any);
+
+    const checks = await scenario.run(serverUrl);
+
+    expect(checks).toHaveLength(1);
+
+    const check = checks[0];
+    expect(check.status).toBe('SUCCESS');
+    expect(check.errorMessage).toBeUndefined();
+    expect(check.details).toBeDefined();
+    expect(check.details!.contentType).toContain('application/json');
+    expect((check.details!.body as any).issuer).toBe('https://example.com');
+    expect((check.details!.body as any).authorization_endpoint).toBe(
+      'https://example.com/auth'
+    );
+    expect((check.details!.body as any).token_endpoint).toBe(
+      'https://example.com/token'
+    );
+    expect((check.details!.body as any).response_types_supported).toEqual([
+      'code'
+    ]);
+  });
+});