fix(backend,nextjs): Revert { data, errors } return values to v4 format for clerkClient methods and jwt/token helpers (clerk#2633)

* fix(backend): Make signJwt consistent by returning error instead of throwing

* fix(backend): Revert `{ data, errors }` return value from api client

This change will make all the BackendAPI methods to either return
the value or throw error instead of returning `{ data, errors }`.

We also (temporarily) added the `totalCount` property when we expect
an array of resources to allow passing the `total_count` of paginated
responses from the response body to the return value.

* fix(backend): Revert `{ data, errors }` return value from jwt/token helpers

The following helpers were reverted to return their `data` or throw an error.
Internally we will keep the `{ data, errors }` format but added a compatibility
layer to avoid exposing this breaking change:

- `import { verifyToken } from '@clerk/backend'`
- `import { signJwt, hasValidSignature, decodeJwt, verifyJwt } from '@clerk/backend/jwt'`

* fix(nextjs): Apply changes related to adding legacy return value compat layer

* fix(clerk-sdk-node): Update examples related to backend legacy return value compat layer

* chore(repo): Add changeset

* feat(backend): Paginated responses now return { data, totalCount } instead of an array

---------

Co-authored-by: Nikos Douvlis <nikosdouvlis@gmail.com>

Author: dimkl

.changeset/gentle-radios-shout.md

@@ -0,0 +1,16 @@
+---
+'@clerk/clerk-sdk-node': patch
+'@clerk/backend': major
+'@clerk/nextjs': patch
+---
+
+The following paginated APIs now return `{ data, totalCount }` instead of simple arrays, in order to make building paginated UIs easier:
+- `clerkClient.users.getOrganizationMembershipList(...)`
+- `clerkClient.organization.getOrganizationList(...)`
+- `clerkClient.organization.getOrganizationInvitationList(...)`
+
+Revert changing the `{ data, errors }` return value of the following helpers to throw the `errors` or return the `data` (keep v4 format):
+
+- `import { verifyToken } from '@clerk/backend'`
+- `import { signJwt, hasValidSignature, decodeJwt, verifyJwt } from '@clerk/backend/jwt'`
+- BAPI `clerkClient` methods eg (`clerkClient.users.getUserList(...)`)

integration/cleanup/cleanup.setup.ts

@@ -19,17 +19,12 @@ setup('cleanup instances ', async () => {
   for (const entry of entries) {
     console.log(`Cleanup for ${entry!.secretKey.replace(/(sk_test_)(.+)(...)/, '$1***$3')}`);
     const clerkClient = createClerkClient({ secretKey: entry!.secretKey, apiUrl: entry?.apiUrl });
-    const { data: users, errors } = await clerkClient.users.getUserList({
+    const users = await clerkClient.users.getUserList({
       orderBy: '-created_at',
       query: 'clerkcookie',
       limit: 100,
     });
 
-    if (errors) {
-      console.log(errors);
-      return;
-    }
-
     const batches = batchElements(skipUsersThatWereCreatedToday(users), 5);
     for (const batch of batches) {
       console.log(`Starting batch...`);

integration/testUtils/usersService.ts

@@ -39,7 +39,7 @@ export const createUserService = (clerkClient: ReturnType<typeof Clerk>) => {
     },
     createFakeOrganization: async (userId: string) => {
       const name = faker.animal.dog();
-      const { data: organization } = await clerkClient.organizations.createOrganization({
+      const organization = await clerkClient.organizations.createOrganization({
         name: faker.animal.dog(),
         createdBy: userId,
       });

integration/tests/protect.test.ts

@@ -15,11 +15,10 @@ testAgainstRunningApps({ withEnv: [appConfigs.envs.withCustomRoles] })('authoriz
   test.beforeAll(async () => {
     const m = createTestUtils({ app });
     fakeAdmin = m.services.users.createFakeUser();
-    const { data: admin } = await m.services.users.createBapiUser(fakeAdmin);
+    const admin = await m.services.users.createBapiUser(fakeAdmin);
     fakeOrganization = await m.services.users.createFakeOrganization(admin.id);
     fakeViewer = m.services.users.createFakeUser();
-    const { data: viewer } = await m.services.users.createBapiUser(fakeViewer);
-
+    const viewer = await m.services.users.createBapiUser(fakeViewer);
     await m.services.clerk.organizations.createOrganizationMembership({
       organizationId: fakeOrganization.organization.id,
       role: 'org:viewer' as OrganizationMembershipRole,

packages/backend/src/api/__tests__/factory.test.ts

@@ -4,14 +4,7 @@ import sinon from 'sinon';
 // @ts-ignore
 import userJson from '../../fixtures/user.json';
 import runtime from '../../runtime';
-import {
-  assertErrorResponse,
-  assertResponse,
-  jsonError,
-  jsonNotOk,
-  jsonOk,
-  jsonPaginatedOk,
-} from '../../util/testUtils';
+import { jsonError, jsonNotOk, jsonOk, jsonPaginatedOk } from '../../util/testUtils';
 import { createBackendApiClient } from '../factory';
 
 export default (QUnit: QUnit) => {
@@ -35,16 +28,12 @@ export default (QUnit: QUnit) => {
 
       const response = await apiClient.users.getUser('user_deadbeef');
 
-      assertResponse(assert, response);
-      const { data: payload, totalCount } = response;
-
-      assert.equal(payload.firstName, 'John');
-      assert.equal(payload.lastName, 'Doe');
-      assert.equal(payload.emailAddresses[0].emailAddress, 'john.doe@clerk.test');
-      assert.equal(payload.phoneNumbers[0].phoneNumber, '+311-555-2368');
-      assert.equal(payload.externalAccounts[0].emailAddress, 'john.doe@clerk.test');
-      assert.equal(payload.publicMetadata.zodiac_sign, 'leo');
-      assert.equal(totalCount, undefined);
+      assert.equal(response.firstName, 'John');
+      assert.equal(response.lastName, 'Doe');
+      assert.equal(response.emailAddresses[0].emailAddress, 'john.doe@clerk.test');
+      assert.equal(response.phoneNumbers[0].phoneNumber, '+311-555-2368');
+      assert.equal(response.externalAccounts[0].emailAddress, 'john.doe@clerk.test');
+      assert.equal(response.publicMetadata.zodiac_sign, 'leo');
 
       assert.ok(
         fakeFetch.calledOnceWith('https://api.clerk.test/v1/users/user_deadbeef', {
@@ -63,16 +52,13 @@ export default (QUnit: QUnit) => {
       fakeFetch.onCall(0).returns(jsonOk([userJson]));
 
       const response = await apiClient.users.getUserList({ offset: 2, limit: 5 });
-      assertResponse(assert, response);
-      const { data: payload, totalCount } = response;
 
-      assert.equal(payload[0].firstName, 'John');
-      assert.equal(payload[0].lastName, 'Doe');
-      assert.equal(payload[0].emailAddresses[0].emailAddress, 'john.doe@clerk.test');
-      assert.equal(payload[0].phoneNumbers[0].phoneNumber, '+311-555-2368');
-      assert.equal(payload[0].externalAccounts[0].emailAddress, 'john.doe@clerk.test');
-      assert.equal(payload[0].publicMetadata.zodiac_sign, 'leo');
-      assert.equal(totalCount, 1);
+      assert.equal(response[0].firstName, 'John');
+      assert.equal(response[0].lastName, 'Doe');
+      assert.equal(response[0].emailAddresses[0].emailAddress, 'john.doe@clerk.test');
+      assert.equal(response[0].phoneNumbers[0].phoneNumber, '+311-555-2368');
+      assert.equal(response[0].externalAccounts[0].emailAddress, 'john.doe@clerk.test');
+      assert.equal(response[0].publicMetadata.zodiac_sign, 'leo');
 
       assert.ok(
         fakeFetch.calledOnceWith('https://api.clerk.test/v1/users?offset=2&limit=5', {
@@ -88,32 +74,18 @@ export default (QUnit: QUnit) => {
 
     test('executes a successful backend API request for a paginated response', async assert => {
       fakeFetch = sinon.stub(runtime, 'fetch');
-      fakeFetch.onCall(0).returns(jsonPaginatedOk([userJson], 3));
+      fakeFetch.onCall(0).returns(jsonPaginatedOk([{ id: '1' }], 3));
 
-      const response = await apiClient.users.getUserList({ offset: 2, limit: 5 });
-      assertResponse(assert, response);
-      const { data: payload, totalCount } = response;
-
-      assert.equal(payload[0].firstName, 'John');
-      assert.equal(payload[0].lastName, 'Doe');
-      assert.equal(payload[0].emailAddresses[0].emailAddress, 'john.doe@clerk.test');
-      assert.equal(payload[0].phoneNumbers[0].phoneNumber, '+311-555-2368');
-      assert.equal(payload[0].externalAccounts[0].emailAddress, 'john.doe@clerk.test');
-      assert.equal(payload[0].publicMetadata.zodiac_sign, 'leo');
+      const { data: response, totalCount } = await apiClient.users.getOrganizationMembershipList({
+        offset: 2,
+        limit: 5,
+        userId: 'user_123',
+      });
+
+      assert.equal(response[0].id, '1');
       // payload.length is different from response total_count to check that totalCount use the total_count from response
-      assert.equal(payload.length, 1);
       assert.equal(totalCount, 3);
-
-      assert.ok(
-        fakeFetch.calledOnceWith('https://api.clerk.test/v1/users?offset=2&limit=5', {
-          method: 'GET',
-          headers: {
-            Authorization: 'Bearer deadbeef',
-            'Content-Type': 'application/json',
-            'User-Agent': '@clerk/backend@0.0.0-test',
-          },
-        }),
-      );
+      assert.equal(response.length, 1);
     });
 
     test('executes a successful backend API request to create a new resource', async assert => {
@@ -127,10 +99,8 @@ export default (QUnit: QUnit) => {
           star_sign: 'Leon',
         },
       });
-      assertResponse(assert, response);
-      const { data: payload } = response;
 
-      assert.equal(payload.firstName, 'John');
+      assert.equal(response.firstName, 'John');
 
       assert.ok(
         fakeFetch.calledOnceWith('https://api.clerk.test/v1/users', {
@@ -162,16 +132,14 @@ export default (QUnit: QUnit) => {
       fakeFetch = sinon.stub(runtime, 'fetch');
       fakeFetch.onCall(0).returns(jsonNotOk({ errors: [mockErrorPayload], clerk_trace_id: traceId }));
 
-      const response = await apiClient.users.getUser('user_deadbeef');
-      assertErrorResponse(assert, response);
+      const errResponse = await apiClient.users.getUser('user_deadbeef').catch(err => err);
 
-      assert.equal(response.clerkTraceId, traceId);
-      assert.equal(response.status, 422);
-      assert.equal(response.statusText, '422');
-      assert.equal(response.errors[0].code, 'whatever_error');
-      assert.equal(response.errors[0].message, 'whatever error');
-      assert.equal(response.errors[0].longMessage, 'some long message');
-      assert.equal(response.errors[0].meta.paramName, 'some param');
+      assert.equal(errResponse.clerkTraceId, traceId);
+      assert.equal(errResponse.status, 422);
+      assert.equal(errResponse.errors[0].code, 'whatever_error');
+      assert.equal(errResponse.errors[0].message, 'whatever error');
+      assert.equal(errResponse.errors[0].longMessage, 'some long message');
+      assert.equal(errResponse.errors[0].meta.paramName, 'some param');
 
       assert.ok(
         fakeFetch.calledOnceWith('https://api.clerk.test/v1/users/user_deadbeef', {
@@ -189,12 +157,10 @@ export default (QUnit: QUnit) => {
       fakeFetch = sinon.stub(runtime, 'fetch');
       fakeFetch.onCall(0).returns(jsonError({ errors: [] }));
 
-      const response = await apiClient.users.getUser('user_deadbeef');
-      assertErrorResponse(assert, response);
+      const errResponse = await apiClient.users.getUser('user_deadbeef').catch(err => err);
 
-      assert.equal(response.status, 500);
-      assert.equal(response.statusText, '500');
-      assert.equal(response.clerkTraceId, 'mock_cf_ray');
+      assert.equal(errResponse.status, 500);
+      assert.equal(errResponse.clerkTraceId, 'mock_cf_ray');
 
       assert.ok(
         fakeFetch.calledOnceWith('https://api.clerk.test/v1/users/user_deadbeef', {

packages/backend/src/api/endpoints/OrganizationApi.ts

@@ -8,6 +8,7 @@ import type {
   OrganizationInvitationStatus,
   OrganizationMembership,
 } from '../resources';
+import type { PaginatedResourceResponse } from '../resources/Deserializer';
 import type { OrganizationMembershipRole } from '../resources/Enums';
 import { AbstractAPI } from './AbstractApi';
 
@@ -95,7 +96,7 @@ type RevokeOrganizationInvitationParams = {
 
 export class OrganizationAPI extends AbstractAPI {
   public async getOrganizationList(params?: GetOrganizationListParams) {
-    return this.request<Organization[]>({
+    return this.request<PaginatedResourceResponse<Organization[]>>({
       method: 'GET',
       path: basePath,
       queryParams: params,
@@ -234,7 +235,7 @@ export class OrganizationAPI extends AbstractAPI {
     const { organizationId, status, limit, offset } = params;
     this.requireId(organizationId);
 
-    return this.request<OrganizationInvitation[]>({
+    return this.request<PaginatedResourceResponse<OrganizationInvitation[]>>({
       method: 'GET',
       path: joinPaths(basePath, organizationId, 'invitations'),
       queryParams: { status, limit, offset },

packages/backend/src/api/endpoints/UserApi.ts

@@ -3,6 +3,7 @@ import type { ClerkPaginationRequest, OAuthProvider } from '@clerk/types';
 import runtime from '../../runtime';
 import { joinPaths } from '../../util/path';
 import type { OauthAccessToken, OrganizationMembership, User } from '../resources';
+import type { PaginatedResourceResponse } from '../resources/Deserializer';
 import { AbstractAPI } from './AbstractApi';
 
 const basePath = '/users';
@@ -199,7 +200,7 @@ export class UserAPI extends AbstractAPI {
     const { userId, limit, offset } = params;
     this.requireId(userId);
 
-    return this.request<OrganizationMembership[]>({
+    return this.request<PaginatedResourceResponse<OrganizationMembership[]>>({
       method: 'GET',
       path: joinPaths(basePath, userId, 'organization_memberships'),
       queryParams: { limit, offset },

packages/backend/src/api/request.ts

@@ -1,4 +1,4 @@
-import { parseError } from '@clerk/shared/error';
+import { ClerkAPIResponseError, parseError } from '@clerk/shared/error';
 import type { ClerkAPIError, ClerkAPIErrorJSON } from '@clerk/types';
 import snakecaseKeys from 'snakecase-keys';
 
@@ -55,7 +55,7 @@ type BuildRequestOptions = {
   userAgent?: string;
 };
 export function buildRequest(options: BuildRequestOptions) {
-  return async <T>(requestOptions: ClerkBackendApiRequestOptions): Promise<ClerkBackendApiResponse<T>> => {
+  const requestFn = async <T>(requestOptions: ClerkBackendApiRequestOptions): Promise<ClerkBackendApiResponse<T>> => {
     const { secretKey, apiUrl = API_URL, apiVersion = API_VERSION, userAgent = USER_AGENT } = options;
     const { path, method, queryParams, headerParams, bodyParams, formData } = requestOptions;
 
@@ -149,6 +149,8 @@ export function buildRequest(options: BuildRequestOptions) {
       };
     }
   };
+
+  return withLegacyRequestReturn(requestFn);
 }
 
 // Returns either clerk_trace_id if present in response json, otherwise defaults to CF-Ray header
@@ -169,3 +171,31 @@ function parseErrors(data: unknown): ClerkAPIError[] {
   }
   return [];
 }
+
+type LegacyRequestFunction = <T>(requestOptions: ClerkBackendApiRequestOptions) => Promise<T>;
+
+// TODO(dimkl): Will be probably be dropped in next major version
+function withLegacyRequestReturn(cb: any): LegacyRequestFunction {
+  return async (...args) => {
+    // @ts-ignore
+    const { data, errors, totalCount, status, statusText, clerkTraceId } = await cb<T>(...args);
+    if (errors) {
+      // instead of passing `data: errors`, we have set the `error.errors` because
+      // the errors returned from callback is already parsed and passing them as `data`
+      // will not be able to assign them to the instance
+      const error = new ClerkAPIResponseError(statusText || '', {
+        data: [],
+        status,
+        clerkTraceId,
+      });
+      error.errors = errors;
+      throw error;
+    }
+
+    if (typeof totalCount !== 'undefined') {
+      return { data, totalCount };
+    }
+
+    return data;
+  };
+}

packages/backend/src/api/resources/Deserializer.ts

@@ -24,19 +24,16 @@ type ResourceResponse<T> = {
   data: T;
 };
 
-type PaginatedResponse<T> = {
-  data: T;
-  totalCount?: number;
+export type PaginatedResourceResponse<T> = ResourceResponse<T> & {
+  totalCount: number;
 };
 
-export function deserialize<U = any>(payload: unknown): PaginatedResponse<U> | ResourceResponse<U> {
+export function deserialize<U = any>(payload: unknown): PaginatedResourceResponse<U> | ResourceResponse<U> {
   let data, totalCount: number | undefined;
 
   if (Array.isArray(payload)) {
-    data = payload.map(item => jsonToObject(item)) as U;
-    totalCount = payload.length;
-
-    return { data, totalCount };
+    const data = payload.map(item => jsonToObject(item)) as U;
+    return { data };
   } else if (isPaginated(payload)) {
     data = payload.data.map(item => jsonToObject(item)) as U;
     totalCount = payload.total_count;

packages/backend/src/index.ts

@@ -4,12 +4,15 @@ import type { SDKMetadata } from '@clerk/types';
 
 import type { ApiClient, CreateBackendApiOptions } from './api';
 import { createBackendApiClient } from './api';
+import { withLegacyReturn } from './jwt/legacyReturn';
 import type { CreateAuthenticateRequestOptions } from './tokens/factory';
 import { createAuthenticateRequest } from './tokens/factory';
+import { verifyToken as _verifyToken } from './tokens/verify';
 
 export type { Organization, Session, User, WebhookEvent, WebhookEventType } from './api/resources';
 export type { VerifyTokenOptions } from './tokens/verify';
-export { verifyToken } from './tokens/verify';
+
+export const verifyToken = withLegacyReturn(_verifyToken);
 
 export type ClerkOptions = CreateBackendApiOptions &
   Partial<