improve HttpApi handling of payload encoding types (#4024)

Author: tim-smart

.changeset/spicy-apes-rescue.md

@@ -0,0 +1,6 @@
+---
+"@effect/platform-node": patch
+"@effect/platform": patch
+---
+
+improve HttpApi handling of payload encoding types

packages/platform-node/test/HttpApi.test.ts

@@ -211,7 +211,7 @@ describe("HttpApi", () => {
       assert.strictEqual(response.status, 200)
     }).pipe(Effect.provide(HttpLive)))
 
-  it.effect("multipart or json", () =>
+  it.effect("multiple payload types", () =>
     Effect.gen(function*() {
       const client = yield* HttpApiClient.make(Api)
       let [group, response] = yield* client.groups.create({
@@ -229,6 +229,11 @@ describe("HttpApi", () => {
       })
       assert.deepStrictEqual(group, new Group({ id: 1, name: "Some group" }))
       assert.strictEqual(response.status, 200)
+
+      group = yield* client.groups.create({
+        payload: { foo: "Some group" }
+      })
+      assert.deepStrictEqual(group, new Group({ id: 1, name: "Some group" }))
     }).pipe(Effect.provide(HttpLive)))
 
   it("OpenAPI spec", () => {
@@ -287,6 +292,9 @@ class GroupsApi extends HttpApiGroup.make("groups")
     HttpApiEndpoint.post("create", "/")
       .setPayload(Schema.Union(
         Schema.Struct(Struct.pick(Group.fields, "name")),
+        Schema.Struct({ foo: Schema.String }).pipe(
+          HttpApiSchema.withEncoding({ kind: "UrlParams" })
+        ),
         HttpApiSchema.Multipart(
           Schema.Struct(Struct.pick(Group.fields, "name"))
         )
@@ -456,7 +464,13 @@ const HttpGroupsLive = HttpApiBuilder.group(
         path.id === 0
           ? Effect.fail(new GroupError())
           : Effect.succeed(new Group({ id: 1, name: "foo" })))
-      .handle("create", ({ payload }) => Effect.succeed(new Group({ id: 1, name: payload.name })))
+      .handle("create", ({ payload }) =>
+        Effect.succeed(
+          new Group({
+            id: 1,
+            name: "foo" in payload ? payload.foo : payload.name
+          })
+        ))
 )
 
 const HttpApiLive = Layer.provide(HttpApiBuilder.api(Api), [

packages/platform-node/test/fixtures/openapi.json

@@ -113,6 +113,18 @@
                 "additionalProperties": false
               }
             },
+            "application/x-www-form-urlencoded": {
+              "schema": {
+                "type": "object",
+                "required": ["foo"],
+                "properties": {
+                  "foo": {
+                    "type": "string"
+                  }
+                },
+                "additionalProperties": false
+              }
+            },
             "multipart/form-data": {
               "schema": {
                 "type": "object",

packages/platform/src/HttpApi.ts

@@ -282,6 +282,10 @@ export const reflect = <Groups extends HttpApiGroup.HttpApiGroup.Any, Error, R>(
       readonly endpoint: HttpApiEndpoint.HttpApiEndpoint<string, HttpMethod>
       readonly mergedAnnotations: Context.Context<never>
       readonly middleware: ReadonlySet<HttpApiMiddleware.TagClassAny>
+      readonly payloads: ReadonlyMap<string, {
+        readonly encoding: HttpApiSchema.Encoding
+        readonly ast: AST.AST
+      }>
       readonly successes: ReadonlyMap<number, Option.Option<AST.AST>>
       readonly errors: ReadonlyMap<number, Option.Option<AST.AST>>
     }) => void
@@ -311,6 +315,7 @@ export const reflect = <Groups extends HttpApiGroup.HttpApiGroup.Any, Error, R>(
         endpoint,
         middleware: new Set([...group.middlewares, ...endpoint.middlewares]),
         mergedAnnotations: Context.merge(groupAnnotations, endpoint.annotations),
+        payloads: endpoint.payloadSchema._tag === "Some" ? extractPayloads(endpoint.payloadSchema.value.ast) : emptyMap,
         successes: extractMembers(endpoint.successSchema.ast, new Map(), HttpApiSchema.getStatusSuccessAST),
         errors
       })
@@ -320,6 +325,8 @@ export const reflect = <Groups extends HttpApiGroup.HttpApiGroup.Any, Error, R>(
 
 // -------------------------------------------------------------------------------------
 
+const emptyMap = new Map<never, never>()
+
 const extractMembers = (
   topAst: AST.AST,
   inherited: ReadonlyMap<number, Option.Option<AST.AST>>,
@@ -361,6 +368,44 @@ const extractMembers = (
   return members
 }
 
+const extractPayloads = (topAst: AST.AST): ReadonlyMap<string, {
+  readonly encoding: HttpApiSchema.Encoding
+  readonly ast: AST.AST
+}> => {
+  const members = new Map<string, {
+    encoding: HttpApiSchema.Encoding
+    ast: AST.AST
+  }>()
+  function process(ast: AST.AST) {
+    if (ast._tag === "NeverKeyword") {
+      return
+    }
+    ast = AST.annotations(ast, {
+      ...topAst.annotations,
+      ...ast.annotations
+    })
+    const encoding = HttpApiSchema.getEncoding(ast)
+    const contentType = HttpApiSchema.getMultipart(ast) ? "multipart/form-data" : encoding.contentType
+    const current = members.get(contentType)
+    if (current === undefined) {
+      members.set(contentType, {
+        encoding,
+        ast
+      })
+    } else {
+      current.ast = AST.Union.make([current.ast, ast])
+    }
+  }
+  if (topAst._tag === "Union") {
+    for (const type of topAst.types) {
+      process(type)
+    }
+  } else {
+    process(topAst)
+  }
+  return members
+}
+
 /**
  * @since 1.0.0
  * @category tags

packages/platform/src/HttpApiBuilder.ts

@@ -39,6 +39,7 @@ import * as HttpServerRequest from "./HttpServerRequest.js"
 import * as HttpServerResponse from "./HttpServerResponse.js"
 import * as OpenApi from "./OpenApi.js"
 import type { Path } from "./Path.js"
+import * as UrlParams from "./UrlParams.js"
 
 /**
  * The router that the API endpoints are attached to.
@@ -516,12 +517,24 @@ const requestPayload = (
   | FileSystem
   | Path
   | Scope
-> =>
-  HttpMethod.hasBody(request.method)
-    ? request.headers["content-type"].includes("multipart/form-data")
-      ? Effect.orDie(request.multipart)
-      : Effect.orDie(request.json)
-    : Effect.succeed(urlParams)
+> => {
+  if (!HttpMethod.hasBody(request.method)) {
+    return Effect.succeed(urlParams)
+  }
+  const contentType = request.headers["content-type"]
+    ? request.headers["content-type"].toLowerCase().trim()
+    : "application/json"
+  if (contentType.includes("application/json")) {
+    return Effect.orDie(request.json)
+  } else if (contentType.includes("multipart/form-data")) {
+    return Effect.orDie(request.multipart)
+  } else if (contentType.includes("x-www-form-urlencoded")) {
+    return Effect.map(Effect.orDie(request.urlParamsBody), UrlParams.toRecord)
+  } else if (contentType.startsWith("text/")) {
+    return Effect.orDie(request.text)
+  }
+  return Effect.map(Effect.orDie(request.arrayBuffer), (buffer) => new Uint8Array(buffer))
+}
 
 type MiddlewareMap = Map<string, {
   readonly tag: HttpApiMiddleware.TagClassAny

packages/platform/src/HttpApiClient.ts

@@ -4,6 +4,7 @@
 import * as Context from "effect/Context"
 import * as Effect from "effect/Effect"
 import { identity } from "effect/Function"
+import { globalValue } from "effect/GlobalValue"
 import * as Option from "effect/Option"
 import * as ParseResult from "effect/ParseResult"
 import type * as Predicate from "effect/Predicate"
@@ -15,12 +16,14 @@ import * as HttpApi from "./HttpApi.js"
 import type { HttpApiEndpoint } from "./HttpApiEndpoint.js"
 import type { HttpApiGroup } from "./HttpApiGroup.js"
 import * as HttpApiSchema from "./HttpApiSchema.js"
+import * as HttpBody from "./HttpBody.js"
 import * as HttpClient from "./HttpClient.js"
 import * as HttpClientError from "./HttpClientError.js"
 import * as HttpClientRequest from "./HttpClientRequest.js"
 import * as HttpClientResponse from "./HttpClientResponse.js"
 import * as HttpMethod from "./HttpMethod.js"
 import type { HttpApiMiddleware } from "./index.js"
+import * as UrlParams from "./UrlParams.js"
 
 /**
  * @since 1.0.0
@@ -160,8 +163,13 @@ const makeClient = <Groups extends HttpApiGroup.Any, ApiError, ApiR>(
         successes.forEach((ast, status) => {
           decodeMap[status] = ast._tag === "None" ? responseAsVoid : schemaToResponse(ast.value)
         })
-        const encodePayload = endpoint.payloadSchema.pipe(
-          Option.map(Schema.encodeUnknown)
+        const encodePayloadBody = endpoint.payloadSchema.pipe(
+          Option.map((schema) => {
+            if (HttpMethod.hasBody(endpoint.method)) {
+              return Schema.encodeUnknown(payloadSchemaBody(schema as any))
+            }
+            return Schema.encodeUnknown(schema)
+          })
         )
         const encodeHeaders = endpoint.headersSchema.pipe(
           Option.map(Schema.encodeUnknown)
@@ -180,13 +188,16 @@ const makeClient = <Groups extends HttpApiGroup.Any, ApiError, ApiR>(
             let httpRequest = HttpClientRequest.make(endpoint.method)(
               request && request.path ? makeUrl(request.path) : endpoint.path
             )
-            if (request.payload instanceof FormData) {
+            if (request && request.payload instanceof FormData) {
               httpRequest = HttpClientRequest.bodyFormData(httpRequest, request.payload)
-            } else if (encodePayload._tag === "Some") {
-              const payload = yield* encodePayload.value(request.payload)
-              httpRequest = HttpMethod.hasBody(endpoint.method)
-                ? yield* Effect.orDie(HttpClientRequest.bodyJson(httpRequest, payload))
-                : HttpClientRequest.setUrlParams(httpRequest, payload as any)
+            } else if (encodePayloadBody._tag === "Some") {
+              if (HttpMethod.hasBody(endpoint.method)) {
+                const body = (yield* encodePayloadBody.value(request.payload)) as HttpBody.HttpBody
+                httpRequest = HttpClientRequest.setBody(httpRequest, body)
+              } else {
+                const urlParams = (yield* encodePayloadBody.value(request.payload)) as Record<string, string>
+                httpRequest = HttpClientRequest.setUrlParams(httpRequest, urlParams)
+              }
             }
             if (encodeHeaders._tag === "Some") {
               httpRequest = HttpClientRequest.setHeaders(
@@ -416,3 +427,58 @@ const statusCodeError = (response: HttpClientResponse.HttpClientResponse) =>
   )
 
 const responseAsVoid = (_response: HttpClientResponse.HttpClientResponse) => Effect.void
+
+const HttpBodyFromSelf = Schema.declare(HttpBody.isHttpBody)
+
+const payloadSchemaBody = (schema: Schema.Schema.All): Schema.Schema<any, HttpBody.HttpBody> => {
+  const members = schema.ast._tag === "Union" ? schema.ast.types : [schema.ast]
+  return Schema.Union(...members.map(bodyFromPayload)) as any
+}
+
+const bodyFromPayloadCache = globalValue(
+  "@effect/platform/HttpApiClient/bodyFromPayloadCache",
+  () => new WeakMap<AST.AST, Schema.Schema.Any>()
+)
+
+const bodyFromPayload = (ast: AST.AST) => {
+  if (bodyFromPayloadCache.has(ast)) {
+    return bodyFromPayloadCache.get(ast)!
+  }
+  const schema = Schema.make(ast)
+  const encoding = HttpApiSchema.getEncoding(ast)
+  const transform = Schema.transformOrFail(
+    HttpBodyFromSelf,
+    schema,
+    {
+      decode(fromA, _, ast) {
+        return ParseResult.fail(new ParseResult.Forbidden(ast, fromA, "encode only schema"))
+      },
+      encode(toI, _, ast) {
+        switch (encoding.kind) {
+          case "Json": {
+            return HttpBody.json(toI).pipe(
+              ParseResult.mapError((error) => new ParseResult.Type(ast, toI, `Could not encode as JSON: ${error}`))
+            )
+          }
+          case "Text": {
+            if (typeof toI !== "string") {
+              return ParseResult.fail(new ParseResult.Type(ast, toI, "Expected a string"))
+            }
+            return ParseResult.succeed(HttpBody.text(toI))
+          }
+          case "UrlParams": {
+            return ParseResult.succeed(HttpBody.urlParams(UrlParams.fromInput(toI as any)))
+          }
+          case "Uint8Array": {
+            if (!(toI instanceof Uint8Array)) {
+              return ParseResult.fail(new ParseResult.Type(ast, toI, "Expected a Uint8Array"))
+            }
+            return ParseResult.succeed(HttpBody.uint8Array(toI))
+          }
+        }
+      }
+    }
+  )
+  bodyFromPayloadCache.set(ast, transform)
+  return transform
+}

packages/platform/src/OpenApi.ts

@@ -231,7 +231,7 @@ export const fromApi = <A extends HttpApi.HttpApi.Any>(self: A): OpenAPISpec =>
       })
       spec.tags!.push(tag)
     },
-    onEndpoint({ endpoint, errors, group, middleware, successes }) {
+    onEndpoint({ endpoint, errors, group, middleware, payloads, successes }) {
       const path = endpoint.path.replace(/:(\w+)[^/]*/g, "{$1}")
       const method = endpoint.method.toLowerCase() as OpenAPISpecMethodName
       const op: DeepMutable<OpenAPISpecOperation> = {
@@ -266,35 +266,15 @@ export const fromApi = <A extends HttpApi.HttpApi.Any>(self: A): OpenAPISpec =>
           op.security!.push({ [name]: [] })
         }
       })
-      endpoint.payloadSchema.pipe(
-        Option.filter(() => HttpMethod.hasBody(endpoint.method)),
-        Option.map((schema) => {
-          const content: Mutable<OpenApiSpecContent> = {}
-          const members = schema.ast._tag === "Union" ? schema.ast.types : [schema.ast]
-          const jsonTypes: Array<AST.AST> = []
-          const multipartTypes: Array<AST.AST> = []
-
-          for (const member of members) {
-            if (HttpApiSchema.getMultipart(member)) {
-              multipartTypes.push(member)
-            } else {
-              jsonTypes.push(member)
-            }
-          }
-
-          if (jsonTypes.length > 0) {
-            content["application/json"] = {
-              schema: makeJsonSchemaOrRef(Schema.make(AST.Union.make(jsonTypes)))
-            }
+      if (payloads.size > 0) {
+        const content: Mutable<OpenApiSpecContent> = {}
+        payloads.forEach(({ ast }, contentType) => {
+          content[contentType as OpenApiSpecContentType] = {
+            schema: makeJsonSchemaOrRef(Schema.make(ast))
           }
-          if (multipartTypes.length > 0) {
-            content["multipart/form-data"] = {
-              schema: makeJsonSchemaOrRef(Schema.make(AST.Union.make(multipartTypes)))
-            }
-          }
-          op.requestBody = { content, required: true }
         })
-      )
+        op.requestBody = { content, required: true }
+      }
       for (const [status, ast] of successes) {
         if (op.responses![status]) continue
         op.responses![status] = {

packages/platform/src/UrlParams.ts

@@ -210,6 +210,40 @@ const baseUrl = (): string | undefined => {
   return undefined
 }
 
+/**
+ * Builds a `Record` containing all the key-value pairs in the given `UrlParams`
+ * as `string` (if only one value for a key) or a `NonEmptyArray<string>`
+ * (when more than one value for a key)
+ *
+ * @example
+ * import { UrlParams } from "@effect/platform"
+ *
+ * const urlParams = UrlParams.fromInput({ a: 1, b: true, c: "string", e: [1, 2, 3] })
+ * const result = UrlParams.toRecord(urlParams)
+ *
+ * assert.deepStrictEqual(
+ *   result,
+ *   { "a": "1", "b": "true", "c": "string", "e": ["1", "2", "3"] }
+ * )
+ *
+ * @since 1.0.0
+ * @category conversions
+ */
+export const toRecord = (self: UrlParams): Record<string, string | Arr.NonEmptyArray<string>> => {
+  const out: Record<string, string | Arr.NonEmptyArray<string>> = {}
+  for (const [k, value] of self) {
+    const curr = out[k]
+    if (curr === undefined) {
+      out[k] = value
+    } else if (typeof curr === "string") {
+      out[k] = [curr, value]
+    } else {
+      curr.push(value)
+    }
+  }
+  return out
+}
+
 /**
  * @since 1.0.0
  * @category schema