Missing transient payload in registration form (server-side, browser)

[akhayyat]

Preflight checklist

• I could not find a solution in the existing issues, docs, nor discussions.
• I agree to follow this project's Code of Conduct.
• I have read and am following this repository's Contribution Guidelines.
• I have joined the Ory Community Slack.
• I am signed up to the Ory Security Patch Newsletter.

Ory Network Project

No response

Describe the bug

In a server-side web application, I am trying to get additional data in the registration form using transient payload, as documented in https://www.ory.sh/docs/kratos/bring-your-own-ui/custom-ui-basic-integration and https://www.ory.sh/docs/guides/integrate-with-ory-cloud-through-webhooks.

Instead of getting the input value entered in the form, my application web hook receives a transient_payload value of { }.

My understanding is that this would work if the registration request was json-encoded. My question is: doesn't transient payload support plain HTML forms (form-encoded)? (with multiple fields in the transient payload). And is this documented?

Reproducing the bug

1. Run kratos: docker compose -f quickstart.yml up --build --force-recreate
2. Submit registration form to http://127.0.0.1:4433/self-service/registration?flow=aa5be409-40f2-48eb-95a8-bcae4c59bdc1 - the form includes an HTML input element whose name is transient_payload.channel_name (I'm testing with one transient payload field now, but I will need to be able to include multiple fields).
3. Observe the body of the Ory web hook request received by the application server: "transient_payload": {}.

Relevant log output

`ctx` object

This is the ctx object received by the web application as a result of the web hook.

{
  "ctx": {
    "flow": {
      "expires_at": "2023-08-12T09:05:36.36373425Z",
      "id": "771c6318-55d9-4813-b90e-b51c3f16d516",
      "issued_at": "2023-08-12T08:55:36.36373425Z",
      "request_url": "http://127.0.0.1:4433/self-service/registration/browser",
      "transient_payload": {},
      "type": "browser",
      "ui": {
        "action": "http://127.0.0.1:4433/self-service/registration?flow=771c6318-55d9-4813-b90e-b51c3f16d516",
        "method": "POST",
        "nodes": [
          {
            "attributes": {
              "disabled": false,
              "name": "csrf_token",
              "node_type": "input",
              "required": true,
              "type": "hidden",
              "value": "goAwCBRkSB6GVDKToagMV45O7u6689IoyO64ON9Ywyw82wCqI21Ggok8fGwFJuXMGyCZbLGbBzBszkyY9TeiFA=="
            },
            "group": "default",
            "messages": [],
            "meta": {},
            "type": "input"
          },
          {
            "attributes": {
              "autocomplete": "email",
              "disabled": false,
              "name": "traits.email",
              "node_type": "input",
              "required": true,
              "type": "email",
              "value": "someone@example.com"
            },
            "group": "password",
            "messages": [],
            "meta": {
              "label": {
                "id": 1070002,
                "text": "E-Mail",
                "type": "info"
              }
            },
            "type": "input"
          },
          {
            "attributes": {
              "autocomplete": "new-password",
              "disabled": false,
              "name": "password",
              "node_type": "input",
              "required": true,
              "type": "password"
            },
            "group": "password",
            "messages": [
              {
                "context": {
                  "reason": "the password has been found in data breaches and must no longer be used"
                },
                "id": 4000005,
                "text": "The password can not be used because the password has been found in data breaches and must no longer be used.",
                "type": "error"
              }
            ],
            "meta": {
              "label": {
                "id": 1070001,
                "text": "Password",
                "type": "info"
              }
            },
            "type": "input"
          },
          {
            "attributes": {
              "disabled": false,
              "name": "method",
              "node_type": "input",
              "type": "submit",
              "value": "password"
            },
            "group": "password",
            "messages": [],
            "meta": {
              "label": {
                "context": {},
                "id": 1040001,
                "text": "Sign up",
                "type": "info"
              }
            },
            "type": "input"
          }
        ]
      }
    },
    "identity": {
      "created_at": "0001-01-01T00:00:00Z",
      "id": "00000000-0000-0000-0000-000000000000",
      "metadata_public": null,
      "recovery_addresses": [
        {
          "created_at": "0001-01-01T00:00:00Z",
          "id": "00000000-0000-0000-0000-000000000000",
          "updated_at": "0001-01-01T00:00:00Z",
          "value": "someone@example.com",
          "via": "email"
        }
      ],
      "schema_id": "default",
      "schema_url": "",
      "state": "active",
      "state_changed_at": "2023-08-12T08:56:25.117242092Z",
      "traits": {
        "email": "someone@example.com"
      },
      "updated_at": "0001-01-01T00:00:00Z",
      "verifiable_addresses": [
        {
          "created_at": "0001-01-01T00:00:00Z",
          "id": "00000000-0000-0000-0000-000000000000",
          "status": "pending",
          "updated_at": "0001-01-01T00:00:00Z",
          "value": "someone@example.com",
          "verified": false,
          "via": "email"
        }
      ]
    },
    "request_cookies": {
      "csrf_token_806060ca5bf70dff3caa0e5c860002aade9d470a5a4dce73bcfa7ba10778f481": "vlswojcJDpwPaE7/pI7pm5Vud4ILaNUYpCD0oCpvYTg="
    },
    "request_headers": {
      "Accept": [
        "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8"
      ],
      "Accept-Encoding": [
        "gzip, deflate, br"
      ],
      "Accept-Language": [
        "en-US,en;q=0.5"
      ],
      "Connection": [
        "keep-alive"
      ],
      "Content-Length": [
        "220"
      ],
      "Content-Type": [
        "application/x-www-form-urlencoded"
      ],
      "Cookie": [
        "csrf_token_806060ca5bf70dff3caa0e5c860002aade9d470a5a4dce73bcfa7ba10778f481=vlswojcJDpwPaE7/pI7pm5Vud4ILaNUYpCD0oCpvYTg="
      ],
      "Origin": [
        "null"
      ],
      "Sec-Fetch-Dest": [
        "document"
      ],
      "Sec-Fetch-Mode": [
        "navigate"
      ],
      "Sec-Fetch-Site": [
        "same-site"
      ],
      "Sec-Fetch-User": [
        "?1"
      ],
      "Upgrade-Insecure-Requests": [
        "1"
      ],
      "User-Agent": [
        "Mozilla/5.0 (X11; Linux x86_64; rv:102.0) Gecko/20100101 Firefox/102.0"
      ]
    },
    "request_method": "POST",
    "request_url": "http://127.0.0.1:4433/self-service/registration?flow=771c6318-55d9-4813-b90e-b51c3f16d516"
  },
  "email": "someone@example.com",
  "primary_channel": {
    "name": {},
    "visibility": "public"
  },
  "ui_language": "en",
  "user_id": {
    "created_at": "0001-01-01T00:00:00Z",
    "id": "00000000-0000-0000-0000-000000000000",
    "metadata_public": null,
    "recovery_addresses": [
      {
        "created_at": "0001-01-01T00:00:00Z",
        "id": "00000000-0000-0000-0000-000000000000",
        "updated_at": "0001-01-01T00:00:00Z",
        "value": "someone@example.com",
        "via": "email"
      }
    ],
    "schema_id": "default",
    "schema_url": "",
    "state": "active",
    "state_changed_at": "2023-08-12T08:56:25.117242092Z",
    "traits": {
      "email": "someone@example.com"
    },
    "updated_at": "0001-01-01T00:00:00Z",
    "verifiable_addresses": [
      {
        "created_at": "0001-01-01T00:00:00Z",
        "id": "00000000-0000-0000-0000-000000000000",
        "status": "pending",
        "updated_at": "0001-01-01T00:00:00Z",
        "value": "someone@example.com",
        "verified": false,
        "via": "email"
      }
    ]
  }
}

Relevant configuration

`quickstart.yml`

version: '3.7'
services:
  kratos-migrate:
    image: oryd/kratos:v1.0.0
    environment:
      - DSN=sqlite:///var/lib/sqlite/db.sqlite?_fk=true&mode=rwc
    volumes:
      - type: volume
        source: kratos-sqlite
        target: /var/lib/sqlite
        read_only: false
      - type: bind
        source: ./kratos-config
        target: /etc/config/kratos
    command: -c /etc/config/kratos/kratos.yml migrate sql -e --yes
    restart: on-failure
    networks:
      - intranet
  kratos:
    depends_on:
      - kratos-migrate
    image: oryd/kratos:v1.0.0
    ports:
      - '4433:4433' # public
      - '4434:4434' # admin
    restart: unless-stopped
    environment:
      - DSN=sqlite:///var/lib/sqlite/db.sqlite?_fk=true
      - LOG_LEVEL=trace
    command: serve -c /etc/config/kratos/kratos.yml --dev --watch-courier
    volumes:
      - type: volume
        source: kratos-sqlite
        target: /var/lib/sqlite
        read_only: false
      - type: bind
        source: ./kratos-config
        target: /etc/config/kratos
    networks:
      - intranet
  mailslurper:
    image: oryd/mailslurper:latest-smtps
    ports:
      - '4436:4436'
      - '4437:4437'
    networks:
      - intranet
networks:
  intranet:
volumes:
  kratos-sqlite:

`kratos.yml`

version: v0.13.0

dsn: memory

serve:
  public:
    base_url: http://127.0.0.1:4433/
    cors:
      enabled: true
  admin:
    base_url: http://kratos:4434/

selfservice:
  default_browser_return_url: http://127.0.0.1:8000/
  allowed_return_urls:
    - http://127.0.0.1:8000

  methods:
    password:
      enabled: true
    totp:
      config:
        issuer: Kratos
      enabled: true
    lookup_secret:
      enabled: true
    link:
      enabled: true
    code:
      enabled: true

  flows:
    error:
      ui_url: http://127.0.0.1:8000/users/error/

    settings:
      ui_url: http://127.0.0.1:8000/settings
      privileged_session_max_age: 15m
      required_aal: highest_available

    recovery:
      enabled: true
      ui_url: http://127.0.0.1:8000/recovery
      use: code

    verification:
      enabled: true
      ui_url: http://127.0.0.1:8000/users/verify/
      use: code
      after:
        default_browser_return_url: http://127.0.0.1:8000/

    logout:
      after:
        default_browser_return_url: http://127.0.0.1:8000/users/login

    login:
      ui_url: http://127.0.0.1:8000/users/login
      lifespan: 10m

    registration:
      lifespan: 10m
      ui_url: http://127.0.0.1:8000/users/join/
      after:
        password:
          hooks:
            - hook: session
            - hook: show_verification_ui
            - hook: web_hook
              config:
                url: http://172.17.0.1:8000/users/
                method: POST
                body: base64://ZnVuY3Rpb24oY3R4KSB7IHVzZXJfaWQ6IGN0eC5pZGVudGl0eSwgZW1haWw6IGN0eC5pZGVudGl0eS50cmFpdHMuZW1haWwsIHByaW1hcnlfY2hhbm5lbDogeyBuYW1lOiBjdHguZmxvdy50cmFuc2llbnRfcGF5bG9hZCwgdmlzaWJpbGl0eTogInB1YmxpYyIgfSwgdWlfbGFuZ3VhZ2U6ICJlbiIsIGN0eDogY3R4IH0=
                response:
                  parse: true
log:
  level: debug
  format: text
  leak_sensitive_values: true

secrets:
  cookie:
    - PLEASE-CHANGE-ME-I-AM-VERY-INSECURE
  cipher:
    - 32-LONG-SECRET-NOT-SECURE-AT-ALL

ciphers:
  algorithm: xchacha20-poly1305

hashers:
  algorithm: bcrypt
  bcrypt:
    cost: 8

identity:
  default_schema_id: default
  schemas:
    - id: default
      url: file:///etc/config/kratos/identity.schema.json

courier:
  smtp:
    connection_uri: smtps://test:test@mailslurper:1025/?skip_ssl_verify=true

Web Hook Body

The base64-encoded registration web hook body included in the kratos.yml file above decodes to:

function(ctx) { user_id: ctx.identity, email: ctx.identity.traits.email, primary_channel: { name: ctx.flow.transient_payload, visibility: "public" }, ui_language: "en", ctx: ctx }

Registration Form

<form action="http://127.0.0.1:4433/self-service/registration?flow=aa5be409-40f2-48eb-95a8-bcae4c59bdc1" method="POST">
  <div>
    <label for="id_traits.email">E-Mail:</label>
    <input type="text" name="traits.email" required id="id_traits.email">
  </div>
  <div>
    <label for="id_password">Password:</label>
    <input type="password" name="password" required id="id_password">
</div>
  <div>
    <label for="id_transient_payload.channel_name">Channel name:</label>
    <input type="text" name="transient_payload.channel_name" maxlength="200" required id="id_transient_payload.channel_name">
    <input type="hidden" name="csrf_token" value="uJeeOUCKxVmU9CwGM36rYNwSgVMF/p2qyNYfLLGkhhIGzK6bd4PLxZucYvmX8EL7SXz20Q6WSLJs9uuMm8vnKg==" id="id_csrf_token">
  </div>
  <button name="method" type="submit" value="password">Join</button>
</form>

Version

1.0.0

On which operating system are you observing this issue?

Linux

In which environment are you deploying?

Docker Compose

Additional Context

No response

[meysam81]

@akhayyat and anyone else encountering the same issue...
I was struggling with the same problem and it turns out that unlike traits, you can't pass the transient_payload as dot-separated key-values because that is supposed to be nested when passed to the Kratos server.
So, effectively here's the payload the Kratos is expecting:

{
  "traits.something": "some-value",
  "traits.some-other-thing": "another-value",
  "transient_payload": {
    "nested-key": "this-works"
  }
}

Note that traits are dot separated keys but transient_payload has to be nested! This is not documented anywhere!!!

The problem with this approach however, is that I don't know how to embed this in HTML because when it was simple dot separated names, you could simply have an input in the HTML with property name="traits.something" but I have no idea how to make it nested for the transient_payload.

(The realization of the proposed solution for me was to initiate a native-mobile flow and then passing the nested JSON in the terminal)

The jsonnet for the above payload would be the following:

function(ctx) {
  [if 'nested-key' in ctx.flow.transient_payload then 'nested-key']: ctx.flow.transient_payload.nested-key
}

Another problem with the documentation is that when trying to explain how to use transient_payload in the Jsonnet, it is omitting the flow between the ctx and the transient_payload as seen in the following screenshot.

I expect to see ctx.flow.transient_payload.custom_data but the flow is missing.

[rbnbr]

I had the same problem and was glad I found your comment @meysam81, otherwise I would probably have too wasted many more hours trying to figure out why stuff is not working even though I stick to the documentation.

To @akhayyat, my current solution for still using the dotted input field names and x-www-form-urlencoded encoding is to keep an empty hidden transient_payload field in the form and then update its content via onsubmit callback as in the example below:

<form method="POST" action="/.ory/self-service/registration?flow={{.FlowID}}" onsubmit="combineTransientPayload();" id="registrationFormID">
  <input type="hidden" name="transient_payload" value="{}" id="transientPayloadID">
  <input type="hidden" name="transient_payload.captcha_token" value="{{.CaptchaToken}}">
  <input type="hidden" name="csrf_token" value="{{.CSRFToken}}">
  <input type="hidden" name="method" value="password">
  <input type="email" name="traits.email" class="form-control" id="inputEmail" required>
  <input type="text" name="traits.username" id="inputUsername">
  <input type="password" name="password" class="form-control" id="inputPassword" required>
  <img id="captchaImageID" src="data:image/png;base64, {{.CaptchaBase64}}" alt="captcha">
  <input type="text" name="transient_payload.captcha" id="inputCaptchaID" required>
  <button type="submit" class="btn btn-primary">Register</button>
</form>

<script>
    function combineTransientPayload(event) {
    let form = document.getElementById("registrationFormID");
    let transientPayloadInputs = Array.from(form.getElementsByTagName("input")).filter(el => el.name.startsWith("transient_payload."));

    let transientPayload = {};
    transientPayloadInputs.forEach(el => {
      let name = el.name;
      let identifiers = name.split('.').slice(1);
      let currentObject = transientPayload;
      for (let i = 0; i < identifiers.length-1; i++) {
        if (!(identifiers[i] in currentObject)) {
          currentObject[identifiers[i]] = {};
        }
        currentObject = currentObject[identifiers[i]];
      }
      currentObject[identifiers[identifiers.length-1]] = el.value;
    });

    document.getElementById("transientPayloadID").value = JSON.stringify(transientPayload);
  }
</script>

With this I can access the payload without problems using the jsonnet function below:

function(ctx) {
  captcha: ctx.flow.transient_payload.captcha,
  captchaToken: ctx.flow.transient_payload.captcha_token
}

[aeneasr]

I‘m moving this to docs so we can improve this there. fyi @christiannwamba