[BUG] rest_cherrypy fails wheel auth with token

[nf-brentsaner]

Description
Try as I might, rest_cherrypy fails to auth properly for wheel commands when using a token. (And returns a 500 instead of a 401.)

Setup
(some values replaced with facsimiles)

master config

rest_cherrypy:
  port: 8000
  host: 127.0.0.1
  log_access_file: /var/log/salt/api.access.log
  log_error_file: /var/log/salt/api.error.log
  disable_ssl: True
  debug: True
  static: /srv/http/salt_api
auth.ldap.server: SOME.CORPDOMAIN.TLD
auth.ldap.port: 389
auth.ldap.tls: False
auth.ldap.starttls: False
auth.ldap.scope: 2
auth.ldap.no_verify: True
auth.ldap.anonymous: False
auth.ldap.activedirectory: True
auth.ldap.persontype: 'person'
auth.ldap.basedn: 'dc=domain,dc=tld'
auth.ldap.binddn: "{{ username }}@SOME.CORPDOMAIN.TLD"
auth.ldap.accountattributename: 'sAMAccountName'
auth.ldap.auth_by_group_membership_only: False
auth.ldap.groupclass: 'group'
auth.ldap.groupattribute: 'memberOf'
external_auth:
  ldap:
    Corp Engineers%:
      - .*
      - '@wheel'
      - '@runner'
      - '@jobs'
    'Corp Technicians%':
      - '*':
        - 'state.apply'
        - 'test.ping'
ext_pillar:
  - saltsources: []
  - file_tree:
      root_dir: '/srv/salt/sys/_pillars/files'
      follow_dir_links: False
      keep_newline: True
user: saltstack
group: saltstack
pidfile: /var/run/salt/master.pid
log_level: warning
log_level_logfile: debug
minion_data_cache: False
use_superseded:
  - module.run
interface: '::'
ipv6: True
timeout: 15
show_jid: True
strip_colors: False
cli_summary: True
fileserver_backend:
  - roots
  - gitfs
file_roots:
  base:
    - /srv/salt/sys
    - /srv/salt/winpkg
gitfs_provider: pygit2
pillar_roots:
  base:
    - /srv/salt/sys/_pillars
    - /srv/salt/winpkg/_pillars
ext_pillar_first: True
module_dirs:
  - /srv/salt/_meta/lib
extension_modules: None
keysize: 4096
sign_pub_messages: True
hash_type: sha512
failhard: False
roster: saltsources
ssh_port: 22
ssh_scan_ports: 22
ssh_scan_timeout: 0.5
ssh_sudo: False
ssh_run_pre_flight: True
ssh_timeout: 60
ssh_user: root
ssh_use_home_key: True
roster_defaults:
  user: root
  sudo: False
  priv: /var/lib/saltstack/.ssh/id_ed25519
  tty: True
  ssh_options:
    - StrictHostKeyChecking=accept-new
    - ChallengeResponseAuthentication=no
    - KexAlgorithms=curve25519-sha256@libssh.org,diffie-hellman-group-exchange-sha256
    - Ciphers=chacha20-poly1305@openssh.com,aes256-gcm@openssh.com,aes128-gcm@openssh.com,aes256-ctr,aes192-ctr,aes128-ctr
    - MACs=hmac-sha2-512-etm@openssh.com,hmac-sha2-256-etm@openssh.com,umac-128-etm@openssh.com,hmac-sha2-512,hmac-sha2-256,umac-128@openssh.com
    - HostKeyAlgorithms=ssh-ed25519-cert-v01@openssh.com,ssh-rsa-cert-v01@openssh.com,ssh-ed25519,ssh-rsa
winrepo_remotes: []
winrepo_provider: pygit2
winrepo_dir_ng: /srv/salt/winpkg/win/repo-ng
winrepo_pubkey: /var/local/saltstack/.ssh/winrepo_ecdsa.pub
winrepo_privkey: /var/local/saltstack/.ssh/winrepo_ecdsa
winrepo_remotes_ng:
  - https://github.com/saltstack/salt-winrepo-ng.git
  - git@github.com:corpgit/winrepo-ng.git

nginx config

server {
        listen [::]:80 default_server ipv6only=off;
        server_name _;

        location /.well-known/acme-challenge {
                root            /var/lib/letsencrypt;
                default_type    "text/plain";
                try_files       $uri =404;
        }

        location / {
                return 302 "https://$host$request_uri";
        }
}

server {
        listen [::]:443 default_server ipv6only=off ssl http2;
        server_name _;

        error_page 404 /usr/share/nginx/errors/404.html;

        access_log /var/log/nginx/vhost/salt/access.log main;
        error_log /var/log/nginx/vhost/salt/error.log;
        #error_log /var/log/nginx/vhost/salt/debug.log debug;

        ssl_certificate /etc/letsencrypt/live/domain.tld/fullchain.pem;
        ssl_certificate_key /etc/letsencrypt/live/domain.tld/privkey.pem;

        ssl_dhparam /etc/nginx/tls/key/dh4096.pem;
        ssl_prefer_server_ciphers on;
        keepalive_timeout 300;
        ssl_session_cache shared:SSL:60m;
        ssl_session_timeout 60m;
        ssl_stapling on;
        ssl_stapling_verify on;
        resolver 198.51.100.1 198.51.100.2 valid=300s;
        resolver_timeout 5s;
        ssl_protocols TLSv1.3;
        ssl_ciphers 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:CAMELLIA:DES-CBC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA';
        add_header Strict-Transport-Security "max-age=31536000; includeSubdomains";

        root /srv/http/salt_api;
        index index.php index.html index.htm;

        location @salt_api {
                proxy_pass http://127.0.0.1:8000;
                proxy_http_version 1.1;
                proxy_set_header Host $host;
                proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
                proxy_set_header Accept-Encoding "";
                proxy_set_header X-Forwarded-Proto $scheme;
                proxy_set_header X-Real-IP $remote_addr;
        }

        location / {
                try_files $uri $uri/ @salt_api;
                autoindex off;
        }
}

Please be as specific as possible and give set-up details.

• on-prem machine
• VM (KVM)
• VM running on a cloud service, please be explicit and add details
• container (Kubernetes, Docker, containerd, etc. please specify)
• or a combination, please be explicit
• jails if it is FreeBSD
• classic packaging
• onedir packaging
• used bootstrap to install

Steps to Reproduce the behavior
Refer to above master log and nginx config.

Reverse-proxy to salt-api via Nginx using above configuration.

Use LDAP auth to AD.

Tomfoolery ensues. Assume that ${user} below is indeed a member of Corp Engineers, and per the master debug log is properly detected as such.

Getting a token succeeds.

token_out=$(curl -sL \
	-H "Accept: application/json" \
	-d username="${uname}" \
	-d password="${passwd}" \
	-d eauth="ldap" \
	"https://domain.tld/login")

token=$(echo "${token_out}" | jq -r .return[0].token)

and SOME endpoints work fine...

curl -sSL \
	-H "Accept: application/json" \
	-H "X-Auth-Token: ${token}" \
	"https://domain.tld/stats" | jq .

...and some calls, like `/keys`, fail catastrophically if...

X-Auth-Token is used:

curl -sSL \
	-o /tmp/keygen/minionkeys.tar \
	-H "Accept: application/x-tar" \
	-d mid="minion_id_name" \
	-H "X-Auth-Token: ${token}" \
	"https://domain.tld/keys"
####
<!DOCTYPE html PUBLIC
"-//W3C//DTD XHTML 1.0 Transitional//EN"
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html>
<head>
    <meta http-equiv="Content-Type" content="text/html; charset=utf-8"></meta>
    <title>500 Internal Server Error</title>
    <style type="text/css">
    #powered_by {
        margin-top: 20px;
        border-top: 2px solid black;
        font-style: italic;
    }

    #traceback {
        color: red;
    }
    </style>
</head>
    <body>
        <h2>500 Internal Server Error</h2>
        <p>The server encountered an unexpected condition which prevented it from fulfilling the request.</p>
        <pre id="traceback">Traceback (most recent call last):
  File "cherrypy/_cprequest.py", line 638, in respond
    self._do_respond(path_info)
  File "cherrypy/_cprequest.py", line 697, in _do_respond
    response.body = self.handler()
  File "cherrypy/lib/encoding.py", line 223, in __call__
    self.body = self.oldhandler(*args, **kwargs)
  File "cherrypy/_cpdispatch.py", line 54, in __call__
    return self.callable(*self.args, **self.kwargs)
  File "salt/netapi/rest_cherrypy/app.py", line 1721, in POST
    ret = next(result, {}).get("data", {}).get("return", {})
  File "salt/netapi/rest_cherrypy/app.py", line 1217, in exec_lowstate
    ret = self.api.run(chunk)
  File "salt/netapi/__init__.py", line 161, in run
    raise salt.exceptions.EauthAuthenticationError(
salt.exceptions.EauthAuthenticationError: No authentication credentials given
</pre>
    <div id="powered_by">
      <span>
        Powered by <a href="http://www.cherrypy.org">CherryPy 18.6.1</a>
      </span>
    </div>
    </body>
</html>

or -d token=${token} is used:

curl -sSL \
	-o /tmp/keygen/minionkeys.tar \
	-H "Accept: application/x-tar" \
	-d mid="minion_id_name" \
	-d token="${token}" \
	"https://domain.tld/keys" 
####
!DOCTYPE html PUBLIC
"-//W3C//DTD XHTML 1.0 Transitional//EN"
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html>
<head>
    <meta http-equiv="Content-Type" content="text/html; charset=utf-8"></meta>
    <title>500 Internal Server Error</title>
    <style type="text/css">
    #powered_by {
        margin-top: 20px;
        border-top: 2px solid black;
        font-style: italic;
    }

    #traceback {
        color: red;
    }
    </style>
</head>
    <body>
        <h2>500 Internal Server Error</h2>
        <p>The server encountered an unexpected condition which prevented it from fulfilling the request.</p>
        <pre id="traceback">Traceback (most recent call last):
  File "cherrypy/_cprequest.py", line 638, in respond
    self._do_respond(path_info)
  File "cherrypy/_cprequest.py", line 697, in _do_respond
    response.body = self.handler()
  File "cherrypy/lib/encoding.py", line 223, in __call__
    self.body = self.oldhandler(*args, **kwargs)
  File "cherrypy/_cpdispatch.py", line 54, in __call__
    return self.callable(*self.args, **self.kwargs)
  File "salt/netapi/rest_cherrypy/app.py", line 1721, in POST
    ret = next(result, {}).get("data", {}).get("return", {})
  File "salt/netapi/rest_cherrypy/app.py", line 1217, in exec_lowstate
    ret = self.api.run(chunk)
  File "salt/netapi/__init__.py", line 175, in run
    return l_fun(*f_call.get("args", ()), **f_call.get("kwargs", {}))
  File "salt/netapi/__init__.py", line 286, in wheel
    return wheel.cmd_sync(kwargs)
  File "salt/wheel/__init__.py", line 102, in cmd_sync
    return self.master_call(**low)
  File "salt/wheel/__init__.py", line 80, in master_call
    salt.utils.error.raise_error(**ret["error"])
  File "salt/utils/error.py", line 29, in raise_error
    raise ex(message)
salt.exceptions.TokenAuthenticationError: Authentication failure of type "token" occurred.
</pre>
    <div id="powered_by">
      <span>
        Powered by <a href="http://www.cherrypy.org">CherryPy 18.6.1</a>
      </span>
    </div>
    </body>
</html>

But succeeds for eauth.

curl -sSL \
	-o /tmp/keygen/minionkeys.tar \
	-H "Accept: application/x-tar" \
	-d mid="minion_id_name" \
	-d username="${uname}" \
	-d password="${passwd}" \
	-d eauth="ldap" \
	"https://domain.tld/keys"

$ file minionkeys.tar 
minionkeys.tar: POSIX tar archive

Expected behavior
A valid token should work for e.g. @wheel endpoints provided permissions allow for it.

Screenshots
N/A

Versions Report

salt --versions-report

(Provided by running salt --versions-report. Please also mention any differences in master/minion versions.)

salt-master:

Salt Version:
          Salt: 3005.1
 
Dependency Versions:
          cffi: 1.14.6
      cherrypy: unknown
      dateutil: 2.8.1
     docker-py: Not Installed
         gitdb: Not Installed
     gitpython: Not Installed
        Jinja2: 3.1.0
       libgit2: 1.5.0
      M2Crypto: Not Installed
          Mako: Not Installed
       msgpack: 1.0.2
  msgpack-pure: Not Installed
  mysql-python: Not Installed
     pycparser: 2.21
      pycrypto: Not Installed
  pycryptodome: 3.9.8
        pygit2: 1.11.1
        Python: 3.9.16 (main, Jan  6 2023, 22:52:19)
  python-gnupg: 0.4.8
        PyYAML: 5.4.1
         PyZMQ: 23.2.0
         smmap: Not Installed
       timelib: 0.2.4
       Tornado: 4.5.3
           ZMQ: 4.3.4
 
System Versions:
          dist: almalinux 8.7 Stone Smilodon
        locale: utf-8
       machine: x86_64
       release: 4.18.0-425.13.1.el8_7.x86_64
        system: Linux
       version: AlmaLinux 8.7 Stone Smilodon

salt-api (same VM, same installation method, etc.):

Salt Version:
          Salt: 3005.1
 
Dependency Versions:
          cffi: 1.14.6
      cherrypy: 18.6.1
      dateutil: 2.8.1
     docker-py: Not Installed
         gitdb: Not Installed
     gitpython: Not Installed
        Jinja2: 3.1.0
       libgit2: 1.5.0
      M2Crypto: Not Installed
          Mako: Not Installed
       msgpack: 1.0.2
  msgpack-pure: Not Installed
  mysql-python: Not Installed
     pycparser: 2.21
      pycrypto: Not Installed
  pycryptodome: 3.9.8
        pygit2: 1.11.1
        Python: 3.9.16 (main, Jan  6 2023, 22:52:19)
  python-gnupg: 0.4.8
        PyYAML: 5.4.1
         PyZMQ: 23.2.0
         smmap: Not Installed
       timelib: 0.2.4
       Tornado: 4.5.3
           ZMQ: 4.3.4
 
System Versions:
          dist: almalinux 8.7 Stone Smilodon
        locale: utf-8
       machine: x86_64
       release: 4.18.0-425.13.1.el8_7.x86_64
        system: Linux
       version: AlmaLinux 8.7 Stone Smilodon

Additional context
N/A

[welcome]

Hi there! Welcome to the Salt Community! Thank you for making your first contribution. We have a lengthy process for issues and PRs. Someone from the Core Team will follow up as soon as possible. In the meantime, here’s some information that may help as you continue your Salt journey.
Please be sure to review our Code of Conduct. Also, check out some of our community resources including:

• Community Wiki
• Salt’s Contributor Guide
• Join our Community Slack
• IRC on LiberaChat
• Salt Project YouTube channel
• Salt Project Twitch channel

There are lots of ways to get involved in our community. Every month, there are around a dozen opportunities to meet with other contributors and the Salt Core team and collaborate in real time. The best way to keep track is by subscribing to the Salt Community Events Calendar.
If you have additional questions, email us at saltproject@vmware.com. We’re glad you’ve joined our community and look forward to doing awesome things with you!