fix(deploy,update): controller-only manifest paths, and a provenance marker that bricks pip (#17331, #17332, #17246)

autobot-slm-backend/ansible/roles/_shared/tasks/stage_shared_manifests.yml

@@ -0,0 +1,64 @@
+---
+# Stage the repo-root manifests that a node's pip steps read (#17331, #17246).
+#
+# `code_source` is the controller's git checkout and NOTHING syncs it to a node
+# -- the invariant #17242 established the hard way, with an rc=127 on every
+# non-manager host. Two shapes still reach into it from a fleet node:
+#
+#   1. A `pip:` task whose `extra_args` named the checkout's own
+#      constraints/shared.txt (#17246, seven sites). Spelled out rather
+#      than quoted as a Jinja expression: every _shared/tasks/ file is
+#      scanned for a templated code_source_dir without a default, and that
+#      scan reads comments too (#16750).
+#   2. A manifest GENERATED on the controller by
+#      scripts/build-filtered-requirements.sh, whose rewritten `-c`/`-r` lines
+#      named the same controller-only path and were then copied to the node
+#      (#17331). pip resolves a nested `-c` against the directory of the file
+#      that contains it, so the path baked into the generated file is the path
+#      pip opens on the target.
+#
+# Both are fixed the same way, which is why this is one task file and not two:
+# put the files the node actually needs ON the node, at a path derived from the
+# SSOT base_dir, and point the consumers there. `playbooks/update-all-nodes.yml`
+# already did exactly this for its own three pip steps (#17243); this lifts that
+# into a shared task so a role carries its own precondition and does not depend
+# on which playbook happened to invoke it.
+#
+# `copy:` reads `src` on the CONTROLLER, which is where these files exist, so no
+# delegation is involved and none is wanted: the destination is the node.
+#
+# Idempotent and guarded by a per-host fact, so including it from several roles
+# in one run stages once. Including it is always safe.
+
+- name: "Shared manifests | Ensure the constraints directory exists on the node (#17331)"
+  ansible.builtin.file:
+    path: "{{ autobot.base_dir }}/constraints"
+    state: directory
+    owner: "{{ autobot.user }}"
+    mode: "0755"
+  become: true
+
+- name: "Shared manifests | Stage constraints/shared.txt onto the node (#17331)"
+  ansible.builtin.copy:
+    src: "{{ playbook_dir }}/../../../constraints/shared.txt"
+    dest: "{{ autobot.base_dir }}/constraints/shared.txt"
+    owner: "{{ autobot.user }}"
+    mode: "0644"
+  become: true
+
+# The backend manifest's `-r ../requirements.txt` pulls ~23 root runtime deps
+# (paramiko, asyncssh, pypdf, python-docx/pptx, openpyxl, ...). #11135 filed
+# that as a silent drop when the line was stripped instead of rewritten; the
+# rewrite is only honest if the file it names is present on the node too.
+# It declares no includes of its own, so staging it needs nothing further.
+- name: "Shared manifests | Stage the root requirements.txt onto the node (#17331)"
+  ansible.builtin.copy:
+    src: "{{ playbook_dir }}/../../../requirements.txt"
+    dest: "{{ autobot.base_dir }}/requirements.txt"
+    owner: "{{ autobot.user }}"
+    mode: "0644"
+  become: true
+
+- name: "Shared manifests | Record that this host is staged (#17331)"
+  ansible.builtin.set_fact:
+    autobot_shared_manifests_staged: true

autobot-slm-backend/ansible/roles/agent_config/tasks/openvino.yml

@@ -73,13 +73,27 @@
 # would pull Intel's NPU runtime into the backend's venv too, which nothing
 # in autobot-backend imports. See MULTI_SOURCE_VENVS in
 # repo_tests/ansible_manifest_resolution.py for the recorded reason.
+# #17246: `code_source` is the controller's git checkout and no task syncs it
+# to a node, so this constraints path did not exist here -- the #17242 class,
+# caught by the widened detector rather than by another failed provision. The
+# file is staged onto the node first and read from there.
+- name: "Stage the manifests this node's pip step reads (#17246, #17331)"
+  ansible.builtin.include_tasks: ../../_shared/tasks/stage_shared_manifests.yml
+  when: not (autobot_shared_manifests_staged | default(false))
+  # Tagged `always` for the reason update-all-nodes.yml gives at its own
+  # staging task: the staged file is a precondition of the pip step below,
+  # so a tag selection that runs the install but not the staging would
+  # constrain against a file that is not there. Enumerating the consumer
+  # tags would go stale; `always` cannot.
+  tags: [always]
+
 - name: Set up OpenVINO environment in venv
   pip:
     name: openvino
     virtualenv: "{{ venv_dir }}"
     # #16641: see roles/browser/tasks/main.yml for the full reason.
     virtualenv_command: "{{ python_interpreter_binary }} -m venv"
-    extra_args: "-c {{ code_source_dir | default(autobot.base_dir ~ '/code_source') }}/constraints/shared.txt"
+    extra_args: "-c {{ autobot.base_dir }}/constraints/shared.txt"
   become_user: "{{ service_user }}"
   environment:
     PIP_DEFAULT_TIMEOUT: "120"

autobot-slm-backend/ansible/roles/agent_config/tasks/playwright.yml

@@ -32,6 +32,20 @@
 # would pull a headless-browser automation stack into the backend's venv too,
 # which nothing in autobot-backend imports. See MULTI_SOURCE_VENVS in
 # repo_tests/ansible_manifest_resolution.py for the recorded reason.
+# #17246: `code_source` is the controller's git checkout and no task syncs it
+# to a node, so this constraints path did not exist here -- the #17242 class,
+# caught by the widened detector rather than by another failed provision. The
+# file is staged onto the node first and read from there.
+- name: "Stage the manifests this node's pip step reads (#17246, #17331)"
+  ansible.builtin.include_tasks: ../../_shared/tasks/stage_shared_manifests.yml
+  when: not (autobot_shared_manifests_staged | default(false))
+  # Tagged `always` for the reason update-all-nodes.yml gives at its own
+  # staging task: the staged file is a precondition of the pip step below,
+  # so a tag selection that runs the install but not the staging would
+  # constrain against a file that is not there. Enumerating the consumer
+  # tags would go stale; `always` cannot.
+  tags: [always]
+
 - name: Install Playwright via pip in venv
   pip:
     name: playwright
@@ -40,7 +54,7 @@
     virtualenv_command: "{{ python_interpreter_binary }} -m venv"
     # #15684: constraints/shared.txt applied here too, the same shape
     # openvino.yml's pip task in this role already uses.
-    extra_args: "-c {{ code_source_dir | default(autobot.base_dir ~ '/code_source') }}/constraints/shared.txt"
+    extra_args: "-c {{ autobot.base_dir }}/constraints/shared.txt"
   become_user: "{{ service_user }}"
   environment:
     PIP_DEFAULT_TIMEOUT: "120"

autobot-slm-backend/ansible/roles/agent_config/tasks/python_deps.yml

@@ -10,6 +10,20 @@
     path: "{{ project_root }}/{{ requirements_file }}"
   register: requirements_stat
 
+# #17246: `code_source` is the controller's git checkout and no task syncs it
+# to a node, so this constraints path did not exist here -- the #17242 class,
+# caught by the widened detector rather than by another failed provision. The
+# file is staged onto the node first and read from there.
+- name: "Stage the manifests this node's pip step reads (#17246, #17331)"
+  ansible.builtin.include_tasks: ../../_shared/tasks/stage_shared_manifests.yml
+  when: not (autobot_shared_manifests_staged | default(false))
+  # Tagged `always` for the reason update-all-nodes.yml gives at its own
+  # staging task: the staged file is a precondition of the pip step below,
+  # so a tag selection that runs the install but not the staging would
+  # constrain against a file that is not there. Enumerating the consumer
+  # tags would go stale; `always` cannot.
+  tags: [always]
+
 - name: Install Python packages from requirements.txt
   pip:
     requirements: "{{ project_root }}/{{ requirements_file }}"
@@ -19,7 +33,7 @@
     # #15684: constraints/shared.txt applied here too, the same shape every
     # other pip site in this venv now uses -- see openvino.yml, already
     # ahead of this one.
-    extra_args: "-c {{ code_source_dir | default(autobot.base_dir ~ '/code_source') }}/constraints/shared.txt"
+    extra_args: "-c {{ autobot.base_dir }}/constraints/shared.txt"
   become_user: "{{ service_user }}"
   environment:
     PIP_DEFAULT_TIMEOUT: "120"

autobot-slm-backend/ansible/roles/ai-stack/tasks/main.yml

@@ -201,12 +201,26 @@
   # the SAME host share a controller file, and the later generation can
   # replace it before the earlier copy reads it.
 
-- name: Create filtered AI-stack requirements (on the controller, #14272, #17242)
+# #17331: the generated file is installed ON THE NODE, so the rewritten `-c`
+# must name a path that exists THERE. It used to name code_source, which only
+# the controller has -- the same defect #17242 fixed for the execution side and
+# left on the payload side. Confirmed failing in the backend role on a
+# non-manager host; unexercised here only because no ai_stack node has been
+# provisioned since, which makes it a prediction rather than an observation.
+- name: "AI Stack | Stage the manifests this node's pip step reads (#17331)"
+  ansible.builtin.include_tasks: ../../_shared/tasks/stage_shared_manifests.yml
+  when:
+    - _ai_requirements_file.stat.exists
+    - not (autobot_shared_manifests_staged | default(false))
+  tags: ['ai', 'deploy']
+
+- name: Create filtered AI-stack requirements (on the controller, #14272, #17242, #17331)
   ansible.builtin.shell:
     cmd: >-
       bash {{ code_source_dir | default(autobot.base_dir ~ '/code_source') }}/scripts/build-filtered-requirements.sh
       {{ code_source_dir | default(autobot.base_dir ~ '/code_source') }}/{{ ai_stack_requirements_rel }}
       {{ code_source_dir | default(autobot.base_dir ~ '/code_source') }}
+      {{ autobot.base_dir }}
       > /tmp/filtered-requirements-ai.{{ inventory_hostname }}.{{ _req_run_id }}.txt
   changed_when: false
   delegate_to: localhost

autobot-slm-backend/ansible/roles/backend/tasks/main.yml

@@ -943,15 +943,23 @@
       - restart backend
       - restart celery
 
-  # The filtered file is written to /tmp, so its sibling-relative `-c
-  # ../constraints/shared.txt` (#10524) and `-r ../requirements.txt` (#11135)
-  # includes would resolve relative to /tmp and fail (pip errors on a missing
-  # file). Delegate the filter+rewrite to the canonical
-  # scripts/build-filtered-requirements.sh (#11134), which rewrites both to the
-  # code_source path, which always exists as the git checkout, so the pinned
-  # constraints AND root runtime deps are actually applied on deploy (#11117,
-  # #11135). services/role_registry.py's backend post_sync_cmd invokes the same
-  # script, so both deploy paths share one implementation instead of drifting.
+  # The manifest's sibling-relative `-c ../constraints/shared.txt` (#10524) and
+  # `-r ../requirements.txt` (#11135) includes do not survive being moved, so
+  # the filter+rewrite is delegated to the canonical
+  # scripts/build-filtered-requirements.sh (#11134), which rewrites both to an
+  # absolute path so the pinned constraints AND root runtime deps are actually
+  # applied on deploy (#11117, #11135). services/role_registry.py's backend
+  # post_sync_cmd invokes the same script, so both deploy paths share one
+  # implementation instead of drifting.
+  #
+  # #17331: that absolute path is the NODE's staged copy, not `code_source`.
+  # The comment here used to assert that code_source "always exists as the git
+  # checkout" -- #17242 disproved the execution half of that claim and this is
+  # the other half: the path the rewrite bakes INTO the generated file is read
+  # by pip on the target, where code_source is absent, and provisioning aborted
+  # with "Could not open constraint file". The staging task below puts the two
+  # files on the node first; the third argument to the script names where they
+  # landed. See roles/_shared/tasks/stage_shared_manifests.yml.
   # #17242: generated ON THE CONTROLLER, not on the target. `code_source` is the
   # git checkout and no task syncs it to a remote node, so running this script
   # from it on the target exits 127 on every host that is not the manager --
@@ -971,12 +979,17 @@
     # the SAME host share a controller file, and the later generation can
     # replace it before the earlier copy reads it.
 
-  - name: Create filtered backend requirements (on the controller, #17242)
+  - name: "Backend | Stage the manifests this node's pip step reads (#17331)"
+    ansible.builtin.include_tasks: ../../_shared/tasks/stage_shared_manifests.yml
+    when: not (autobot_shared_manifests_staged | default(false))
+
+  - name: Create filtered backend requirements (on the controller, #17242, #17331)
     ansible.builtin.shell:
       cmd: >-
         bash {{ code_source_dir }}/scripts/build-filtered-requirements.sh
         {{ code_source_dir }}/autobot-backend/requirements.txt
         {{ code_source_dir }}
+        {{ autobot.base_dir }}
         > /tmp/filtered-requirements.{{ inventory_hostname }}.{{ _req_run_id }}.txt
     changed_when: false
     delegate_to: localhost
@@ -1059,6 +1072,7 @@
         bash {{ code_source_dir }}/scripts/build-filtered-requirements.sh
         {{ code_source_dir }}/requirements-gpu-torch.txt
         {{ code_source_dir }}
+        {{ autobot.base_dir }}
         > /tmp/filtered-requirements-gpu-torch.{{ inventory_hostname }}.{{ _req_run_id }}.txt
     changed_when: false
     delegate_to: localhost
@@ -1080,6 +1094,7 @@
         bash {{ code_source_dir }}/scripts/build-filtered-requirements.sh
         {{ code_source_dir }}/requirements-gpu-faiss.txt
         {{ code_source_dir }}
+        {{ autobot.base_dir }}
         > /tmp/filtered-requirements-gpu-faiss.{{ inventory_hostname }}.{{ _req_run_id }}.txt
     changed_when: false
     delegate_to: localhost
@@ -1101,6 +1116,7 @@
         bash {{ code_source_dir }}/scripts/build-filtered-requirements.sh
         {{ code_source_dir }}/requirements-gpu.txt
         {{ code_source_dir }}
+        {{ autobot.base_dir }}
         > /tmp/filtered-requirements-gpu.{{ inventory_hostname }}.{{ _req_run_id }}.txt
     changed_when: false
     delegate_to: localhost

autobot-slm-backend/ansible/roles/backend_services/tasks/main.yml

@@ -50,14 +50,28 @@
 # itself, which these hosts do not have. `virtualenv_command:` below replaces
 # it with the stdlib's `venv` module, using the interpreter
 # python_interpreter (included above) already installed.
+# #17246: `code_source` is the controller's git checkout and no task syncs it
+# to a node, so this constraints path did not exist here -- the #17242 class,
+# caught by the widened detector rather than by another failed provision. The
+# file is staged onto the node first and read from there.
+- name: "Stage the manifests this node's pip step reads (#17246, #17331)"
+  ansible.builtin.include_tasks: ../../_shared/tasks/stage_shared_manifests.yml
+  when: not (autobot_shared_manifests_staged | default(false))
+  # Tagged `always` for the reason update-all-nodes.yml gives at its own
+  # staging task: the staged file is a precondition of the pip step below,
+  # so a tag selection that runs the install but not the staging would
+  # constrain against a file that is not there. Enumerating the consumer
+  # tags would go stale; `always` cannot.
+  tags: [always]
+
 - name: Install Python dependencies
   pip:
     requirements: /opt/autobot/app/requirements.txt
     virtualenv: /opt/autobot/venv
     virtualenv_command: "{{ python_interpreter_binary }} -m venv"
     # #15684: constraints/shared.txt applied here too, the same shape every
     # other pip site in this venv now uses.
-    extra_args: "-c {{ code_source_dir | default(autobot.base_dir ~ '/code_source') }}/constraints/shared.txt"
+    extra_args: "-c {{ autobot.base_dir }}/constraints/shared.txt"
   become_user: autobot
   environment:
     PIP_DEFAULT_TIMEOUT: "120"

autobot-slm-backend/ansible/roles/browser/tasks/main.yml

@@ -169,14 +169,28 @@
 # against the CWD, not the file it is written in. `chdir` anchors that to the
 # manifest's own directory -- the deployed component root the sync task above
 # already refreshed, so no worker code sync order changes.
+# #17246: `code_source` is the controller's git checkout and no task syncs it
+# to a node, so this constraints path did not exist here -- the #17242 class,
+# caught by the widened detector rather than by another failed provision. The
+# file is staged onto the node first and read from there.
+- name: "Stage the manifests this node's pip step reads (#17246, #17331)"
+  ansible.builtin.include_tasks: ../../_shared/tasks/stage_shared_manifests.yml
+  when: not (autobot_shared_manifests_staged | default(false))
+  # Tagged `always` for the reason update-all-nodes.yml gives at its own
+  # staging task: the staged file is a precondition of the pip step below,
+  # so a tag selection that runs the install but not the staging would
+  # constrain against a file that is not there. Enumerating the consumer
+  # tags would go stale; `always` cannot.
+  tags: [always]
+
 - name: "Browser | Install Playwright and FastAPI dependencies via pip"
   pip:
     requirements: "{{ browser_install_dir }}/requirements.txt"
     state: present
     virtualenv: "{{ browser_install_dir }}/venv"
     virtualenv_command: "{{ python_interpreter_binary }} -m venv"
     chdir: "{{ browser_install_dir }}"
-    extra_args: "-c {{ code_source_dir | default(autobot.base_dir ~ '/code_source') }}/constraints/shared.txt"
+    extra_args: "-c {{ autobot.base_dir }}/constraints/shared.txt"
   become: true
   environment:
     PIP_DEFAULT_TIMEOUT: "120"

autobot-slm-backend/ansible/roles/tts-worker/tasks/main.yml

@@ -243,6 +243,20 @@
   timeout: 600  # Issue #2835: torch downloads are large, prevent indefinite hang
   tags: ['tts', 'packages']
 
+# #17246: `code_source` is the controller's git checkout and no task syncs it
+# to a node, so this constraints path did not exist here -- the #17242 class,
+# caught by the widened detector rather than by another failed provision. The
+# file is staged onto the node first and read from there.
+- name: "Stage the manifests this node's pip step reads (#17246, #17331)"
+  ansible.builtin.include_tasks: ../../_shared/tasks/stage_shared_manifests.yml
+  when: not (autobot_shared_manifests_staged | default(false))
+  # Tagged `always` for the reason update-all-nodes.yml gives at its own
+  # staging task: the staged file is a precondition of the pip step below,
+  # so a tag selection that runs the install but not the staging would
+  # constrain against a file that is not there. Enumerating the consumer
+  # tags would go stale; `always` cannot.
+  tags: [always]
+
 - name: "TTS Worker | Install pocket-tts and audio dependencies"
   ansible.builtin.pip:
     name:
@@ -281,7 +295,7 @@
     state: present
     extra_args: >-
       --timeout 300
-      -c {{ code_source_dir | default(autobot.base_dir ~ '/code_source') }}/constraints/shared.txt
+      -c {{ autobot.base_dir }}/constraints/shared.txt
   become_user: "{{ tts_user }}"
   environment:
     PIP_DEFAULT_TIMEOUT: "300"

autobot-slm-backend/api/venv_reconcile.py

@@ -496,6 +496,13 @@ async def install_pip_deps_for_component(component: str, steps: List[str]) -> bo
 
 
 async def _run_pip_install(component: str, req_path: str, pip_bin: str, steps: List[str]) -> bool:
+    # #17332: a dist-info husk left by this tool's own previously-unrecorded
+    # marker makes pip abort with `uninstall-no-record-file` the moment it has
+    # to upgrade that package -- so the repair has to happen BEFORE the install,
+    # not after a failure. pip stops at the first husk, so this clears the whole
+    # venv in one pass rather than one failed deploy per husked package.
+    for site_packages in provenance.site_packages_dirs(Path(pip_bin).parents[1]):
+        provenance.clear_provenance_husks(site_packages, steps)
     steps.append(f"pip: installing {req_path} into {Path(pip_bin).parent}")
     try:
         proc = await asyncio.create_subprocess_exec(