fix(deploy): provisioning aborts on every non-manager host, and the failure summary hid it (#17240, #17242, #17243, #17244)

.coderabbit.yaml

@@ -25,6 +25,25 @@ reviews:
   review_status: true
   collapse_walkthrough: true
 
+  # Signal, not volume. Every finding on #17241 was valid -- including three that
+  # were real bugs in new code and one that exposed a detector blind spot -- so
+  # `profile: assertive` above stays. What is switched off here is the wrapper
+  # around each finding, not the finding.
+  #
+  # The largest single block per comment: a restatement of the finding addressed
+  # to an AI agent, duplicating the prose immediately above it.
+  enable_prompt_for_ai_agents: false
+  # Walkthrough chrome. None of it is read here: the PR body states the intent,
+  # and labels and reviewers are decided by the repo's own rules, not suggested.
+  sequence_diagrams: false
+  estimate_code_review_effort: false
+  suggested_labels: false
+  suggested_reviewers: false
+  # Cross-references are guessed from text and have been wrong often enough that
+  # checking each one costs more than finding the real link by hand.
+  related_issues: false
+  related_prs: false
+
   auto_review:
     enabled: true
     drafts: false

autobot-slm-backend/ansible/playbooks/update-all-nodes.yml

@@ -801,6 +801,36 @@
       tags: [always]
 
   tasks:
+    # #17243: the pip steps below constrain against constraints/shared.txt, but
+    # they used to reference it inside `code_source` -- the git checkout, which
+    # exists only on the manager. On an infrastructure node that path is absent
+    # and pip aborts with "Could not open constraint file", the exact failure
+    # scripts/build-filtered-requirements.sh warns about in its own header.
+    # Stage the file onto the node and constrain against the local copy.
+    # Tagged `always` deliberately. The staged file is a precondition for the
+    # backend, npu/npu-worker, browser/browser-worker and deps task groups, so
+    # a tag selection like `--tags backend` would otherwise skip the staging
+    # and leave the pip step constraining against a file that is not there.
+    # Enumerating the union of consumer tags would go stale the moment another
+    # tagged consumer is added; `always` cannot, and this play already uses it.
+    - name: "[PLAY 2] Ensure the constraints directory exists on the node (#17243)"
+      ansible.builtin.file:
+        path: "{{ autobot.base_dir }}/constraints"
+        state: directory
+        owner: autobot
+        mode: "0755"
+      become: true
+      tags: [always]
+
+    - name: "[PLAY 2] Stage constraints/shared.txt onto the node (#17243)"
+      ansible.builtin.copy:
+        src: "{{ playbook_dir }}/../../../constraints/shared.txt"
+        dest: "{{ autobot.base_dir }}/constraints/shared.txt"
+        owner: autobot
+        mode: "0644"
+      become: true
+      tags: [always]
+
     - name: "[PLAY 2] Starting Node Update"
       debug:
         msg:
@@ -938,7 +968,7 @@
       command:
         cmd: /opt/autobot/autobot-backend/venv/bin/pip install
              -r /opt/autobot/autobot-backend/requirements.txt
-             -c {{ code_source_dir | default(autobot.base_dir ~ '/code_source') }}/constraints/shared.txt
+             -c {{ autobot.base_dir }}/constraints/shared.txt
              --quiet --timeout 120
         # #15733: pip resolves a relative editable in a requirements file
         # against the CWD, not against the file. This manifest opens with
@@ -1452,7 +1482,7 @@
       command:
         cmd: /opt/autobot/autobot-npu-worker/venv/bin/pip install
              -r /opt/autobot/autobot-npu-worker/requirements.txt
-             -c {{ code_source_dir | default(autobot.base_dir ~ '/code_source') }}/constraints/shared.txt
+             -c {{ autobot.base_dir }}/constraints/shared.txt
              --quiet --timeout 120
         # #15733: pip resolves a relative editable in a requirements file
         # against the CWD, not against the file. This manifest opens with
@@ -1598,7 +1628,7 @@
       command:
         cmd: /opt/autobot/autobot-browser-worker/venv/bin/pip install
              -r /opt/autobot/autobot-browser-worker/requirements.txt
-             -c {{ code_source_dir | default(autobot.base_dir ~ '/code_source') }}/constraints/shared.txt
+             -c {{ autobot.base_dir }}/constraints/shared.txt
              --quiet --timeout 120
         # #15733: pip resolves a relative editable in a requirements file
         # against the CWD, not against the file. This manifest opens with

autobot-slm-backend/ansible/roles/_shared/tasks/record_slm_deployed_commit.yml

@@ -38,6 +38,13 @@
       rev-parse HEAD
   register: _deployed_commit_git
   changed_when: false
+  # #17243: read code_source on the CONTROLLER. It is the git checkout and no
+  # task syncs it to a remote node, so this git call fails on any non-manager
+  # caller that does not pass deployed_commit_value in. sync_deletions.yml
+  # delegates the identical `rev-parse HEAD` for the same reason -- two shared
+  # tasks doing one operation should not disagree about where it runs.
+  delegate_to: localhost
+  become: false
   when: deployed_commit_value | default('') | length == 0
   tags: ['slm', 'deploy']
 

autobot-slm-backend/ansible/roles/ai-stack/defaults/main.yml

@@ -95,3 +95,14 @@ ai_system_commands_model: "dolphin-llama3:8b"
 # a database and that is the only role that installs it. On an ai-stack-only
 # host the ai-stack role installs chromadb itself, so owning it there is correct.
 chromadb_service_owner: "ai-stack"
+
+# #14809 guards that tasks/main.yml names this manifest exactly ONCE -- a second
+# reference used to be a second, too-late delivery -- while #14272 reads the copy
+# task's src BASENAME, so that one literal has to live in the copy. The filter
+# step added by #17242 therefore composes the path from this relative tail.
+#
+# Deliberately NOT a Jinja template: ansible_backend_path_defects_15560 renders
+# every templated default in this file against the other defaults only, so a
+# reference to autobot.base_dir (an inventory group_var, not a role default)
+# raises UndefinedError there. A plain string composes fine at the call site.
+ai_stack_requirements_rel: "autobot-infrastructure/shared/docker/ai-stack/requirements-ai.txt"

autobot-slm-backend/ansible/roles/ai-stack/tasks/main.yml

@@ -166,7 +166,7 @@
 # exists to unbreak.
 - name: "AI Stack | Deploy requirements-ai.txt before installing from it (#14809)"
   ansible.builtin.copy:
-    src: "{{ playbook_dir }}/../../../autobot-infrastructure/shared/docker/ai-stack/requirements-ai.txt"
+    src: "{{ code_source_dir | default(autobot.base_dir ~ '/code_source') }}/autobot-infrastructure/shared/docker/ai-stack/requirements-ai.txt"
     dest: "{{ ai_install_dir }}/requirements-ai.txt"
     owner: "{{ ai_user }}"
     group: "{{ ai_group }}"
@@ -188,21 +188,45 @@
 # than with a second sed here: the backend role and services/role_registry.py
 # already share that script for exactly this, and it says so in its own header
 # (#11117). It was never wired in here, which is the whole bug.
-- name: Create filtered AI-stack requirements file (#14272)
+# #17242: generated ON THE CONTROLLER. `code_source` is the git checkout and
+# nothing syncs it to a remote node, so running this script out of it on an
+# ai_stack host exits 127 -- the same defect #17242 hit in the backend role, in
+# a role that never runs on the manager at all. The source manifest is read from
+# the same controller path the deploy task above uses.
+- name: "AI Stack | Unique id for this run's controller temp files (#17242)"
+  ansible.builtin.set_fact:
+    _req_run_id: "{{ 999999999 | random }}"
+  # Set once per host per run, so the generate step and the copy that reads
+  # its output agree. inventory_hostname alone was not enough: two runs for
+  # the SAME host share a controller file, and the later generation can
+  # replace it before the earlier copy reads it.
+
+- name: Create filtered AI-stack requirements (on the controller, #14272, #17242)
   ansible.builtin.shell:
     cmd: >-
       bash {{ code_source_dir | default(autobot.base_dir ~ '/code_source') }}/scripts/build-filtered-requirements.sh
-      {{ ai_install_dir }}/requirements-ai.txt
+      {{ code_source_dir | default(autobot.base_dir ~ '/code_source') }}/{{ ai_stack_requirements_rel }}
       {{ code_source_dir | default(autobot.base_dir ~ '/code_source') }}
-      > {{ ai_install_dir }}/requirements-ai-filtered.txt
-  become_user: "{{ ai_user }}"
-  changed_when: true
+      > /tmp/filtered-requirements-ai.{{ inventory_hostname }}.{{ _req_run_id }}.txt
+  changed_when: false
+  delegate_to: localhost
+  become: false
+  when: _ai_requirements_file.stat.exists
+  tags: ['ai', 'deploy']
+
+- name: Write filtered AI-stack requirements to the target (#17242)
+  ansible.builtin.copy:
+    src: "/tmp/filtered-requirements-ai.{{ inventory_hostname }}.{{ _req_run_id }}.txt"
+    dest: "{{ ai_install_dir }}/filtered-requirements-ai.txt"
+    owner: "{{ ai_user }}"
+    mode: "0644"
+  become: true
   when: _ai_requirements_file.stat.exists
   tags: ['ai', 'deploy']
 
 - name: Install AI Python packages from requirements-ai.txt
   ansible.builtin.pip:
-    requirements: "{{ ai_install_dir }}/requirements-ai-filtered.txt"
+    requirements: "{{ ai_install_dir }}/filtered-requirements-ai.txt"
     virtualenv: "{{ ai_install_dir }}/venv"
     # #16641: see "Upgrade pip in venv" above.
     virtualenv_command: "python3.14 -m venv"

autobot-slm-backend/ansible/roles/backend/defaults/main.yml

@@ -138,7 +138,10 @@ backend_browser_host: ""    # Set to browser VM IP in fleet deployments; empty =
 backend_browser_port: 9001  # Port 9001 is canonical; 3000 was Grafana's port (#5608)
 
 # AI Stack connection (co-located: use 127.0.0.1; WSL2: may need 10.255.255.254)
-backend_ai_stack_host: "127.0.0.1"
+# The shipped default, named once so the WSL2 override below can ask "is this still
+# the default?" without restating the address (#17244).
+backend_ai_stack_host_shipped_default: "127.0.0.1"
+backend_ai_stack_host: "{{ backend_ai_stack_host_shipped_default }}"
 backend_ai_stack_port: 8080
 
 # ChromaDB connection (co-located: ai-stack node; fleet deployment: ai-stack node IP)
@@ -183,3 +186,11 @@ backend_audit_log_file: /opt/autobot/logs/audit.log
 #   backend_has_gpu: "auto" runs nvidia-smi at deploy time; "true"/"false" override.
 backend_vllm_enabled: false
 backend_has_gpu: "auto"
+
+# WSL2 routing (#3503, #17244). Defined here rather than inline in tasks so a
+# non-default WSL2 network can be overridden from inventory, and so the two
+# addresses are named rather than appearing as bare literals in an expression.
+# `..._loopback_host` is used when the AI stack is co-located on this node;
+# `..._gateway_host` reaches the Windows side when it is not.
+backend_wsl2_loopback_host: "127.0.0.1"
+backend_wsl2_gateway_host: "10.255.255.254"

autobot-slm-backend/ansible/roles/backend/tasks/main.yml

@@ -48,13 +48,20 @@
   failed_when: false
   tags: ['backend', 'config']
 
-- name: "Backend | Set backend_ai_stack_host=10.255.255.254 on WSL2 hosts (#3503)"
+- name: "Backend | Set backend_ai_stack_host on WSL2 hosts (#3503, #17244)"
   ansible.builtin.set_fact:
-    backend_ai_stack_host: "10.255.255.254"
+    # #17244: ask the locality question too, not just "is this WSL2".
+    # The WSL2 gateway address reaches the Windows side; it is the right
+    # answer only when the AI stack is NOT on this node. When the stack is
+    # co-located, loopback is. This mirrors the backend_chromadb_host task
+    # below, which has always asked role_ai_stack_active -- the two sat
+    # twelve lines apart and disagreed. Behaviour is unchanged whenever the
+    # stack is not on this node, which is the case #3503 fixed.
+    backend_ai_stack_host: "{{ backend_wsl2_loopback_host if (role_ai_stack_active | default(false) | bool) else backend_wsl2_gateway_host }}"
   when:
     - _proc_version.content is defined
     - (_proc_version.content | b64decode | lower) is search('microsoft')
-    - backend_ai_stack_host == '127.0.0.1'
+    - backend_ai_stack_host == backend_ai_stack_host_shipped_default
   tags: ['backend', 'config']
 
 - name: "Backend | Set backend_chromadb_host on WSL2 hosts (#3541, #4084, #4087)"
@@ -945,21 +952,53 @@
   # constraints AND root runtime deps are actually applied on deploy (#11117,
   # #11135). services/role_registry.py's backend post_sync_cmd invokes the same
   # script, so both deploy paths share one implementation instead of drifting.
-  - name: Create filtered backend requirements file
+  # #17242: generated ON THE CONTROLLER, not on the target. `code_source` is the
+  # git checkout and no task syncs it to a remote node, so running this script
+  # from it on the target exits 127 on every host that is not the manager --
+  # which is why this passed for as long as the manager was the only backend
+  # host. Same reason `_shared/tasks/sync_deletions.yml` delegates its own
+  # code_source reads to localhost.
+  #
+  # The output is carried in a registered variable rather than through a file,
+  # which also removes the fixed `/tmp/requirements-filtered.txt`: a shared name
+  # in a world-writable directory that every host in the run would have
+  # collided on the moment this task moved to the controller.
+  - name: "Backend | Unique id for this run's controller temp files (#17242)"
+    ansible.builtin.set_fact:
+      _req_run_id: "{{ 999999999 | random }}"
+    # Set once per host per run, so the generate step and the copy that reads
+    # its output agree. inventory_hostname alone was not enough: two runs for
+    # the SAME host share a controller file, and the later generation can
+    # replace it before the earlier copy reads it.
+
+  - name: Create filtered backend requirements (on the controller, #17242)
     ansible.builtin.shell:
       cmd: >-
         bash {{ code_source_dir }}/scripts/build-filtered-requirements.sh
-        {{ backend_code_dir }}/requirements.txt
+        {{ code_source_dir }}/autobot-backend/requirements.txt
         {{ code_source_dir }}
-        > /tmp/requirements-filtered.txt
-    become_user: "{{ backend_user }}"
+        > /tmp/filtered-requirements.{{ inventory_hostname }}.{{ _req_run_id }}.txt
+    changed_when: false
+    delegate_to: localhost
+    become: false
+
+  - name: Write filtered backend requirements to the target (#17242)
+    ansible.builtin.copy:
+      src: "/tmp/filtered-requirements.{{ inventory_hostname }}.{{ _req_run_id }}.txt"
+      dest: "{{ backend_code_dir }}/filtered-requirements.txt"
+      owner: "{{ backend_user }}"
+      mode: "0644"
+    become: true
 
   # (#2872) Removed failed_when: false — pip install failures must surface.
   # The Ansible pip module raises on any pip error; the old bypass (Issue #892)
   # masked real dependency failures during deployment.
   - name: Install filtered backend requirements
+    # #17242: reads the copy written above, inside the backend tree. The old
+    # path was a fixed name under /tmp, shared by every host and pre-creatable
+    # by any local user.
     ansible.builtin.pip:
-      requirements: "/tmp/requirements-filtered.txt"
+      requirements: "{{ backend_code_dir }}/filtered-requirements.txt"
       virtualenv: "{{ backend_code_dir }}/venv"
       # #16641: see "Upgrade pip" above.
       virtualenv_command: "python3.14 -m venv"
@@ -997,9 +1036,89 @@
   # reporting the downgrade. Uses the repo-root requirements-gpu-torch.txt from
   # code_source (sibling of the autobot-backend subtree; not part of the
   # backend code rsync).
+  # #17243: these three manifests sit at the REPO ROOT inside code_source. The
+  # backend sync copies only code_source/autobot-backend/, so on a non-manager
+  # backend the root-level files are absent and pip aborts -- the same defect as
+  # #17242, one directory up, and missed by the first sweep because these are
+  # `pip:` tasks rather than shell ones. Stage them from the controller.
+  # These three manifests sit at the REPO ROOT inside code_source, which is the
+  # controller's checkout; the backend sync copies only autobot-backend/, so the
+  # root-level files are absent on a non-manager backend and pip aborts (#17243).
+  #
+  # Filtered through the canonical script rather than copied raw: a file copied
+  # out of the repo lands at a different depth, so any relative -c/-r include it
+  # gains later would resolve somewhere else (#14272). None of the three carries
+  # one today, which makes this cheap now and correct if that changes.
+  #
+  # One task per manifest, not a loop: a loop renders the destination as a
+  # template, and the manifest-resolution guard cannot tie that back to the
+  # manifest it carries.
+  - name: "Backend | Filter requirements-gpu-torch.txt on the controller (#17243)"
+    ansible.builtin.shell:
+      cmd: >-
+        bash {{ code_source_dir }}/scripts/build-filtered-requirements.sh
+        {{ code_source_dir }}/requirements-gpu-torch.txt
+        {{ code_source_dir }}
+        > /tmp/filtered-requirements-gpu-torch.{{ inventory_hostname }}.{{ _req_run_id }}.txt
+    changed_when: false
+    delegate_to: localhost
+    become: false
+    when: backend_gpu_available | default(false) | bool
+
+  - name: "Backend | Stage filtered requirements-gpu-torch.txt onto the target (#17243)"
+    ansible.builtin.copy:
+      src: "/tmp/filtered-requirements-gpu-torch.{{ inventory_hostname }}.{{ _req_run_id }}.txt"
+      dest: "{{ backend_code_dir }}/filtered-requirements-gpu-torch.txt"
+      owner: "{{ backend_user }}"
+      mode: "0644"
+    become: true
+    when: backend_gpu_available | default(false) | bool
+
+  - name: "Backend | Filter requirements-gpu-faiss.txt on the controller (#17243)"
+    ansible.builtin.shell:
+      cmd: >-
+        bash {{ code_source_dir }}/scripts/build-filtered-requirements.sh
+        {{ code_source_dir }}/requirements-gpu-faiss.txt
+        {{ code_source_dir }}
+        > /tmp/filtered-requirements-gpu-faiss.{{ inventory_hostname }}.{{ _req_run_id }}.txt
+    changed_when: false
+    delegate_to: localhost
+    become: false
+    when: backend_gpu_available | default(false) | bool
+
+  - name: "Backend | Stage filtered requirements-gpu-faiss.txt onto the target (#17243)"
+    ansible.builtin.copy:
+      src: "/tmp/filtered-requirements-gpu-faiss.{{ inventory_hostname }}.{{ _req_run_id }}.txt"
+      dest: "{{ backend_code_dir }}/filtered-requirements-gpu-faiss.txt"
+      owner: "{{ backend_user }}"
+      mode: "0644"
+    become: true
+    when: backend_gpu_available | default(false) | bool
+
+  - name: "Backend | Filter requirements-gpu.txt on the controller (#17243)"
+    ansible.builtin.shell:
+      cmd: >-
+        bash {{ code_source_dir }}/scripts/build-filtered-requirements.sh
+        {{ code_source_dir }}/requirements-gpu.txt
+        {{ code_source_dir }}
+        > /tmp/filtered-requirements-gpu.{{ inventory_hostname }}.{{ _req_run_id }}.txt
+    changed_when: false
+    delegate_to: localhost
+    become: false
+    when: backend_gpu_available | default(false) | bool
+
+  - name: "Backend | Stage filtered requirements-gpu.txt onto the target (#17243)"
+    ansible.builtin.copy:
+      src: "/tmp/filtered-requirements-gpu.{{ inventory_hostname }}.{{ _req_run_id }}.txt"
+      dest: "{{ backend_code_dir }}/filtered-requirements-gpu.txt"
+      owner: "{{ backend_user }}"
+      mode: "0644"
+    become: true
+    when: backend_gpu_available | default(false) | bool
+
   - name: "Backend | Install CUDA torch requirements (requirements-gpu-torch.txt) (#15162)"
     ansible.builtin.pip:
-      requirements: "{{ code_source_dir }}/requirements-gpu-torch.txt"
+      requirements: "{{ backend_code_dir }}/filtered-requirements-gpu-torch.txt"
       virtualenv: "{{ backend_code_dir }}/venv"
       # #16641: see "Upgrade pip" above.
       virtualenv_command: "python3.14 -m venv"
@@ -1032,7 +1151,7 @@
 
   - name: "Backend | Install GPU faiss requirements (requirements-gpu-faiss.txt) (#15163)"
     ansible.builtin.pip:
-      requirements: "{{ code_source_dir }}/requirements-gpu-faiss.txt"
+      requirements: "{{ backend_code_dir }}/filtered-requirements-gpu-faiss.txt"
       virtualenv: "{{ backend_code_dir }}/venv"
       # #16641: see "Upgrade pip" above.
       virtualenv_command: "python3.14 -m venv"
@@ -1048,7 +1167,7 @@
   # duplicate CUDA-torch download happens here.
   - name: "Backend | Install GPU/vLLM requirements (requirements-gpu.txt) (#10288)"
     ansible.builtin.pip:
-      requirements: "{{ code_source_dir }}/requirements-gpu.txt"
+      requirements: "{{ backend_code_dir }}/filtered-requirements-gpu.txt"
       virtualenv: "{{ backend_code_dir }}/venv"
       # #16641: see "Upgrade pip" above.
       virtualenv_command: "python3.14 -m venv"