feat(lambda): stamp the deployed version, and gate that every deploy does (#654)

.github/workflows/ci.yml

@@ -50,6 +50,19 @@ jobs:
       - name: Lambda runtime census
         run: make lambda-runtimes
 
+      # Lambda version census (spawn#654). Same two-half shape; offline only here.
+      # Every deploy mechanism must stamp spawn:version, so a NEW Lambda cannot
+      # arrive unreadable — which is the structural half of the problem. Nothing
+      # synchronises the CLI with the control plane, so skew is the normal state;
+      # until this it was invisible, answerable only by comparing a function's
+      # LastModified against `git log -- lambda/<name>/`.
+      #
+      # The deployed half (`make lambda-versions-deployed`) needs the infra
+      # profile and is run by hand. On its first run it found six functions last
+      # deployed in February and three nobody would have thought to list.
+      - name: Lambda version census
+        run: make lambda-versions
+
       # Changelog gate. CLAUDE.md requires an entry under `## [Unreleased]` in the
       # SAME PR as any user-facing change, and the release ritual promotes that
       # section verbatim. Nothing enforced it, and #627 merged with no entry at all —

CHANGELOG.md

@@ -7,6 +7,28 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+
+- **Every spore.host-operated Lambda now stamps the version it was deployed
+  from, and CI gates that it does** (#654). Nothing synchronises the CLI with the
+  control plane — `spawn` upgrades when a user upgrades it, a Lambda only when
+  someone runs its deploy — so version skew is the *normal* state. That is
+  defensible, since the two are deployed independently; what was not is that it
+  was **invisible**. `spawn-ttl-reaper-production` sat untouched from 2026-07-31
+  to 2026-10-04 while the CLI went from ~v0.9x to v0.116.0, missing three merged
+  fixes, with nothing anywhere saying so.
+  All six deploy mechanisms now set a `spawn:version` tag readable in one API
+  call without invoking the function, matching what `spawn reaper deploy` already
+  did for the self-hosted reaper: three SAM templates take a `Version` parameter,
+  and three shell deploys call `aws lambda tag-resource`. The value comes from
+  `git describe --tags --always --dirty`, so a deploy from an uncommitted tree
+  stamps `-dirty` and says so rather than claiming a release.
+  `make lambda-versions` gates PRs offline — a new Lambda cannot arrive
+  unreadable — and `make lambda-versions-deployed` reports live versions for an
+  operator. Replaces comparing a function's `LastModified` against
+  `git log -- lambda/<name>/`, which is archaeology and is wrong whenever a
+  redeploy carried no source change, or a source change was never deployed.
+
 ## [0.126.1] - 2026-10-09
 
 ### Security

Makefile

@@ -267,5 +267,17 @@ smoke-needed:
 lambda-runtimes:
 	@scripts/lambda-runtime-census.sh
 
+# Version census (#654). The offline half gates PRs: every deploy mechanism must
+# stamp spawn:version, so a new Lambda cannot arrive unreadable. The --deployed
+# half is operator-facing and needs the infra profile — these functions are in
+# spore.host's account, so a user's CLI cannot see them (the user-facing half of
+# #654 is `spawn reaper status`, for the self-hosted reaper in their own account).
+.PHONY: lambda-versions lambda-versions-deployed
+lambda-versions:
+	@scripts/lambda-version-census.sh
+
+lambda-versions-deployed:
+	@scripts/lambda-version-census.sh --deployed
+
 lambda-runtimes-deployed:
 	@scripts/lambda-runtime-census.sh --deployed

lambda/autoscale-orchestrator/Makefile

@@ -1,5 +1,12 @@
 .PHONY: build deploy clean test
 
+# The version stamped onto the function as spawn:version (#654). `git describe`
+# rather than a bare tag: it yields 0.126.1 on a tag, 0.126.1-3-gabc when ahead,
+# and a -dirty suffix for uncommitted changes — so the stamp says "deployed from
+# uncommitted code" out loud instead of claiming a release. Leading v stripped to
+# match pkg/reaperdeploy.Tags.
+VERSION?=$(shell git describe --tags --always --dirty 2>/dev/null | sed 's/^v//' || echo unknown)
+
 BINARY_NAME=build/bootstrap
 ENV?=production
 
@@ -21,7 +28,7 @@ deploy: build
 		--template-file template.yaml \
 		--stack-name spawn-autoscale-orchestrator-$(ENV) \
 		--capabilities CAPABILITY_IAM \
-		--parameter-overrides Environment=$(ENV) \
+		--parameter-overrides Environment=$(ENV) Version=$(VERSION) \
 		--resolve-s3
 
 deploy-staging:

lambda/autoscale-orchestrator/template.yaml

@@ -13,6 +13,17 @@ Parameters:
     Default: 'arn:aws:iam::435415984226:role/spawn-autoscale-ec2'
     Description: ARN of the cross-account EC2 management role in spore-host-dev
 
+  Version:
+    Type: String
+    Default: unknown
+    Description: >
+      The spawn version this artifact was built from, stamped onto the function as
+      the spawn:version tag so an operator can read it in ONE API call (spawn#654).
+      Nothing synchronises this stack with the CLI, so version skew is the normal
+      state — and until now it was invisible, answerable only by comparing the
+      function's LastModified against git log. `unknown` means it was deployed by
+      something that did not stamp it.
+
 Resources:
   AutoScaleOrchestratorFunction:
     Type: AWS::Serverless::Function
@@ -35,6 +46,15 @@ Resources:
           HYBRID_REGISTRY_TABLE: spawn-hybrid-registry
           EC2_ROLE_ARN: !Ref EC2RoleArn
           EC2_EXTERNAL_ID: spawn-autoscale-orchestrator
+          SPAWN_VERSION: !Ref Version
+      # Readable without invoking the function (spawn#654). Mirrors
+      # pkg/reaperdeploy.Tags so every spore.host lambda answers the same
+      # question the same way.
+      Tags:
+        spawn:managed: 'true'
+        spawn:component: autoscale-orchestrator
+        spawn:created-by: sam deploy
+        spawn:version: !Ref Version
       Policies:
         - DynamoDBCrudPolicy:
             TableName: !Ref AutoScaleGroupsTable

lambda/pipeline-orchestrator/Makefile

@@ -1,5 +1,12 @@
 .PHONY: build deploy clean test
 
+# The version stamped onto the function as spawn:version (#654). `git describe`
+# rather than a bare tag: it yields 0.126.1 on a tag, 0.126.1-3-gabc when ahead,
+# and a -dirty suffix for uncommitted changes — so the stamp says "deployed from
+# uncommitted code" out loud instead of claiming a release. Leading v stripped to
+# match pkg/reaperdeploy.Tags.
+VERSION?=$(shell git describe --tags --always --dirty 2>/dev/null | sed 's/^v//' || echo unknown)
+
 # The deployed function's name, in ONE place. It must match template.yaml's
 # `FunctionName:` exactly — there is no Environment suffix. update-code, logs and
 # invoke-test each used to hardcode `spawn-pipeline-orchestrator-production`,
@@ -28,7 +35,7 @@ deploy: build
 		--template-file template.yaml \
 		--stack-name $(STACK) \
 		--capabilities CAPABILITY_IAM \
-		--parameter-overrides Environment=production \
+		--parameter-overrides Environment=production Version=$(VERSION) \
 		--resolve-s3
 
 # Deploy to staging
@@ -37,7 +44,7 @@ deploy-staging: build
 		--template-file template.yaml \
 		--stack-name $(STACK)-staging \
 		--capabilities CAPABILITY_IAM \
-		--parameter-overrides Environment=staging \
+		--parameter-overrides Environment=staging Version=$(VERSION) \
 		--resolve-s3
 
 # Test locally (requires SAM CLI)

lambda/pipeline-orchestrator/template.yaml

@@ -12,6 +12,17 @@ Parameters:
       - development
     Description: Environment name
 
+  Version:
+    Type: String
+    Default: unknown
+    Description: >
+      The spawn version this artifact was built from, stamped onto the function as
+      the spawn:version tag so an operator can read it in ONE API call (spawn#654).
+      Nothing synchronises this stack with the CLI, so version skew is the normal
+      state — and until now it was invisible, answerable only by comparing the
+      function's LastModified against git log. `unknown` means it was deployed by
+      something that did not stamp it.
+
 Resources:
   # DynamoDB table for pipeline state
   PipelineOrchestrationTable:
@@ -108,6 +119,11 @@ Resources:
         Environment: !Ref Environment
         Application: spawn
         Component: pipeline-orchestrator
+        # Readable without invoking the function (spawn#654).
+        spawn:managed: 'true'
+        spawn:component: pipeline-orchestrator
+        spawn:created-by: sam deploy
+        spawn:version: !Ref Version
 
   # CloudWatch Log Group with retention
   PipelineOrchestratorLogGroup:

lambda/ttl-reaper/Makefile

@@ -2,6 +2,12 @@
 
 BINARY_NAME=build/bootstrap
 ENV?=production
+# The version stamped onto the function as spawn:version (#654). `git describe`
+# rather than a bare tag on purpose: it yields v0.126.1 on a tag, v0.126.1-3-gabc
+# when ahead, and a -dirty suffix when the tree has uncommitted changes — so the
+# stamp says "deployed from uncommitted code" out loud instead of claiming a
+# release. The leading v is stripped to match pkg/reaperdeploy.Tags.
+VERSION?=$(shell git describe --tags --always --dirty 2>/dev/null | sed 's/^v//' || echo unknown)
 # The reaper stack lives in us-east-1. Explicit, because every AWS call below
 # must target the same region the stack is in (#650 follow-up).
 REGION?=us-east-1
@@ -92,7 +98,8 @@ PARAM_MAP = \
 	AlarmTopicArn:ALARM_TOPIC_ARN \
 	MaxAge:MAX_AGE \
 	Regions:REGIONS \
-	Schedule:SCHEDULE
+	Schedule:SCHEDULE \
+	Version:VERSION
 
 # Emit one `CfnKey=value` line per EXPLICITLY-supplied parameter. Deliberately not
 # JSON: Make has no comma literal (`$(,)` is undefined and silently expands to
@@ -137,7 +144,7 @@ deploy: build
 	@# off — but never NotifyUrl, which is a Slack webhook secret. These are read
 	@# back from the MERGED file, so this shows what will actually be sent rather
 	@# than what this Makefile happens to default to.
-	@awk -F': ' '/^(DryRun|ScanSelf|DnsSweep|DnsExpire|NetResources|AlarmsEnabled|DnsZoneId|DnsDomain|AlarmTopicArn|MaxAge|Schedule):/ \
+	@awk -F': ' '/^(DryRun|ScanSelf|DnsSweep|DnsExpire|NetResources|AlarmsEnabled|DnsZoneId|DnsDomain|AlarmTopicArn|MaxAge|Schedule|Version):/ \
 		{ printf "  %s %s\n", $$1, $$2 }' $(PARAMS_FILE)
 	sam deploy \
 		--template-file template.yaml \

lambda/ttl-reaper/template.yaml

@@ -161,6 +161,22 @@ Parameters:
       — which is the state this whole change exists to fix. Empty creates the alarms
       with no action, so they are visible in the console but will page no one.
 
+  Version:
+    Type: String
+    Default: unknown
+    Description: >
+      The spawn version this artifact was built from, stamped onto the function as
+      the spawn:version tag so an operator can read it in ONE API call (spawn#654).
+      Nothing synchronises this stack with the CLI — a `sam deploy` is the only
+      thing that moves it — so version skew is the NORMAL state, and until now it
+      was invisible: this stack sat untouched from 2026-07-31 to 2026-10-04 while
+      the CLI went from ~v0.9x to v0.116.0, missing three merged fixes, and nothing
+      anywhere said so. Answering "is the deployed reaper current?" required
+      comparing the function's LastModified against `git log -- lambda/ttl-reaper/`,
+      which is archaeology rather than a check.
+      Passed explicitly by the Makefile on every deploy. `unknown` means it was
+      deployed by something that did not stamp it.
+
 Conditions:
   ScanSelfEnabled: !Equals [!Ref ScanSelf, 'true']
   RegistryReadEnabled: !And
@@ -209,6 +225,16 @@ Resources:
           REAPER_NET_RESOURCES: !Ref NetResources
           REAPER_DNS_EXPIRE: !Ref DnsExpire
           ACCOUNTS_TABLE: !Ref AccountsTable
+          SPAWN_VERSION: !Ref Version
+      # Readable without invoking the function, which is the whole point: a check
+      # that has to invoke a reaper to ask its version is not a check anyone will
+      # run against production. Mirrors pkg/reaperdeploy.Tags, so the self-hosted
+      # and operated reapers answer the same question the same way (spawn#654).
+      Tags:
+        spawn:managed: 'true'
+        spawn:component: ttl-reaper
+        spawn:created-by: sam deploy
+        spawn:version: !Ref Version
       Policies:
         # Assume each per-account role that actually describes/terminates EC2.
         - Statement:

scripts/deploy-custom-dns.sh

@@ -246,6 +246,20 @@ LAMBDA_ARN=$(aws lambda get-function \
   --output text)
 
 info "Lambda ARN: $LAMBDA_ARN"
+
+# Stamp the version so an operator can read it in ONE API call (#654). This
+# function is not CloudFormation-managed, so a tag here is not stack drift.
+#
+# This script was the one the version census found: it deploys spawn-dns-updater
+# from lambda/dns-updater/, but the script's own name says "custom-dns", so an
+# enumeration keyed on the lambda directory name missed it entirely — the same
+# shape as the runtime census finding github-oauth-bridge.
+SPAWN_VERSION=$(git describe --tags --always --dirty 2>/dev/null | sed 's/^v//' || echo unknown)
+aws lambda tag-resource \
+  --resource "$LAMBDA_ARN" \
+  --tags "spawn:managed=true,spawn:component=dns-updater,spawn:created-by=deploy-custom-dns.sh,spawn:version=$SPAWN_VERSION" \
+  > /dev/null
+info "Stamped spawn:version=$SPAWN_VERSION"
 echo ""
 
 # Step 5: Create API Gateway

scripts/deploy-scheduler-handler.sh

@@ -181,6 +181,20 @@ aws lambda wait function-updated \
     --region "$REGION" \
     --profile "$PROFILE"
 
+# Stamp the version so an operator can read it in ONE API call (#654). Not
+# CloudFormation-managed, so a tag added here is not stack drift. The create
+# branch above sets Application/Component tags but no version, and the UPDATE
+# path — the one that actually runs on a redeploy — set no tags at all.
+SPAWN_VERSION=$(git describe --tags --always --dirty 2>/dev/null | sed 's/^v//' || echo unknown)
+SCHED_ARN=$(aws lambda get-function-configuration \
+    --function-name "$FUNCTION_NAME" --region "$REGION" --profile "$PROFILE" \
+    --query FunctionArn --output text)
+aws lambda tag-resource \
+    --resource "$SCHED_ARN" \
+    --tags "spawn:managed=true,spawn:component=scheduler-handler,spawn:created-by=deploy-scheduler-handler.sh,spawn:version=$SPAWN_VERSION" \
+    --region "$REGION" --profile "$PROFILE" >/dev/null
+echo "stamped spawn:version=$SPAWN_VERSION"
+
 echo ""
 echo "✅ Deployment complete!"
 echo ""

scripts/deploy-sweep-orchestrator.sh

@@ -100,6 +100,27 @@ else
     --function-name "$FUNCTION_NAME" --region "$REGION"
 fi
 
+# Stamp the version so an operator can read it in ONE API call (#654).
+#
+# This function is NOT CloudFormation-managed, so a tag added here is not stack
+# drift — unlike the SAM-deployed lambdas, where the tag is declared in the
+# template for exactly that reason.
+#
+# `git describe` rather than a bare tag: it yields 0.126.1 on a tag,
+# 0.126.1-3-gabc when ahead, and a -dirty suffix for uncommitted changes, so the
+# stamp says "deployed from uncommitted code" out loud instead of claiming a
+# release. Before this, answering "is the deployed sweep orchestrator current?"
+# meant comparing LastModified against `git log -- lambda/sweep-orchestrator/`.
+SPAWN_VERSION=$(git describe --tags --always --dirty 2>/dev/null | sed 's/^v//' || echo unknown)
+FUNCTION_ARN=$(aws lambda get-function-configuration \
+  --function-name "$FUNCTION_NAME" --region "$REGION" \
+  --query FunctionArn --output text)
+aws lambda tag-resource \
+  --resource "$FUNCTION_ARN" \
+  --tags "spawn:managed=true,spawn:component=sweep-orchestrator,spawn:created-by=deploy-sweep-orchestrator.sh,spawn:version=$SPAWN_VERSION" \
+  --region "$REGION" >/dev/null
+echo "stamped spawn:version=$SPAWN_VERSION"
+
 rm function.zip
 
 # Verify the function is actually running the binary just built. An HTTP 200 from