ROOTFS snapshot fidelity: micro-VM capture and restore, API opened for micro-VM templates

cmd/ateapi/internal/apivalidation/actor_template.go

@@ -61,14 +61,24 @@ func ValidateActorTemplateUpdate(ctx context.Context, fldPath *field.Path, newVa
 	return Validate_ActorTemplate(ctx, op, fldPath, newVal, oldVal)
 }
 
-// ValidateCustom_CreateActorTemplateRequest_ActorTemplate rejects container
-// volume mounts that reference volumes the template does not declare.
+// ValidateCustom_CreateActorTemplateRequest_ActorTemplate holds the rules
+// that span more than one field of a template: container volume mounts must
+// reference volumes the template declares, and a ROOTFS preferred fidelity
+// needs a sandbox class that can capture rootfs changes without memory, which
+// today is the micro-VM class alone.
 func ValidateCustom_CreateActorTemplateRequest_ActorTemplate(_ context.Context, _ operation.Operation, fldPath *field.Path, value, _ *ateapipb.ActorTemplate) field.ErrorList {
+	var errs field.ErrorList
+	if value.GetSnapshotConfig().GetPreferredFidelity() == ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS &&
+		value.GetSandboxConfig().GetSandboxClass() != ateapipb.SandboxClass_SANDBOX_CLASS_MICROVM {
+		errs = append(errs, field.Invalid(
+			fldPath.Child("snapshot_config", "preferred_fidelity"),
+			value.GetSnapshotConfig().GetPreferredFidelity().String(),
+			"ROOTFS fidelity requires sandbox_config.sandbox_class SANDBOX_CLASS_MICROVM"))
+	}
 	declared := make(map[string]bool, len(value.GetVolumes()))
 	for _, vol := range value.GetVolumes() {
 		declared[vol.GetName()] = true
 	}
-	var errs field.ErrorList
 	for i, ctr := range value.GetContainers() {
 		for j, mount := range ctr.GetVolumeMounts() {
 			name := mount.GetName()
@@ -237,15 +247,6 @@ func ValidateCustom_SnapshotConfig_StorageLocation(_ context.Context, _ operatio
 	return nil
 }
 
-// ValidateCustom_SnapshotConfig_PreferredFidelity rejects ROOTFS until a
-// sandbox runtime can capture root filesystem changes without memory.
-func ValidateCustom_SnapshotConfig_PreferredFidelity(_ context.Context, _ operation.Operation, fldPath *field.Path, value, _ *ateapipb.SnapshotFidelity) field.ErrorList {
-	if *value == ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS {
-		return field.ErrorList{field.Invalid(fldPath, value.String(), "ROOTFS fidelity is not supported yet")}
-	}
-	return nil
-}
-
 // envVarNameRE constrains env var names to any printable ASCII character
 // except '='.
 var envVarNameRE = regexp.MustCompile(`^[ -<>-~]+$`)

cmd/ateapi/internal/apivalidation/actor_template_test.go

@@ -69,13 +69,20 @@ func TestValidateCreateActorTemplateRequest(t *testing.T) {
 		})},
 		nil,
 	}, {
-		// ROOTFS is in the enum for the API's sake but no runtime captures it
-		// yet, so templates may not ask for it.
-		"rootfs fidelity not supported",
+		// Only the micro-VM runtime captures rootfs changes without memory;
+		// the fixture is a gVisor template.
+		"rootfs fidelity on gvisor rejected",
 		&ateapipb.CreateActorTemplateRequest{ActorTemplate: validActorTemplate(func(tmpl *ateapipb.ActorTemplate) {
 			tmpl.SnapshotConfig.PreferredFidelity = ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS
 		})},
 		field.ErrorList{field.Invalid(field.NewPath("actor_template", "snapshot_config", "preferred_fidelity"), "SNAPSHOT_FIDELITY_ROOTFS", "")},
+	}, {
+		"rootfs fidelity on microvm accepted",
+		&ateapipb.CreateActorTemplateRequest{ActorTemplate: validActorTemplate(func(tmpl *ateapipb.ActorTemplate) {
+			tmpl.SnapshotConfig.PreferredFidelity = ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS
+			tmpl.SandboxConfig = &ateapipb.SandboxConfig{SandboxClass: ateapipb.SandboxClass_SANDBOX_CLASS_MICROVM, ConfigName: "microvm-default"}
+		})},
+		nil,
 	}, {
 		"invalid worker_selector label key",
 		&ateapipb.CreateActorTemplateRequest{ActorTemplate: validActorTemplate(func(tmpl *ateapipb.ActorTemplate) {

cmd/ateapi/internal/controlapi/workflow_pause_test.go

@@ -163,8 +163,9 @@ func TestEnsurePausedFinalized_RecordsFidelity(t *testing.T) {
 		fidelity ateapipb.SnapshotFidelity
 		want     ateapipb.SnapshotFidelity
 	}{
-		{"data", ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES},
-		{"full", ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY},
+		{"volumes", ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES},
+		{"rootfs", ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS},
+		{"memory", ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY},
 	}
 	for _, tc := range tests {
 		t.Run(tc.name, func(t *testing.T) {

cmd/ateapi/internal/controlapi/workflow_resume_test.go

@@ -1138,6 +1138,7 @@ func TestResumeActor_AteletWireRequest(t *testing.T) {
 	goldenURI := someActorSnapshotURI(t, "gs://bucket/golden-root", "ate-golden", "golden-1")
 
 	fullScope := ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY
+	rootfsScope := ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS
 	dataScope := ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES
 	unspecScope := ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED
 
@@ -1258,6 +1259,29 @@ func TestResumeActor_AteletWireRequest(t *testing.T) {
 				scope:          ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES,
 			},
 		},
+		{
+			name:  "08a Rootfs durable snapshot restores as Rootfs",
+			actor: actorSeed{externalSnapshot: extSnap(actorURI, rootfsScope)},
+			want: restoreWant{
+				checkpointType: ateletpb.CheckpointType_CHECKPOINT_TYPE_EXTERNAL,
+				snapshotURI:    actorURI,
+				scope:          ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS,
+			},
+		},
+		{
+			// A rootfs upper is only valid over the image it was written on,
+			// so a repointed actor drops to Data like a Full snapshot does.
+			name: "08b repointed actor's Rootfs durable snapshot drops to Data",
+			actor: actorSeed{
+				externalSnapshot: extSnap(actorURI, rootfsScope),
+				tmplUID:          "mismatch",
+			},
+			want: restoreWant{
+				checkpointType: ateletpb.CheckpointType_CHECKPOINT_TYPE_EXTERNAL,
+				snapshotURI:    actorURI,
+				scope:          ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES,
+			},
+		},
 		{
 			name:  "09 Data durable snapshot restores as Data",
 			actor: actorSeed{externalSnapshot: extSnap(actorURI, dataScope)},

cmd/atelet/lifecycle_test.go

@@ -66,6 +66,16 @@ type fakeAteom struct {
 	// preserveRestoreDir records the PreserveRestoreDir flag from the most
 	// recent RestoreWorkload request.
 	preserveRestoreDir bool
+	// fidelities records the fidelity each checkpoint or restore arrived
+	// with, by RPC name.
+	fidelities map[string]ateompb.SnapshotFidelity
+}
+
+func (f *fakeAteom) recordFidelity(rpc string, fidelity ateompb.SnapshotFidelity) {
+	if f.fidelities == nil {
+		f.fidelities = map[string]ateompb.SnapshotFidelity{}
+	}
+	f.fidelities[rpc] = fidelity
 }
 
 func (f *fakeAteom) recordActorDirs(rpc string, actorDirs *ateompb.ActorDirs) {
@@ -82,6 +92,7 @@ func (f *fakeAteom) RunWorkload(_ context.Context, req *ateompb.RunWorkloadReque
 
 func (f *fakeAteom) CheckpointWorkload(_ context.Context, req *ateompb.CheckpointWorkloadRequest) (*ateompb.CheckpointWorkloadResponse, error) {
 	f.recordActorDirs("CheckpointWorkload", req.GetActorDirs())
+	f.recordFidelity("CheckpointWorkload", req.GetFidelity())
 	dir := req.GetActorDirs().GetCheckpointDir()
 	names := make([]string, 0, len(f.snapshotFiles))
 	for name, body := range f.snapshotFiles {
@@ -95,6 +106,7 @@ func (f *fakeAteom) CheckpointWorkload(_ context.Context, req *ateompb.Checkpoin
 
 func (f *fakeAteom) RestoreWorkload(_ context.Context, req *ateompb.RestoreWorkloadRequest) (*ateompb.RestoreWorkloadResponse, error) {
 	f.recordActorDirs("RestoreWorkload", req.GetActorDirs())
+	f.recordFidelity("RestoreWorkload", req.GetFidelity())
 	f.preserveRestoreDir = req.GetPreserveRestoreDir()
 	dir := req.GetActorDirs().GetRestoreDir()
 	f.restored = map[string]string{}
@@ -140,8 +152,20 @@ func serveFakeAteom(t *testing.T, f *fakeAteom) {
 
 // TestLocalSnapshotGC walks an actor through
 // run -> pause -> resume -> terminate over atelet's RPC surface and ensures that
-// the local snapshot is garbage collected after the actor is terminated.
+// the local snapshot is garbage collected after the actor is terminated. It
+// runs at every fidelity atelet forwards, pinning that the pause and the
+// resume hand ateom the fidelity the control plane asked for, unchanged.
 func TestLocalSnapshotGC(t *testing.T) {
+	for _, fidelity := range []ateletpb.SnapshotFidelity{
+		ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES,
+		ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS,
+		ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY,
+	} {
+		t.Run(fidelity.String(), func(t *testing.T) { runLocalSnapshotGC(t, fidelity) })
+	}
+}
+
+func runLocalSnapshotGC(t *testing.T, fidelity ateletpb.SnapshotFidelity) {
 	useTempNodeDirs(t)
 	ctx := t.Context()
 
@@ -207,7 +231,7 @@ func TestLocalSnapshotGC(t *testing.T) {
 		ActorTemplateName:     "counter",
 		WorkerPodUid:          workerPodUID,
 		Spec:                  spec,
-		Fidelity:              ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY,
+		Fidelity:              fidelity,
 		Type:                  ateletpb.CheckpointType_CHECKPOINT_TYPE_LOCAL,
 		Config: &ateletpb.CheckpointRequest_LocalConfig{
 			LocalConfig: &ateletpb.LocalCheckpointConfiguration{SnapshotName: snapshotName},
@@ -230,7 +254,7 @@ func TestLocalSnapshotGC(t *testing.T) {
 		WorkerPodUid:          workerPodUID,
 		SandboxAssets:         sandboxAssets,
 		Spec:                  spec,
-		Fidelity:              ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY,
+		Fidelity:              fidelity,
 		Type:                  ateletpb.CheckpointType_CHECKPOINT_TYPE_LOCAL,
 		Config: &ateletpb.RestoreRequest_LocalConfig{
 			LocalConfig: &ateletpb.LocalCheckpointConfiguration{SnapshotName: snapshotName},
@@ -241,6 +265,11 @@ func TestLocalSnapshotGC(t *testing.T) {
 	if got := ateom.restored["checkpoint.img"]; got != "guest-memory" {
 		t.Fatalf("restore staged %q for ateom, want the pause snapshot's %q", got, "guest-memory")
 	}
+	for _, rpc := range []string{"CheckpointWorkload", "RestoreWorkload"} {
+		if got := ateom.fidelities[rpc]; got != toAteomFidelity(fidelity) {
+			t.Errorf("%s carried fidelity %v, want %v", rpc, got, toAteomFidelity(fidelity))
+		}
+	}
 	if !ateom.preserveRestoreDir {
 		t.Errorf("RestoreWorkload preserve_restore_dir = false, want true for pure-local restore")
 	}

cmd/atelet/main.go

@@ -1003,18 +1003,12 @@ func (s *AteomHerder) uploadLocalCheckpointDir(ctx context.Context, req *ateletp
 	if capturedFidelity == "" {
 		return rec.SandboxClass, apierror.FailedPrecondition("local snapshot %q has no fidelity recorded in its manifest; resume and pause the actor again before suspending it", req.GetLocalSnapshotName())
 	}
-	desiredFidelity := ateattr.SnapshotFidelityValue(req.GetDesiredFidelity())
-
-	switch {
-	case capturedFidelity == desiredFidelity:
-	case capturedFidelity == ateattr.SnapshotFidelityVolumes && desiredFidelity == ateattr.SnapshotFidelityMemory:
-		// The control plane rejects this before marking SUSPENDING; reaching
-		// it here means the template changed mid-flight or store state drifted.
-		return rec.SandboxClass, apierror.FailedPrecondition("pause snapshot captured %s; cannot upload it as %s (memory was never captured)", capturedFidelity, desiredFidelity)
-	default: // captured MEMORY, VOLUMES wanted
-		if err := narrowMemoryCaptureToVolumes(rec); err != nil {
-			return rec.SandboxClass, err
-		}
+	// Pause and suspend capture the template's one preferred fidelity, and a
+	// paused actor's template cannot change, so the upload is always of what
+	// the pause captured. A mismatch means the store or the control plane
+	// drifted; refuse rather than guess.
+	if desiredFidelity := ateattr.SnapshotFidelityValue(req.GetDesiredFidelity()); capturedFidelity != desiredFidelity {
+		return rec.SandboxClass, apierror.FailedPrecondition("pause snapshot captured %s; upload asks for %s", capturedFidelity, desiredFidelity)
 	}
 
 	// The local snapshot stays until the upload succeeds, so a retry can
@@ -1031,19 +1025,6 @@ func readSnapshotManifest(dir string) ([]byte, error) {
 	return root.ReadFile(sandboxManifestName)
 }
 
-// narrowMemoryCaptureToVolumes rewrites rec so a MEMORY capture uploads as a
-// VOLUMES snapshot holding only the volume files ateom reported at checkpoint.
-func narrowMemoryCaptureToVolumes(rec *sandboxAssetsRecord) error {
-	if len(rec.DataSnapshotFiles) == 0 {
-		// Either no durable-dir volumes were attached at pause, or the
-		// manifest predates the list. Neither is retryable.
-		return apierror.FailedPrecondition("memory capture lists no volume files; the actor has no durable data to upload as %s", ateattr.SnapshotFidelityVolumes)
-	}
-	rec.SnapshotFiles = rec.DataSnapshotFiles
-	rec.Fidelity = ateattr.SnapshotFidelityVolumes
-	return nil
-}
-
 func (s *AteomHerder) Restore(ctx context.Context, req *ateletpb.RestoreRequest) (resp *ateletpb.RestoreResponse, err error) {
 	if err := validateRestoreRequest(req); err != nil {
 		return nil, apierror.InvalidArgument("%v", err)
@@ -1819,18 +1800,17 @@ func validateTerminateRequest(req *ateletpb.TerminateRequest) error {
 	return resources.ValidateContainerNames(names)
 }
 
-// validateFidelity rejects a fidelity no runtime can serve. ROOTFS is in the
-// enum but not captured by any sandbox runtime yet, so it is refused here as
-// well as at template admission.
+// validateFidelity accepts every level of the ladder. Which levels a sandbox
+// runtime can actually serve is the runtime's call: atelet forwards the
+// fidelity and ateom rejects what it cannot capture.
 func validateFidelity(fidelity ateletpb.SnapshotFidelity) error {
 	switch fidelity {
-	case ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY,
-		ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES:
+	case ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES,
+		ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS,
+		ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY:
 		return nil
 	case ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED:
 		return fmt.Errorf("snapshot fidelity must be non-zero")
-	case ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS:
-		return fmt.Errorf("ROOTFS fidelity is not supported yet")
 	default:
 		return fmt.Errorf("invalid snapshot fidelity: %v", fidelity)
 	}
@@ -1850,14 +1830,8 @@ func validateUploadPausedCheckpointRequest(req *ateletpb.UploadPausedCheckpointR
 	if _, err := resources.ParseSnapshotURI(req.GetDestinationSnapshotUri()); err != nil {
 		errs = append(errs, field.Invalid(field.NewPath("destination_snapshot_uri"), req.GetDestinationSnapshotUri(), err.Error()))
 	}
-	// Uploads only ever produce MEMORY or VOLUMES snapshots.
-	switch req.GetDesiredFidelity() {
-	case ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES:
-	case ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS:
-		errs = append(errs, field.Invalid(field.NewPath("desired_fidelity"), req.GetDesiredFidelity().String(), "ROOTFS fidelity is not supported yet"))
-	default:
-		errs = append(errs, field.NotSupported(field.NewPath("desired_fidelity"), req.GetDesiredFidelity(),
-			[]string{ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY.String(), ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES.String()}))
+	if err := validateFidelity(req.GetDesiredFidelity()); err != nil {
+		errs = append(errs, field.Invalid(field.NewPath("desired_fidelity"), req.GetDesiredFidelity().String(), err.Error()))
 	}
 	return errs.ToAggregate()
 }