Import image from console sometimes would sometimes attempt to delete the tmp disk a bit too soon #4321
Closed as not planned
Description
This happens only once to me but seems like something we should look into more, given we had an image import connection time issue reported in the field recently on an older release (but the UI part likely hasn't changed since then):
The disk deletion failed as it was triggered while disk-finalize was still running:
root@[fd00:1122:3344:105::3]:32221/omicron> select event_time, saga_id, name, event_type,data from saga_node_event, saga where saga_id in ('e65dab54-4f10-4e3e-a994-27b9ca0b7840','7c1a3623-0a64-4f43-a7d4-a ') and saga.id = saga_id order by 1, 2;
event_time | saga_id | name | event_type | data
--------------------------------+--------------------------------------+---------------+---------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
2023-10-23 21:46:16.108703+00 | 7c1a3623-0a64-4f43-a7d4-ace7ed0e4884 | finalize-disk | started | NULL
2023-10-23 21:46:16.111555+00 | 7c1a3623-0a64-4f43-a7d4-ace7ed0e4884 | finalize-disk | succeeded | null
2023-10-23 21:46:16.113667+00 | 7c1a3623-0a64-4f43-a7d4-ace7ed0e4884 | finalize-disk | started | NULL
2023-10-23 21:46:16.18767+00 | 7c1a3623-0a64-4f43-a7d4-ace7ed0e4884 | finalize-disk | succeeded | 5
2023-10-23 21:46:16.189775+00 | 7c1a3623-0a64-4f43-a7d4-ace7ed0e4884 | finalize-disk | started | NULL
2023-10-23 21:46:16.21611+00 | 7c1a3623-0a64-4f43-a7d4-ace7ed0e4884 | finalize-disk | succeeded | "[fd00:1122:3344:105::4]:17000"
2023-10-23 21:46:16.218302+00 | 7c1a3623-0a64-4f43-a7d4-ace7ed0e4884 | finalize-disk | started | NULL
2023-10-23 21:46:16.220369+00 | 7c1a3623-0a64-4f43-a7d4-ace7ed0e4884 | finalize-disk | succeeded | {"attached_instance_and_sled": null, "create_params": {"description": "snapshot of finalized disk f1da1901-4c50-433b-b5b0-3cae4e8885b4", "disk": "f1da1901-4c50-433b-b5b0-3cae4e8885b4", "name": "tmp-snapshot-44499152"}, "disk_id": "f1da1901-4c50-433b-b5b0-3cae4e8885b4", "project_id": "5e49b6de-cb2d-438d-83af-95c415bbb901", "serialized_authn": {"kind": {"Authenticated": [{"actor": {"SiloUser": {"silo_id": "7bd7623a-68ed-4636-8ecb-b59e3b068787", "silo_user_id": "906e74cb-eab8-4a87-bda8-2cb0914bf853"}}}, {"mapped_fleet_roles": {"admin": ["admin"]}}]}}, "silo_id": "7bd7623a-68ed-4636-8ecb-b59e3b068787"}
2023-10-23 21:46:16.222604+00 | 7c1a3623-0a64-4f43-a7d4-ace7ed0e4884 | finalize-disk | started | NULL
2023-10-23 21:46:16.224716+00 | 7c1a3623-0a64-4f43-a7d4-ace7ed0e4884 | finalize-disk | succeeded | null
2023-10-23 21:46:16.226694+00 | 7c1a3623-0a64-4f43-a7d4-ace7ed0e4884 | finalize-disk | started | NULL
2023-10-23 21:46:16.228662+00 | 7c1a3623-0a64-4f43-a7d4-ace7ed0e4884 | finalize-disk | succeeded | "b5d88ebe-007d-41ee-b672-5387efe5439f"
2023-10-23 21:46:16.230382+00 | 7c1a3623-0a64-4f43-a7d4-ace7ed0e4884 | finalize-disk | started | NULL
2023-10-23 21:46:16.232264+00 | 7c1a3623-0a64-4f43-a7d4-ace7ed0e4884 | finalize-disk | succeeded | "c1f48bdf-1545-4088-96ea-c860c34a327f"
2023-10-23 21:46:16.23419+00 | 7c1a3623-0a64-4f43-a7d4-ace7ed0e4884 | finalize-disk | started | NULL
2023-10-23 21:46:16.236089+00 | 7c1a3623-0a64-4f43-a7d4-ace7ed0e4884 | finalize-disk | succeeded | "155afa02-7a3b-4325-9aef-b6077a225126"
2023-10-23 21:46:16.237929+00 | 7c1a3623-0a64-4f43-a7d4-ace7ed0e4884 | finalize-disk | started | NULL
2023-10-23 21:46:16.346603+00 | 7c1a3623-0a64-4f43-a7d4-ace7ed0e4884 | finalize-disk | succeeded | [[{"identity": {"id": "3b767edf-a72d-4d80-a0fc-65d6801ed0e0", "time_created": "2023-08-30T18:59:10.758511Z", "time_modified": "2023-08-30T18:59:10.758511Z"}, "ip": "fd00:1122:3344:108::e", "kind": "Crucible", "pool_id": "f522118c-5dcd-4116-8044-07f0cceec52e", "port": 32345, "rcgen": 1, "size_used": 1014686023680, "time_deleted": null}, {"block_size": 4096, "blocks_per_extent": 16384, "dataset_id": "3b767edf-a72d-4d80-a0fc-65d6801ed0e0", "extent_count": 16, "identity": {"id": "9aa96905-cf4c-446d-b7ec-2e8fc1bd2d05", "time_created": "2023-10-23T21:46:16.308064Z", "time_modified": "2023-10-23T21:46:16.308064Z"}, "volume_id": "155afa02-7a3b-4325-9aef-b6077a225126"}], [{"identity": {"id": "9c5d88c9-8ff1-4f23-9438-7b81322eaf68", "time_created": "2023-08-30T18:59:10.758512Z", "time_modified": "2023-08-30T18:59:10.758512Z"}, "ip": "fd00:1122:3344:105::b", "kind": "Crucible", "pool_id": "aadf48eb-6ff0-40b5-a092-1fdd06c03e11", "port": 32345, "rcgen": 1, "size_used": 212600881152, "time_deleted": null}, {"block_size": 4096, "blocks_per_extent": 16384, "dataset_id": "9c5d88c9-8ff1-4f23-9438-7b81322eaf68", "extent_count": 16, "identity": {"id": "ea24fb4c-4175-4d0a-90e0-4487824ae65a", "time_created": "2023-10-23T21:46:16.308064Z", "time_modified": "2023-10-23T21:46:16.308064Z"}, "volume_id": "155afa02-7a3b-4325-9aef-b6077a225126"}], [{"identity": {"id": "32995cfa-47ec-4b84-8514-7c1c8a86c19d", "time_created": "2023-08-30T18:59:10.758516Z", "time_modified": "2023-08-30T18:59:10.758516Z"}, "ip": "fd00:1122:3344:102::8", "kind": "Crucible", "pool_id": "eac83f81-eb51-4f3e-874e-82f55dd952ba", "port": 32345, "rcgen": 1, "size_used": 363998478336, "time_deleted": null}, {"block_size": 4096, "blocks_per_extent": 16384, "dataset_id": "32995cfa-47ec-4b84-8514-7c1c8a86c19d", "extent_count": 16, "identity": {"id": "767cc2a0-8a39-4846-ad6d-9156fb9e4d43", "time_created": "2023-10-23T21:46:16.308064Z", "time_modified": "2023-10-23T21:46:16.308064Z"}, "volume_id": "155afa02-7a3b-4325-9aef-b6077a225126"}]]
2023-10-23 21:46:16.349041+00 | 7c1a3623-0a64-4f43-a7d4-ace7ed0e4884 | finalize-disk | started | NULL
2023-10-23 21:46:16.350992+00 | 7c1a3623-0a64-4f43-a7d4-ace7ed0e4884 | finalize-disk | succeeded | null
2023-10-23 21:46:16.352851+00 | 7c1a3623-0a64-4f43-a7d4-ace7ed0e4884 | finalize-disk | started | NULL
2023-10-23 21:46:17.035338+00 | 7c1a3623-0a64-4f43-a7d4-ace7ed0e4884 | finalize-disk | succeeded | "{\\"type\\":\\"volume\\",\\"block_size\\":4096,\\"id\\":\\"155afa02-7a3b-4325-9aef-b6077a225126\\",\\"sub_volumes\\":[{\\"type\\":\\"region\\",\\"block_size\\":4096,\\"blocks_per_extent\\":16384,\\"extent_count\\":16,\\"gen\\":1,\\"opts\\":{\\"id\\":\\"155afa02-7a3b-4325-9aef-b6077a225126\\",\\"key\\":\\"vKHuPCKgqBCnyqv5fck/NuEvq/tr3IT6Wjlyo2faCkE=\\",\\"lossy\\":false,\\"read_only\\":false,\\"target\\":[\\"[fd00:1122:3344:108::e]:19031\\",\\"[fd00:1122:3344:105::b]:19022\\",\\"[fd00:1122:3344:102::8]:19043\\"]}}]}"
2023-10-23 21:46:17.038517+00 | 7c1a3623-0a64-4f43-a7d4-ace7ed0e4884 | finalize-disk | started | NULL
2023-10-23 21:46:17.047636+00 | 7c1a3623-0a64-4f43-a7d4-ace7ed0e4884 | finalize-disk | succeeded | null
2023-10-23 21:46:17.049993+00 | 7c1a3623-0a64-4f43-a7d4-ace7ed0e4884 | finalize-disk | started | NULL
2023-10-23 21:46:17.180083+00 | 7c1a3623-0a64-4f43-a7d4-ace7ed0e4884 | finalize-disk | succeeded | {"block_size": "AdvancedFormat", "destination_volume_id": "155afa02-7a3b-4325-9aef-b6077a225126", "disk_id": "f1da1901-4c50-433b-b5b0-3cae4e8885b4", "gen": 1, "identity": {"description": "snapshot of finalized disk f1da1901-4c50-433b-b5b0-3cae4e8885b4", "id": "b5d88ebe-007d-41ee-b672-5387efe5439f", "name": "tmp-snapshot-44499152", "time_created": "2023-10-23T21:46:17.079896Z", "time_deleted": null, "time_modified": "2023-10-23T21:46:17.079896Z"}, "project_id": "5e49b6de-cb2d-438d-83af-95c415bbb901", "size": 1073741824, "state": "Creating", "volume_id": "c1f48bdf-1545-4088-96ea-c860c34a327f"}
2023-10-23 21:46:17.182285+00 | 7c1a3623-0a64-4f43-a7d4-ace7ed0e4884 | finalize-disk | started | NULL
2023-10-23 21:46:17.191257+00 | 7c1a3623-0a64-4f43-a7d4-ace7ed0e4884 | finalize-disk | succeeded | null
2023-10-23 21:46:17.193201+00 | 7c1a3623-0a64-4f43-a7d4-ace7ed0e4884 | finalize-disk | started | NULL
2023-10-23 21:46:17.218696+00 | 7c1a3623-0a64-4f43-a7d4-ace7ed0e4884 | finalize-disk | succeeded | ["[fd00:1122:3344:105::4]:17000", true]
2023-10-23 21:46:17.221334+00 | 7c1a3623-0a64-4f43-a7d4-ace7ed0e4884 | finalize-disk | started | NULL
2023-10-23 21:46:17.268724+00 | 7c1a3623-0a64-4f43-a7d4-ace7ed0e4884 | finalize-disk | succeeded | 5
2023-10-23 21:46:17.270781+00 | 7c1a3623-0a64-4f43-a7d4-ace7ed0e4884 | finalize-disk | started | NULL
2023-10-23 21:46:17.273093+00 | 7c1a3623-0a64-4f43-a7d4-ace7ed0e4884 | finalize-disk | succeeded | null
2023-10-23 21:46:17.275007+00 | 7c1a3623-0a64-4f43-a7d4-ace7ed0e4884 | finalize-disk | started | NULL
2023-10-23 21:46:17.404666+00 | 7c1a3623-0a64-4f43-a7d4-ace7ed0e4884 | finalize-disk | succeeded | null
2023-10-23 21:46:17.407461+00 | 7c1a3623-0a64-4f43-a7d4-ace7ed0e4884 | finalize-disk | started | NULL
2023-10-23 21:46:17.410266+00 | 7c1a3623-0a64-4f43-a7d4-ace7ed0e4884 | finalize-disk | succeeded | null
2023-10-23 21:46:17.412637+00 | 7c1a3623-0a64-4f43-a7d4-ace7ed0e4884 | finalize-disk | started | NULL
2023-10-23 21:46:17.414575+00 | 7c1a3623-0a64-4f43-a7d4-ace7ed0e4884 | finalize-disk | succeeded | null
2023-10-23 21:46:17.41714+00 | 7c1a3623-0a64-4f43-a7d4-ace7ed0e4884 | finalize-disk | started | NULL
2023-10-23 21:46:17.759018+00 | e65dab54-4f10-4e3e-a994-27b9ca0b7840 | disk-delete | started | NULL
2023-10-23 21:46:17.761684+00 | e65dab54-4f10-4e3e-a994-27b9ca0b7840 | disk-delete | succeeded | null
2023-10-23 21:46:17.763969+00 | e65dab54-4f10-4e3e-a994-27b9ca0b7840 | disk-delete | started | NULL
2023-10-23 21:46:17.770142+00 | e65dab54-4f10-4e3e-a994-27b9ca0b7840 | disk-delete | failed | {"ActionFailed": {"source_error": {"InvalidRequest": {"message": "disk cannot be deleted in state \\"finalizing\\""}}}}
2023-10-23 21:46:17.778133+00 | e65dab54-4f10-4e3e-a994-27b9ca0b7840 | disk-delete | undo_started | NULL
2023-10-23 21:46:17.780272+00 | e65dab54-4f10-4e3e-a994-27b9ca0b7840 | disk-delete | undo_finished | NULL
2023-10-23 21:46:17.934858+00 | 7c1a3623-0a64-4f43-a7d4-ace7ed0e4884 | finalize-disk | succeeded | {"[fd00:1122:3344:109::a]:19032": "[fd00:1122:3344:109::a]:19033", "[fd00:1122:3344:10a::9]:19017": "[fd00:1122:3344:10a::9]:19018", "[fd00:1122:3344:10b::8]:19030": "[fd00:1122:3344:10b::8]:19031"}
2023-10-23 21:46:17.937487+00 | 7c1a3623-0a64-4f43-a7d4-ace7ed0e4884 | finalize-disk | started | NULL
2023-10-23 21:46:17.986857+00 | 7c1a3623-0a64-4f43-a7d4-ace7ed0e4884 | finalize-disk | succeeded | null
2023-10-23 21:46:17.988642+00 | 7c1a3623-0a64-4f43-a7d4-ace7ed0e4884 | finalize-disk | started | NULL
2023-10-23 21:46:18.08896+00 | 7c1a3623-0a64-4f43-a7d4-ace7ed0e4884 | finalize-disk | succeeded | {"block_size": "AdvancedFormat", "destination_volume_id": "155afa02-7a3b-4325-9aef-b6077a225126", "disk_id": "f1da1901-4c50-433b-b5b0-3cae4e8885b4", "gen": 2, "identity": {"description": "snapshot of finalized disk f1da1901-4c50-433b-b5b0-3cae4e8885b4", "id": "b5d88ebe-007d-41ee-b672-5387efe5439f", "name": "tmp-snapshot-44499152", "time_created": "2023-10-23T21:46:17.079896Z", "time_deleted": null, "time_modified": "2023-10-23T21:46:17.079896Z"}, "project_id": "5e49b6de-cb2d-438d-83af-95c415bbb901", "size": 1073741824, "state": "Ready", "volume_id": "c1f48bdf-1545-4088-96ea-c860c34a327f"}
2023-10-23 21:46:18.091661+00 | 7c1a3623-0a64-4f43-a7d4-ace7ed0e4884 | finalize-disk | started | NULL
2023-10-23 21:46:18.151561+00 | 7c1a3623-0a64-4f43-a7d4-ace7ed0e4884 | finalize-disk | succeeded | null
2023-10-23 21:46:18.154115+00 | 7c1a3623-0a64-4f43-a7d4-ace7ed0e4884 | finalize-disk | started | NULL
2023-10-23 21:46:18.156036+00 | 7c1a3623-0a64-4f43-a7d4-ace7ed0e4884 | finalize-disk | succeeded | null
2023-10-23 21:46:18.158301+00 | 7c1a3623-0a64-4f43-a7d4-ace7ed0e4884 | finalize-disk | started | NULL
2023-10-23 21:46:18.236444+00 | 7c1a3623-0a64-4f43-a7d4-ace7ed0e4884 | finalize-disk | succeeded | null
2023-10-23 21:46:18.238711+00 | 7c1a3623-0a64-4f43-a7d4-ace7ed0e4884 | finalize-disk | started | NULL
2023-10-23 21:46:18.288273+00 | 7c1a3623-0a64-4f43-a7d4-ace7ed0e4884 | finalize-disk | succeeded | null
2023-10-23 21:46:18.290238+00 | 7c1a3623-0a64-4f43-a7d4-ace7ed0e4884 | finalize-disk | started | NULL
2023-10-23 21:46:18.338292+00 | 7c1a3623-0a64-4f43-a7d4-ace7ed0e4884 | finalize-disk | succeeded | null
2023-10-23 21:46:18.340397+00 | 7c1a3623-0a64-4f43-a7d4-ace7ed0e4884 | finalize-disk | started | NULL
2023-10-23 21:46:18.342531+00 | 7c1a3623-0a64-4f43-a7d4-ace7ed0e4884 | finalize-disk | succeeded | null
(64 rows)
These are the saga dag details that correlate them to the same disk (id f1da1901-4c50-433b-b5b0-3cae4e8885b4) being processed:
root@[fd00:1122:3344:105::3]:32221/omicron> select time_created, id, saga_dag from saga where id in ('e65dab54-4f10-4e3e-a994-27b9ca0b7840','7c1a3623-0a64-4f43-a7d4-ace7ed0e4884') order by 1;
time_created | id | saga_dag
--------------------------------+--------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
2023-10-23 21:46:16.103895+00 | 7c1a3623-0a64-4f43-a7d4-ace7ed0e4884 | {"end_node": 28, "graph": {"edge_property": "directed", "edges": [[0, 1, null], [1, 2, null], [2, 3, null], [4, 5, null], [5, 6, null], [6, 7, null], [7, 8, null], [8, 9, null], [9, 10, null], [10, 11, null], [11, 12, null], [12, 13, null], [13, 14, null], [14, 15, null], [15, 16, null], [16, 17, null], [17, 18, null], [18, 19, null], [19, 20, null], [20, 21, null], [21, 22, null], [3, 4, null], [22, 23, null], [23, 24, null], [24, 25, null], [25, 26, null], [27, 0, null], [26, 28, null]], "node_holes": [], "nodes": [{"Action": {"action_name": "finalize_disk.set_finalizing_state", "label": "SetFinalizingState", "name": "disk_generation_number"}}, {"Action": {"action_name": "finalize_disk.get_pantry_address", "label": "GetPantryAddress", "name": "pantry_address"}}, {"Constant": {"name": "params_for_snapshot_subsaga", "value": {"attached_instance_and_sled": null, "create_params": {"description": "snapshot of finalized disk f1da1901-4c50-433b-b5b0-3cae4e8885b4", "disk": "f1da1901-4c50-433b-b5b0-3cae4e8885b4", "name": "tmp-snapshot-44499152"}, "disk_id": "f1da1901-4c50-433b-b5b0-3cae4e8885b4", "project_id": "5e49b6de-cb2d-438d-83af-95c415bbb901", "serialized_authn": {"kind": {"Authenticated": [{"actor": {"SiloUser": {"silo_id": "7bd7623a-68ed-4636-8ecb-b59e3b068787", "silo_user_id": "906e74cb-eab8-4a87-bda8-2cb0914bf853"}}}, {"mapped_fleet_roles": {"admin": ["admin"]}}]}}, "silo_id": "7bd7623a-68ed-4636-8ecb-b59e3b068787"}}}, {"SubsagaStart": {"params_node_name": "params_for_snapshot_subsaga", "saga_name": "snapshot-create"}}, {"Action": {"action_name": "common.uuid_generate", "label": "GenerateSnapshotId", "name": "snapshot_id"}}, {"Action": {"action_name": "common.uuid_generate", "label": "GenerateVolumeId", "name": "volume_id"}}, {"Action": {"action_name": "common.uuid_generate", "label": "GenerateDestinationVolumeId", "name": "destination_volume_id"}}, {"Action": {"action_name": "snapshot_create.regions_alloc", "label": "RegionsAlloc", "name": "datasets_and_regions"}}, {"Action": {"action_name": "snapshot_create.regions_ensure_undo", "label": "RegionsEnsureUndo", "name": "regions_ensure_undo"}}, {"Action": {"action_name": "snapshot_create.regions_ensure", "label": "RegionsEnsure", "name": "regions_ensure"}}, {"Action": {"action_name": "snapshot_create.create_destination_volume_record", "label": "CreateDestinationVolumeRecord", "name": "created_destination_volume"}}, {"Action": {"action_name": "snapshot_create.create_snapshot_record", "label": "CreateSnapshotRecord", "name": "created_snapshot"}}, {"Action": {"action_name": "snapshot_create.space_account", "label": "SpaceAccount", "name": "no_result"}}, {"Action": {"action_name": "snapshot_create.get_pantry_address", "label": "GetPantryAddress", "name": "pantry_address"}}, {"Action": {"action_name": "snapshot_create.attach_disk_to_pantry", "label": "AttachDiskToPantry", "name": "disk_generation_number"}}, {"Action": {"action_name": "snapshot_create.call_pantry_attach_for_disk", "label": "CallPantryAttachForDisk", "name": "call_pantry_attach_for_disk"}}, {"Action": {"action_name": "snapshot_create.call_pantry_snapshot_for_disk", "label": "CallPantrySnapshotForDisk", "name": "call_pantry_snapshot_for_disk"}}, {"Action": {"action_name": "snapshot_create.call_pantry_detach_for_disk", "label": "CallPantryDetachForDisk", "name": "call_pantry_detach_for_disk"}}, {"Action": {"action_name": "snapshot_create.start_running_snapshot_undo", "label": "StartRunningSnapshotUndo", "name": "ssc_not_used"}}, {"Action": {"action_name": "snapshot_create.start_running_snapshot", "label": "StartRunningSnapshot", "name": "replace_sockets_map"}}, {"Action": {"action_name": "snapshot_create.create_volume_record", "label": "CreateVolumeRecord", "name": "created_volume"}}, {"Action": {"action_name": "snapshot_create.finalize_snapshot_record", "label": "FinalizeSnapshotRecord", "name": "finalized_snapshot"}}, {"Action": {"action_name": "snapshot_create.detach_disk_from_pantry", "label": "DetachDiskFromPantry", "name": "detach_disk_from_pantry"}}, {"SubsagaEnd": {"name": "snapshot_subsaga_no_result"}}, {"Action": {"action_name": "finalize_disk.call_pantry_detach_for_disk", "label": "CallPantryDetachForDisk", "name": "call_pantry_detach_for_disk"}}, {"Action": {"action_name": "finalize_disk.clear_pantry_address", "label": "ClearPantryAddress", "name": "clear_pantry_address"}}, {"Action": {"action_name": "finalize_disk.set_detached_state", "label": "SetDetachedState", "name": "set_detached_state"}}, {"Start": {"params": {"disk_id": "f1da1901-4c50-433b-b5b0-3cae4e8885b4", "project_id": "5e49b6de-cb2d-438d-83af-95c415bbb901", "serialized_authn": {"kind": {"Authenticated": [{"actor": {"SiloUser": {"silo_id": "7bd7623a-68ed-4636-8ecb-b59e3b068787", "silo_user_id": "906e74cb-eab8-4a87-bda8-2cb0914bf853"}}}, {"mapped_fleet_roles": {"admin": ["admin"]}}]}}, "silo_id": "7bd7623a-68ed-4636-8ecb-b59e3b068787", "snapshot_name": "tmp-snapshot-44499152"}}}, "End"]}, "saga_name": "finalize-disk", "start_node": 27}
2023-10-23 21:46:17.755525+00 | e65dab54-4f10-4e3e-a994-27b9ca0b7840 | {"end_node": 13, "graph": {"edge_property": "directed", "edges": [[0, 1, null], [1, 2, null], [2, 3, null], [4, 5, null], [4, 6, null], [6, 7, null], [5, 7, null], [7, 8, null], [8, 9, null], [9, 10, null], [3, 4, null], [10, 11, null], [12, 0, null], [11, 13, null]], "node_holes": [], "nodes": [{"Action": {"action_name": "disk_delete.delete_disk_record", "label": "DeleteDiskRecord", "name": "deleted_disk"}}, {"Action": {"action_name": "disk_delete.space_account", "label": "SpaceAccount", "name": "no_result1"}}, {"Constant": {"name": "params_for_volume_delete_subsaga", "value": {"serialized_authn": {"kind": {"Authenticated": [{"actor": {"SiloUser": {"silo_id": "7bd7623a-68ed-4636-8ecb-b59e3b068787", "silo_user_id": "906e74cb-eab8-4a87-bda8-2cb0914bf853"}}}, {"mapped_fleet_roles": {"admin": ["admin"]}}]}}, "volume_id": "2d766192-1445-469a-bfbb-a665ebf9eca7"}}}, {"SubsagaStart": {"params_node_name": "params_for_volume_delete_subsaga", "saga_name": "volume-delete"}}, {"Action": {"action_name": "volume_delete.decrease_crucible_resource_count", "label": "DecreaseCrucibleResourceCount", "name": "crucible_resources_to_delete"}}, {"Action": {"action_name": "volume_delete.delete_crucible_regions", "label": "DeleteCrucibleRegions", "name": "no_result_1"}}, {"Action": {"action_name": "volume_delete.delete_crucible_running_snapshots", "label": "DeleteCrucibleRunningSnapshots", "name": "no_result_2"}}, {"Action": {"action_name": "volume_delete.delete_crucible_snapshots", "label": "DeleteCrucibleSnapshots", "name": "no_result_3"}}, {"Action": {"action_name": "volume_delete.delete_crucible_snapshot_records", "label": "DeleteCrucibleSnapshotRecords", "name": "no_result_4"}}, {"Action": {"action_name": "volume_delete.delete_freed_crucible_regions", "label": "DeleteFreedCrucibleRegions", "name": "no_result_5"}}, {"Action": {"action_name": "volume_delete.hard_delete_volume_record", "label": "HardDeleteVolumeRecord", "name": "final_no_result"}}, {"SubsagaEnd": {"name": "volume_delete_subsaga_no_result"}}, {"Start": {"params": {"disk_id": "f1da1901-4c50-433b-b5b0-3cae4e8885b4", "project_id": "5e49b6de-cb2d-438d-83af-95c415bbb901", "serialized_authn": {"kind": {"Authenticated": [{"actor": {"SiloUser": {"silo_id": "7bd7623a-68ed-4636-8ecb-b59e3b068787", "silo_user_id": "906e74cb-eab8-4a87-bda8-2cb0914bf853"}}}, {"mapped_fleet_roles": {"admin": ["admin"]}}]}}, "volume_id": "2d766192-1445-469a-bfbb-a665ebf9eca7"}}}, "End"]}, "saga_name": "disk-delete", "start_node": 12}
(2 rows)
Nexus log file during the issue: catacomb:/data/staff/dogfood/omicron-4321-nexus.log