feat: let a relaunch from failed force failed nodes as successful

ascender/api/serializers/base.py

@@ -92,7 +92,7 @@
     'signature_validation_credential': DEFAULT_SUMMARY_FIELDS + ('kind', 'credential_type_id'),
     'job': DEFAULT_SUMMARY_FIELDS + ('status', 'failed', 'started', 'elapsed', 'type', 'canceled_on'),
     'job_template': DEFAULT_SUMMARY_FIELDS + ('prevent_relaunch',),
-    'workflow_job_template': DEFAULT_SUMMARY_FIELDS,
+    'workflow_job_template': DEFAULT_SUMMARY_FIELDS + ('allow_force_node_success_on_relaunch',),
     'workflow_job': DEFAULT_SUMMARY_FIELDS,
     'workflow_approval_template': DEFAULT_SUMMARY_FIELDS + ('timeout',),
     'workflow_approval': DEFAULT_SUMMARY_FIELDS + ('timeout', 'status'),
@@ -112,6 +112,8 @@
     'target_credential': DEFAULT_SUMMARY_FIELDS + ('kind', 'cloud', 'credential_type_id'),
     'webhook_credential': DEFAULT_SUMMARY_FIELDS + ('kind', 'cloud', 'credential_type_id'),
     'approved_or_denied_by': ('id', 'username', 'first_name', 'last_name'),
+    'forced_success_by': ('id', 'username', 'first_name', 'last_name'),
+    'forced_success_job': DEFAULT_SUMMARY_FIELDS + ('status', 'failed', 'elapsed', 'type'),
     'credential_type': DEFAULT_SUMMARY_FIELDS,
 }
 

ascender/api/serializers/workflow.py

@@ -66,6 +66,7 @@ class Meta:
             'survey_enabled',
             'allow_simultaneous',
             'allow_overwrite_flow_vars_on_relaunch',
+            'allow_force_node_success_on_relaunch',
             'ask_variables_on_launch',
             'inventory',
             'limit',
@@ -162,6 +163,7 @@ class Meta:
             'extra_vars',
             'allow_simultaneous',
             'allow_overwrite_flow_vars_on_relaunch',
+            'allow_force_node_success_on_relaunch',
             'job_template',
             'is_sliced_job',
             '-execution_environment',
@@ -450,6 +452,10 @@ class Meta:
             'do_not_run',
             'prior_run_succeeded',
             'prior_run_elapsed',
+            'forced_success',
+            'forced_success_reason',
+            'forced_success_by',
+            'forced_success_job',
             'max_retries',
             'retry_attempts',
             'retried_jobs',

ascender/api/templates/api/workflow_job_relaunch.md

@@ -2,9 +2,11 @@ Relaunch a workflow job:
 
 Make a POST request to this endpoint to launch a workflow job identical to the parent workflow job. This will spawn jobs, project updates, or inventory updates based on the unified job templates referenced in the workflow nodes in the workflow job.
 
-Two optional parameters are accepted:
+These optional parameters are accepted:
 
 * `nodes`: set it to `failed` to run the nodes that did not succeed and everything downstream of them, carrying the nodes that already succeeded forward instead of running them again.
 * `extra_vars`: variables that overwrite the ones the original run used. They are only accepted together with `nodes=failed`, and only when this workflow job has `allow_overwrite_flow_vars_on_relaunch` set. That field is copied onto the job from its workflow job template when the job is launched, so it reflects what the template said at launch time rather than what it says now: enabling it on the template makes the runs started after that eligible, and disabling it leaves the runs already started eligible. Answers to password questions of the survey may be left as `$encrypted$` to keep the value the original run used; anywhere else `$encrypted$` is rejected as a reserved word.
+* `force_success_nodes`: ids of workflow job nodes of this run whose job ended as `failed`, `error` or `canceled` with no retry pending, to carry forward as though they had succeeded. They are not run again and the workflow goes on down their success paths. Only accepted together with `nodes=failed`, and only when the workflow job template has `allow_force_node_success_on_relaunch` set. Unlike the variables above, that setting is read from the template when relaunching, so turning it off also stops it for runs already started; the run's own copy only counts once its template has been deleted. Approval nodes can never be forced. Whatever the forced job published with `set_stats` before failing is handed down as a successful job's would be.
+* `force_success_reason`: why the nodes are forced. Required together with `force_success_nodes`. The forced nodes of the new run keep it, together with who forced them and the job that failed, in `forced_success_reason`, `forced_success_by` and `forced_success_job`, and have `forced_success` set, so they can be told apart from nodes that succeeded. They stay marked that way if the run is relaunched again.
 
 If successful, the response status code will be 201 and serialized data of the new workflow job will be returned.

ascender/api/views/workflow.py

@@ -450,10 +450,44 @@ def _relaunch_extra_vars(self, request, obj, from_failed):
             raise ParseError({'variables_needed_to_start': errors})
         return extra_vars
 
+    def _forced_success_nodes(self, request, obj, from_failed):
+        """Read which failed nodes the relaunch wants carried forward as though
+        they had succeeded, and why.
+
+        Returns a mapping of node id to (user, reason), as the model expects it.
+        Forcing is off unless the workflow job template turned it on, it only
+        applies to a relaunch from failed nodes, and it always needs a reason:
+        the forced node keeps it, together with who forced it, so the next
+        person looking at the run can tell a skipped failure from a success."""
+        node_ids = request.data.get('force_success_nodes')
+        if node_ids in (None, []):
+            if request.data.get('force_success_reason'):
+                raise ParseError(_('force_success_reason was given without any force_success_nodes.'))
+            return None
+        if not from_failed:
+            raise ParseError(_('Nodes can only be forced as successful when relaunching from failed nodes.'))
+        if not obj.allows_forcing_node_success():
+            raise ParseError(
+                _('This workflow job does not allow forcing nodes as successful. Enable allow_force_node_success_on_relaunch on its workflow job template.')
+            )
+        if not isinstance(node_ids, list) or not all(isinstance(i, int) and not isinstance(i, bool) for i in node_ids):
+            raise ParseError(_('force_success_nodes must be a list of workflow job node ids.'))
+        node_ids = list(dict.fromkeys(node_ids))
+        reason = request.data.get('force_success_reason')
+        if not isinstance(reason, str) or not reason.strip():
+            raise ParseError(_('A force_success_reason is required to force nodes as successful.'))
+        try:
+            nodes = obj.forceable_failed_nodes(node_ids)
+        except ValueError as e:
+            raise ParseError(str(e))
+        reason = reason.strip()
+        return {node.id: (request.user, reason) for node in nodes}
+
     def post(self, request, *args, **kwargs):
         obj = self.get_object()
         from_failed = request.data.get('nodes') == 'failed'
         extra_vars = self._relaunch_extra_vars(request, obj, from_failed)
+        forced = self._forced_success_nodes(request, obj, from_failed)
         if obj.is_sliced_job:
             if from_failed:
                 raise ParseError(_('Cannot relaunch a sliced workflow job from failed nodes.'))
@@ -474,7 +508,7 @@ def post(self, request, *args, **kwargs):
                         _('Cannot relaunch from failed nodes: this workflow failed because a node has no job template, which relaunching cannot recover.')
                     )
                 raise ParseError(_('This workflow job has no failed nodes to relaunch from.'))
-        new_workflow_job = obj.create_relaunch_workflow_job(from_failed=from_failed, extra_vars=extra_vars)
+        new_workflow_job = obj.create_relaunch_workflow_job(from_failed=from_failed, extra_vars=extra_vars, forced=forced)
         new_workflow_job.signal_start()
 
         data = serializers.WorkflowJobSerializer(new_workflow_job, context=self.get_serializer_context()).data

ascender/main/migrations/0226_workflow_force_node_success.py

@@ -0,0 +1,76 @@
+import django.db.models.deletion
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("main", "0225_jobtemplate_prevent_relaunch"),
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="workflowjob",
+            name="allow_force_node_success_on_relaunch",
+            field=models.BooleanField(
+                default=False,
+                help_text="Allow a relaunch from failed nodes to carry chosen failed nodes forward as though they had succeeded, so the workflow continues down their success paths. Each forced node records who forced it and why.",
+            ),
+        ),
+        migrations.AddField(
+            model_name="workflowjobnode",
+            name="forced_success",
+            field=models.BooleanField(
+                default=False,
+                editable=False,
+                help_text="Set when a workflow is relaunched from a failed node and this node was forced as successful: its job failed in the prior run, but whoever relaunched it chose to carry it forward as though it had succeeded. Such a node also has prior_run_succeeded set.",
+            ),
+        ),
+        migrations.AddField(
+            model_name="workflowjobnode",
+            name="forced_success_by",
+            field=models.ForeignKey(
+                blank=True,
+                default=None,
+                editable=False,
+                help_text="The user who forced the node as successful.",
+                null=True,
+                on_delete=django.db.models.deletion.SET_NULL,
+                related_name="+",
+                to=settings.AUTH_USER_MODEL,
+            ),
+        ),
+        migrations.AddField(
+            model_name="workflowjobnode",
+            name="forced_success_job",
+            field=models.ForeignKey(
+                blank=True,
+                default=None,
+                editable=False,
+                help_text="The failed job whose outcome was overridden when the node was forced as successful.",
+                null=True,
+                on_delete=django.db.models.deletion.SET_NULL,
+                related_name="+",
+                to="main.unifiedjob",
+            ),
+        ),
+        migrations.AddField(
+            model_name="workflowjobnode",
+            name="forced_success_reason",
+            field=models.TextField(
+                blank=True,
+                default="",
+                editable=False,
+                help_text="Why the node was forced as successful, as given by whoever relaunched the workflow.",
+            ),
+        ),
+        migrations.AddField(
+            model_name="workflowjobtemplate",
+            name="allow_force_node_success_on_relaunch",
+            field=models.BooleanField(
+                default=False,
+                help_text="Allow a relaunch from failed nodes to carry chosen failed nodes forward as though they had succeeded, so the workflow continues down their success paths. Each forced node records who forced it and why.",
+            ),
+        ),
+    ]