forked from Netflix/conductor
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Why: We need idempotent forked tasks, meaning all tasks get executed, but any failures are still detected upon join. Feature request Netflix#3861 What: Introduced the concept of Permissive tasks. A Permissive task is similar to a Simple task. The difference is, it permits the other tasks to continue - in case a Permissive task failed. Result is: 1. Forked Permissive tasks will let each other be evaluated, until all the forked tasks had terminated. Only then, the join task should fail. In case of Permissive optional tasks, the join will not fail. 2. Permissive sequential tasks will let subsequent tasks continue. While at the end, the workflow will fail in case a permissive task had failed. The workflow would not fail in case of Permissive optional task failure. Testing done: PermissiveTaskMapperTest added, TestDeciderOutcomes.testPermissive() added, WorkflowAndTaskConfigurationSpec "Test simple workflow which has a permissive task" and "Test simple workflow which has a permissive optional task added" that cover retry, ForkJoinSpec "Test a simple workflow with fork join permissive failure flow" added. In addition, performed e2e tests locally running a Conductor instance. Did build a docker image with the code changes made, started it locally, and started a SampleWorker to poll 3 tasks in parallel. Verified e2e scenarios of task_def_permissive, task_def_permissive_optional, task_def_simple.json, task_def_simple_optional.json, each joining on 6 forked tasks, then running simple task 7 after join.
- Loading branch information
Showing
14 changed files
with
930 additions
and
15 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
102 changes: 102 additions & 0 deletions
102
core/src/main/java/com/netflix/conductor/core/execution/mapper/PermissiveTaskMapper.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,102 @@ | ||
| /* | ||
| * Copyright 2023 Netflix, Inc. | ||
| * <p> | ||
| * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with | ||
| * the License. You may obtain a copy of the License at | ||
| * <p> | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * <p> | ||
| * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on | ||
| * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the | ||
| * specific language governing permissions and limitations under the License. | ||
| */ | ||
| package com.netflix.conductor.core.execution.mapper; | ||
|
|
||
| import java.util.List; | ||
| import java.util.Map; | ||
| import java.util.Optional; | ||
|
|
||
| import org.slf4j.Logger; | ||
| import org.slf4j.LoggerFactory; | ||
| import org.springframework.stereotype.Component; | ||
|
|
||
| import com.netflix.conductor.common.metadata.tasks.TaskDef; | ||
| import com.netflix.conductor.common.metadata.tasks.TaskType; | ||
| import com.netflix.conductor.common.metadata.workflow.WorkflowDef; | ||
| import com.netflix.conductor.common.metadata.workflow.WorkflowTask; | ||
| import com.netflix.conductor.core.exception.TerminateWorkflowException; | ||
| import com.netflix.conductor.core.utils.ParametersUtils; | ||
| import com.netflix.conductor.model.TaskModel; | ||
| import com.netflix.conductor.model.WorkflowModel; | ||
|
|
||
| /** | ||
| * An implementation of {@link TaskMapper} to map a {@link WorkflowTask} of type {@link | ||
| * TaskType#PERMISSIVE} to a {@link TaskModel} with status {@link TaskModel.Status#SCHEDULED}. | ||
| */ | ||
| @Component | ||
| public class PermissiveTaskMapper implements TaskMapper { | ||
|
|
||
| public static final Logger LOGGER = LoggerFactory.getLogger(PermissiveTaskMapper.class); | ||
| private final ParametersUtils parametersUtils; | ||
|
|
||
| public PermissiveTaskMapper(ParametersUtils parametersUtils) { | ||
| this.parametersUtils = parametersUtils; | ||
| } | ||
|
|
||
| @Override | ||
| public String getTaskType() { | ||
| return TaskType.PERMISSIVE.name(); | ||
| } | ||
|
|
||
| /** | ||
| * This method maps a {@link WorkflowTask} of type {@link TaskType#PERMISSIVE} to a {@link | ||
| * TaskModel} | ||
| * | ||
| * @param taskMapperContext: A wrapper class containing the {@link WorkflowTask}, {@link | ||
| * WorkflowDef}, {@link WorkflowModel} and a string representation of the TaskId | ||
| * @return a List with just one exclusive task | ||
| * @throws TerminateWorkflowException In case if the task definition does not exist | ||
| */ | ||
| @Override | ||
| public List<TaskModel> getMappedTasks(TaskMapperContext taskMapperContext) | ||
| throws TerminateWorkflowException { | ||
|
|
||
| LOGGER.debug("TaskMapperContext {} in PermissiveTaskMapper", taskMapperContext); | ||
|
|
||
| WorkflowTask workflowTask = taskMapperContext.getWorkflowTask(); | ||
| WorkflowModel workflowModel = taskMapperContext.getWorkflowModel(); | ||
| int retryCount = taskMapperContext.getRetryCount(); | ||
| String retriedTaskId = taskMapperContext.getRetryTaskId(); | ||
|
|
||
| TaskDef taskDefinition = | ||
| Optional.ofNullable(workflowTask.getTaskDefinition()) | ||
| .orElseThrow( | ||
| () -> { | ||
| String reason = | ||
| String.format( | ||
| "Invalid task. Task %s does not have a definition", | ||
| workflowTask.getName()); | ||
| return new TerminateWorkflowException(reason); | ||
| }); | ||
|
|
||
| Map<String, Object> input = | ||
| parametersUtils.getTaskInput( | ||
| workflowTask.getInputParameters(), | ||
| workflowModel, | ||
| taskDefinition, | ||
| taskMapperContext.getTaskId()); | ||
| TaskModel permissiveTask = taskMapperContext.createTaskModel(); | ||
| permissiveTask.setTaskType(workflowTask.getName()); | ||
| permissiveTask.setStartDelayInSeconds(workflowTask.getStartDelay()); | ||
| permissiveTask.setInputData(input); | ||
| permissiveTask.setStatus(TaskModel.Status.SCHEDULED); | ||
| permissiveTask.setRetryCount(retryCount); | ||
| permissiveTask.setCallbackAfterSeconds(workflowTask.getStartDelay()); | ||
| permissiveTask.setResponseTimeoutSeconds(taskDefinition.getResponseTimeoutSeconds()); | ||
| permissiveTask.setRetriedTaskId(retriedTaskId); | ||
| permissiveTask.setRateLimitPerFrequency(taskDefinition.getRateLimitPerFrequency()); | ||
| permissiveTask.setRateLimitFrequencyInSeconds( | ||
| taskDefinition.getRateLimitFrequencyInSeconds()); | ||
| return List.of(permissiveTask); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.