Implemented resume handling #390

sechub-administration/src/main/java/com/mercedesbenz/sechub/domain/administration/scheduler/SchedulerStatusEntryKeys.java

@@ -18,6 +18,8 @@ public enum SchedulerStatusEntryKeys implements StatusEntryKey {
 
     SCHEDULER_JOBS_CANCEL_REQUESTED("status.scheduler.jobs.cancel_requested"),
 
+    SCHEDULER_JOBS_RESUMING("status.scheduler.jobs.resuming"),
+
     SCHEDULER_JOBS_SUSPENDED("status.scheduler.jobs.suspended"),
 
     SCHEDULER_JOBS_ENDED("status.scheduler.jobs.ended"),

sechub-commons-model/src/main/java/com/mercedesbenz/sechub/commons/model/job/ExecutionState.java

@@ -27,7 +27,11 @@ public enum ExecutionState {
 
     SUSPENDED("The job has been suspended and can be resumed by another SecHub instance"),
 
-    ENDED("Has ended - with failure or success");
+    RESUMING("A former suspended job is resuming"),
+
+    ENDED("Has ended - with failure or success"),
+
+    ;
 
     private String description;
 

sechub-doc/src/main/java/com/mercedesbenz/sechub/docgen/util/DocGeneratorUtil.java

@@ -10,6 +10,7 @@
 
 import com.mercedesbenz.sechub.docgen.DocAnnotationData;
 import com.mercedesbenz.sechub.pds.PDSMustBeDocumented;
+import com.mercedesbenz.sechub.sharedkernel.DocumentationScopeConstants;
 import com.mercedesbenz.sechub.sharedkernel.MustBeDocumented;
 
 public class DocGeneratorUtil {
@@ -27,7 +28,7 @@ public static DocAnnotationData buildDataForMustBeDocumented(MustBeDocumented in
         buildSpringScheduledParts(data, element);
 
         /* when class name shall be used... */
-        if (MustBeDocumented.SCOPE_USE_DEFINED_CLASSNAME_LOWERCASED.equals(data.scope)) {
+        if (DocumentationScopeConstants.SCOPE_USE_DEFINED_CLASSNAME_LOWERCASED.equals(data.scope)) {
             data.scope = toCamelOne(fetchClass(element)).toLowerCase();
         }
         return data;
@@ -42,7 +43,7 @@ public static DocAnnotationData buildDataForPDSMustBeDocumented(PDSMustBeDocumen
         buildSpringScheduledParts(data, element);
 
         /* when class name shall be used... */
-        if (MustBeDocumented.SCOPE_USE_DEFINED_CLASSNAME_LOWERCASED.equals(data.scope)) {
+        if (DocumentationScopeConstants.SCOPE_USE_DEFINED_CLASSNAME_LOWERCASED.equals(data.scope)) {
             data.scope = toCamelOne(fetchClass(element)).toLowerCase();
         }
         return data;

sechub-integrationtest/src/main/java/com/mercedesbenz/sechub/integrationtest/api/AsUser.java

@@ -36,9 +36,11 @@
 import com.mercedesbenz.sechub.commons.mapping.MappingData;
 import com.mercedesbenz.sechub.commons.model.JSONConverter;
 import com.mercedesbenz.sechub.commons.model.SecHubConfigurationModel;
+import com.mercedesbenz.sechub.commons.model.TrafficLight;
 import com.mercedesbenz.sechub.domain.scan.project.FalsePositiveProjectData;
 import com.mercedesbenz.sechub.integrationtest.JSONTestSupport;
 import com.mercedesbenz.sechub.integrationtest.internal.IntegrationTestContext;
+import com.mercedesbenz.sechub.integrationtest.internal.IntegrationTestExampleConstants;
 import com.mercedesbenz.sechub.integrationtest.internal.IntegrationTestFileSupport;
 import com.mercedesbenz.sechub.integrationtest.internal.IntegrationTestTemplateFile;
 import com.mercedesbenz.sechub.integrationtest.internal.SecHubClientExecutor.ExecutionResult;
@@ -965,12 +967,52 @@ public ProjectFalsePositivesDefinition startFalsePositiveDefinition(TestProject
         return new ProjectFalsePositivesDefinition(project);
     }
 
-    public UUID triggerAsyncPDSCodeScanWithDifferentDataSections(TestProject project) {
+    /**
+     * This triggers a PDS code scan in asynchronous way. It will use
+     * {@link IntegrationTestExampleConstants#PATH_TO_ZIPFILE_WITH_DIFFERENT_DATA_SECTIONS}.
+     * It will map wanted traffic light to corresponding reference ids: <br>
+     * <table border='1px'>
+     * <tr>
+     * <th>Traffic light</th>
+     * <th>used reference-id</th>
+     * </tr>
+     * <tr>
+     * <td>GREEN</td>
+     * <td>low-id</td>
+     * </tr>
+     * <tr>
+     * <td>YELLOW</td>
+     * <td>medium-id</td>
+     * </tr>
+     * <tr>
+     * <td>RED</td>
+     * <td>critical-id</td>
+     * </tr>
+     * </table>
+     * Please look at
+     * {@link IntegrationTestExampleConstants#PATH_TO_ZIPFILE_WITH_DIFFERENT_DATA_SECTIONS}
+     * for the finding details.
+     *
+     * @param project            - test project
+     * @param wantedTrafficLight - which kind of report is wanted
+     * @return uuid of SecHub job
+     */
+    public UUID triggerAsyncPDSCodeScanWithWantedTrafficLightResult(TestProject project, TrafficLight wantedTrafficLight) {
         /* @formatter:off */
+        String referenceId = switch (wantedTrafficLight) {
+        case GREEN -> "low-id";
+        case YELLOW -> "medium-id";
+        case RED -> "critical-id";
+        default -> {
+            throw new IllegalArgumentException("Traffic light type: "+wantedTrafficLight+" is not supported by this test helper method!");
+        }
+    };
+
+
         UUID jobUUID = createCodeScanWithTemplate(
                 IntegrationTestTemplateFile.CODE_SCAN_3_SOURCES_DATA_ONE_REFERENCE,
                 project, NOT_MOCKED,
-                TemplateData.builder().addReferenceId("files-a").build());
+                TemplateData.builder().addReferenceId(referenceId).build());
 
         uploadSourcecode(project, jobUUID, PATH_TO_ZIPFILE_WITH_DIFFERENT_DATA_SECTIONS).
         approveJob(project, jobUUID);

sechub-integrationtest/src/main/java/com/mercedesbenz/sechub/integrationtest/api/IntegrationTestSetup.java

@@ -30,7 +30,8 @@
 import ch.qos.logback.classic.Level;
 
 public class IntegrationTestSetup implements TestRule {
-    // please keep this private static field at top - reason: initialize setup support as soon as possible...
+    // please keep this private static field at top - reason: initialize setup
+    // support as soon as possible...
     private static final LocalDeveloperFileSetupSupport support = LocalDeveloperFileSetupSupport.INSTANCE;
     public static final String SECHUB_INTEGRATIONTEST_ONLY_NECESSARY_TESTS_FOR_DOCUMENTATION = "sechub.integrationtest.only.necessary4documentation";
     public static final String SECHUB_INTEGRATIONTEST_NEVER_NECESSARY_TESTS_FOR_DOCUMENTATION = "sechub.integrationtest.never.necessary4documentation";
@@ -166,8 +167,7 @@ public void evaluate() throws Throwable {
                     Assume.assumeTrue(message, false);
                 }
             } else {
-                integrationTestEnabled = support.isAlwaysSecHubIntegrationTestRunning()
-                        || Boolean.getBoolean(SECHUB_INTEGRATIONTEST_RUNNING);
+                integrationTestEnabled = support.isAlwaysSecHubIntegrationTestRunning() || Boolean.getBoolean(SECHUB_INTEGRATIONTEST_RUNNING);
                 if (!integrationTestEnabled) {
                     String message = "Skipped test scenario '" + scenario.getName() + "'\nReason: not in integration test mode.\nDefine -D"
                             + SECHUB_INTEGRATIONTEST_RUNNING + "=true to enable integration tests!";

sechub-integrationtest/src/main/java/com/mercedesbenz/sechub/integrationtest/api/TestAPI.java

@@ -357,7 +357,7 @@ public boolean runAndReturnTrueWhenSuccesfulImpl() throws Exception {
 
     public static TestSecHubJobStatus getSecHubJobStatus(TestProject project, UUID jobUUID, TestUser asUser) {
         String status = as(asUser).getJobStatus(project.getProjectId(), jobUUID);
-        LOG.info(">>>>>>>>>JOB:STATUS:" + status);
+        LOG.info(" => Job status: {}", status);
         TestSecHubJobStatus jobStatus = TestSecHubJobStatus.fromJSON(status);
         return jobStatus;
     }
@@ -443,12 +443,10 @@ public static void waitForJobRunning(TestProject project, int timeOutInSeconds,
         LOG.info("wait for job running project:{}, job:{}, timeToWaitInMillis{}, timeOutInSeconds:{}", project.getProjectId(), jobUUID, timeToWaitInMillis,
                 timeOutInSeconds);
 
-        executeUntilSuccessOrTimeout(new AbstractTestExecutable(SUPER_ADMIN, timeOutInSeconds, timeToWaitInMillis, HttpClientErrorException.class) {
+        executeUntilSuccessOrTimeout(new AbstractTestExecutable(SUPER_ADMIN, timeOutInSeconds, HttpClientErrorException.class) {
             @Override
             public boolean runAndReturnTrueWhenSuccesfulImpl() throws Exception {
-                String status = as(getUser()).getJobStatus(project.getProjectId(), jobUUID);
-                LOG.info(">>>>>>>>>JOB:STATUS:" + status);
-                return status.contains("STARTED");
+                return containsStatus(getUser(), project, jobUUID, "STARTED");
             }
         });
     }
@@ -465,13 +463,24 @@ public static void waitForJobStatusCancelRequestedOrCanceled(TestProject project
         executeUntilSuccessOrTimeout(new AbstractTestExecutable(SUPER_ADMIN, 5, HttpClientErrorException.class) {
             @Override
             public boolean runAndReturnTrueWhenSuccesfulImpl() throws Exception {
-                String status = as(getUser()).getJobStatus(project.getProjectId(), jobUUID);
-                LOG.info(">>>>>>>>>JOB:STATUS:" + status);
-                return status.contains("CANCEL_REQUESTED") || status.contains("CANCELED");
+                return containsStatus(getUser(), project, jobUUID, "CANCEL_REQUESTED", "CANCELED");
             }
         });
     }
 
+    private static boolean containsStatus(TestUser user, TestProject project, UUID jobUUID, String... acceptedContainedStatus) {
+        String status = as(user).getJobStatus(project.getProjectId(), jobUUID);
+        LOG.info(">>>>>>>>>JOB:STATUS:" + status);
+
+        for (String accepted : acceptedContainedStatus) {
+            if (status.contains(accepted)) {
+                return true;
+            }
+        }
+
+        return false;
+    }
+
     /**
      * Waits for sechub job being finally canceled - after 5 seconds time out is
      * reached
@@ -489,9 +498,7 @@ public static void waitForJobStatusCanceled(TestProject project, UUID jobUUID, b
         executeUntilSuccessOrTimeout(new AbstractTestExecutable(SUPER_ADMIN, 5, runnable, HttpClientErrorException.class) {
             @Override
             public boolean runAndReturnTrueWhenSuccesfulImpl() throws Exception {
-                String status = as(getUser()).getJobStatus(project.getProjectId(), jobUUID);
-                LOG.info(">>>>>>>>>JOB:STATUS:" + status);
-                return status.contains("CANCELED");
+                return containsStatus(getUser(), project, jobUUID, "CANCELED");
             }
         });
     }
@@ -505,13 +512,10 @@ public boolean runAndReturnTrueWhenSuccesfulImpl() throws Exception {
      */
     public static void waitForJobStatusSuspended(TestProject project, UUID jobUUID) {
         LOG.info("wait for job status is 'suspended'. project:{}, job:{}", project.getProjectId(), jobUUID);
-
         executeUntilSuccessOrTimeout(new AbstractTestExecutable(SUPER_ADMIN, 5, HttpClientErrorException.class) {
             @Override
             public boolean runAndReturnTrueWhenSuccesfulImpl() throws Exception {
-                String status = as(getUser()).getJobStatus(project.getProjectId(), jobUUID);
-                LOG.info(">>>>>>>>>JOB:STATUS:" + status);
-                return status.contains("SUSPENDED");
+                return containsStatus(getUser(), project, jobUUID, "SUSPENDED");
             }
         });
     }
@@ -528,9 +532,7 @@ public static void waitForJobStatusFailed(TestProject project, UUID jobUUID) {
         executeUntilSuccessOrTimeout(new AbstractTestExecutable(SUPER_ADMIN, 5, HttpClientErrorException.class) {
             @Override
             public boolean runAndReturnTrueWhenSuccesfulImpl() throws Exception {
-                String status = as(getUser()).getJobStatus(project.getProjectId(), jobUUID);
-                LOG.info(">>>>>>>>>JOB:STATUS:" + status);
-                return status.contains("FAILED");
+                return containsStatus(getUser(), project, jobUUID, "FAILED");
             }
         });
     }

sechub-integrationtest/src/test/java/com/mercedesbenz/sechub/integrationtest/scenario11/SigTermSimulationJobScenario11IntTest.java

@@ -13,11 +13,12 @@
 import org.junit.Test;
 import org.junit.rules.Timeout;
 
+import com.mercedesbenz.sechub.commons.model.TrafficLight;
 import com.mercedesbenz.sechub.commons.pds.data.PDSJobStatusState;
 import com.mercedesbenz.sechub.integrationtest.api.IntegrationTestSetup;
-import com.mercedesbenz.sechub.integrationtest.api.TestAPI;
 import com.mercedesbenz.sechub.integrationtest.api.TestProject;
 import com.mercedesbenz.sechub.integrationtest.api.TestSecHubJobInfoForUserListPage;
+import com.mercedesbenz.sechub.integrationtest.api.TestUser;
 
 /**
  * Integration tests to check SIGTERM handling operations work
@@ -42,19 +43,23 @@ public class SigTermSimulationJobScenario11IntTest {
      *
      * The PDS job will run in background without stop. When new SecHub server comes
      * up (we simulate by turning off termination flag in integration test mode...)
-     * the result from PDS are reused (if the PDS job is not already done SecHub
-     * will wait and reuse - so we have no race condition problems here)
+     * the result from PDS is reused (if the PDS job is not already done SecHub will
+     * wait and reuse - so we have no race condition problems here)
      */
     public void simulate_SecHub_SIGTERM_handling_leads_to_restart_of_SecHub_job_and_reuse_of_pds_job() {
 
+        TestUser user = USER_1;
+        boolean resetCalled = false;
         try {
 
             /* @formatter:off */
             /* check preconditions */
-            assertThat(TestAPI.isSecHubTerminating()).describedAs("Ensure the server is not already terminating").isFalse();
+            assertThat(isSecHubTerminating()).
+                        describedAs("Ensure the server is not already terminating").
+                        isFalse();
 
-            /* prepare */
-            UUID sechubJobUUD = as(USER_1).triggerAsyncPDSCodeScanWithDifferentDataSections(project);
+            /* prepare 1 */
+            UUID sechubJobUUD = as(user).triggerAsyncPDSCodeScanWithWantedTrafficLightResult(project, TrafficLight.YELLOW);
             waitForJobRunning(project, sechubJobUUD);
             UUID pdsJobUUID = waitForFirstPDSJobOfSecHubJobAndReturnPDSJobUUID(sechubJobUUD);
             waitForPDSJobInState(PDSJobStatusState.RUNNING, 2000,100,pdsJobUUID,true);
@@ -67,29 +72,48 @@ public void simulate_SecHub_SIGTERM_handling_leads_to_restart_of_SecHub_job_and_
             assertPDSJobStatus(pdsJobUUID).isInState(PDSJobStatusState.RUNNING); // pds job is still running
 
             // fetch last user job - must be the one we have created here...
-            TestSecHubJobInfoForUserListPage jobInfo = as(USER_1).
+            TestSecHubJobInfoForUserListPage jobInfo = as(user).
                 fetchUserJobInfoListOneEntryOrNull(project);
-                assertUserJobInfo(jobInfo).
+
+            assertUserJobInfo(jobInfo).
                 hasJobInfoFor(sechubJobUUD).
                 withEndedTimeStampNotNull().
                 withExecutionResult("NONE"); // no result
 
-            /* execute 2 - we simulate here a new server start - means without termination flag set -> job processing will start again and processes old job */
+            /* prepare 2 - we simulate here a new server start - means without termination
+             *             flag set -> job processing will start again and processes old job
+             *             like a new server would do
+             */
             resetSecHubTerminationService();
+            resetCalled=true;
+
+            /* test 2 - check suspended report will resume and be done automatically */
+            waitForJobDone(project, sechubJobUUD, 15, true);
 
-            /* test 2 */
-            waitForJobDone(project, sechubJobUUD, 5, true);
+            /* test 3 - check report is as expected */
+            String report = as(user).getJobReport(project, sechubJobUUD);
+            assertReportUnordered(report).
+                hasTrafficLight(TrafficLight.YELLOW).
+                finding().description("i am a medium error").isContained();
+
+            /* test 4 - check only ONE PDS job was used */
             List<UUID> relatedPSjobUUIDs = fetchAllPDSJobUUIDsForSecHubJob(sechubJobUUD);
             assertThat(relatedPSjobUUIDs).
                 describedAs("Check that ONE PDS job has been reused by SecHub job which was suspended and now restarted").
                 hasSize(1).
             contains(pdsJobUUID);
 
+
             /* @formatter:on */
         } finally {
-            // we always reset the termination service to avoid cross site effects on other
-            // tests or on restart of this test
-            TestAPI.resetSecHubTerminationService();
+            if (!resetCalled) {
+                // IMPORTANT:
+                // -----------
+                // we MUST ensure termination service reset is done to avoid cross site effects
+                // on other
+                // tests when reset was not called in former try block!
+                resetSecHubTerminationService();
+            }
         }
     }
 }

sechub-integrationtest/src/test/java/com/mercedesbenz/sechub/integrationtest/scenario4/JobUsecasesEventTraceScenario4IntTest.java

@@ -120,7 +120,7 @@ public void UC_ADMIN_RESTARTS_JOB_HARD__simulate_jvm_crash_but_product_results_i
                  to("com.mercedesbenz.sechub.domain.scan.ScanService").
            /* 7 */
            syncEvent(MessageID.REQUEST_SCHEDULER_JOB_STATUS).
-                 from("com.mercedesbenz.sechub.domain.scan.ScanProgressMonitor").
+                 from("com.mercedesbenz.sechub.domain.scan.ScanProgressStateFetcher").
                  to("com.mercedesbenz.sechub.domain.schedule.job.SchedulerJobStatusRequestHandler").
            /* 8 */
            asyncEvent(MessageID.JOB_DONE).
@@ -182,7 +182,7 @@ public void UC_ADMIN_RESTARTS_JOB_HARD__simulate_jvm_crash_no_product_results_in
                  to("com.mercedesbenz.sechub.domain.scan.ScanService").
            /* 6 */
            syncEvent(MessageID.REQUEST_SCHEDULER_JOB_STATUS).
-                 from("com.mercedesbenz.sechub.domain.scan.ScanProgressMonitor").
+                 from("com.mercedesbenz.sechub.domain.scan.ScanProgressStateFetcher").
                  to("com.mercedesbenz.sechub.domain.schedule.job.SchedulerJobStatusRequestHandler").
            /* 7 */
            asyncEvent(MessageID.JOB_DONE).
@@ -279,7 +279,7 @@ public void UC_ADMIN_RESTARTS_JOB__simulate_jvm_crash_but_product_results_in_db(
                  to("com.mercedesbenz.sechub.domain.scan.ScanService").
            /* 5 */
            syncEvent(MessageID.REQUEST_SCHEDULER_JOB_STATUS).
-                 from("com.mercedesbenz.sechub.domain.scan.ScanProgressMonitor").
+                 from("com.mercedesbenz.sechub.domain.scan.ScanProgressStateFetcher").
                  to("com.mercedesbenz.sechub.domain.schedule.job.SchedulerJobStatusRequestHandler").
            /* 6 */
            asyncEvent(MessageID.JOB_DONE).
@@ -338,7 +338,7 @@ public void UC_ADMIN_RESTARTS_JOB__simulate_jvm_crash_no_product_results_in_db_u
                  to("com.mercedesbenz.sechub.domain.scan.ScanService").
            /* 5 */
            syncEvent(MessageID.REQUEST_SCHEDULER_JOB_STATUS).
-                 from("com.mercedesbenz.sechub.domain.scan.ScanProgressMonitor").
+                 from("com.mercedesbenz.sechub.domain.scan.ScanProgressStateFetcher").
                  to("com.mercedesbenz.sechub.domain.schedule.job.SchedulerJobStatusRequestHandler").
            /* 6 */
            asyncEvent(MessageID.JOB_DONE).

sechub-notification/src/main/java/com/mercedesbenz/sechub/domain/notification/superadmin/InformAdminsThatNewSchedulerInstanceHasBeenStartedNotificationService.java

@@ -11,6 +11,7 @@
 import com.mercedesbenz.sechub.domain.notification.NotificationConfiguration;
 import com.mercedesbenz.sechub.domain.notification.email.EmailService;
 import com.mercedesbenz.sechub.domain.notification.email.MailMessageFactory;
+import com.mercedesbenz.sechub.sharedkernel.DocumentationScopeConstants;
 import com.mercedesbenz.sechub.sharedkernel.MustBeDocumented;
 import com.mercedesbenz.sechub.sharedkernel.Step;
 import com.mercedesbenz.sechub.sharedkernel.messaging.ClusterMemberMessage;
@@ -22,7 +23,7 @@ public class InformAdminsThatNewSchedulerInstanceHasBeenStartedNotificationServi
     private static final Logger LOG = LoggerFactory.getLogger(InformAdminsThatNewSchedulerInstanceHasBeenStartedNotificationService.class);
 
     @Value("${sechub.notification.scheduler.startup.enabled:true}")
-    @MustBeDocumented(scope = "administration", value = "When enabled, administrators will be informed by notification "
+    @MustBeDocumented(scope = DocumentationScopeConstants.SCOPE_ADMINISTRATION, value = "When enabled, administrators will be informed by notification "
             + "when new scheduler instances are started. " + "Those notifications will also contain information about potential zombie jobs.\n\n"
             + " When disabled, incoming events will be ignored and no notification sent.")
     boolean notificationEnabled;