eficode · farthinder · Oct 26, 2023 · Oct 26, 2023 · Oct 26, 2023
diff --git a/pom.xml b/pom.xml
@@ -6,7 +6,7 @@
 
     <groupId>com.eficode</groupId>
     <artifactId>devstack</artifactId>
-    <version>2.3.11-SNAPSHOT</version>
+    <version>2.3.12-SNAPSHOT</version>
     <packaging>jar</packaging>
 
     <name>DevStack</name>
@@ -35,11 +35,13 @@
             <scope>provided</scope>
             <type>pom</type>
         </dependency>
+        <!-- https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-yaml -->
         <dependency>
-            <groupId>org.codehaus.groovy</groupId>
-            <artifactId>groovy-yaml</artifactId>
-            <version>${groovy.version}</version>
+            <groupId>com.fasterxml.jackson.dataformat</groupId>
+            <artifactId>jackson-dataformat-yaml</artifactId>
+            <version>2.15.3</version>
         </dependency>
+
         <dependency>
             <groupId>org.codehaus.groovy</groupId>
             <artifactId>groovy-json</artifactId>

diff --git a/src/main/groovy/com/eficode/devstack/container/impl/HarborManagerContainer.groovy b/src/main/groovy/com/eficode/devstack/container/impl/HarborManagerContainer.groovy
@@ -1,7 +1,8 @@
 package com.eficode.devstack.container.impl
 
-import groovy.yaml.YamlBuilder
-import groovy.yaml.YamlSlurper
+
+import com.fasterxml.jackson.databind.ObjectMapper
+import com.fasterxml.jackson.dataformat.yaml.YAMLFactory
 import org.apache.groovy.json.internal.LazyMap
 
 import java.nio.file.Files
@@ -14,6 +15,8 @@ class HarborManagerContainer extends DoodContainer {
     String harborBaseUrl
     String basePath
 
+    ObjectMapper objectMapper = new ObjectMapper(new YAMLFactory())
+
     /**
      * HarborManagerContainer setups a container that runs the Harbor installation scripts, which in turn creates all the "real"
      * harbor containers
@@ -30,14 +33,14 @@ class HarborManagerContainer extends DoodContainer {
         this.harborBaseUrl = baseUrl
         this.harborVersion = harborVersion
         this.containerName = host + "-manager"
-        this.basePath =   baseDir[-1] == "/" ? baseDir + host : baseDir + "/"  + host
+        this.basePath = baseDir[-1] == "/" ? baseDir + host : baseDir + "/" + host
 
         if (dockerHost && dockerCertPath) {
             assert setupSecureRemoteConnection(dockerHost, dockerCertPath): "Error setting up secure remote docker connection"
         }
 
         prepareBindMount("/var/run/docker.sock", "/var/run/docker.sock") // Mount docker socket
-        prepareBindMount(baseDir , baseDir, false) //Mount data dir, data in this dir needs to be accessible by both engine and manager-container using the same path
+        prepareBindMount(baseDir, baseDir, false) //Mount data dir, data in this dir needs to be accessible by both engine and manager-container using the same path
 
     }
 
@@ -55,22 +58,19 @@ class HarborManagerContainer extends DoodContainer {
         extractDomainFromUrl(harborBaseUrl)
     }
 
-    String getPort(){
-        extractPortFromUrl( harborBaseUrl)
+    String getPort() {
+        extractPortFromUrl(harborBaseUrl)
     }
 
 
-
-
     @Override
     boolean runAfterDockerSetup() {
 
         log.info("Setting up Harbor")
 
         //Make sure basePath is empty or does not exist
         ArrayList<String> cmdOutput = runBashCommandInContainer("""ls "$basePath" | wc -l""", 5)
-        assert cmdOutput == ["0"] || cmdOutput.any{it.startsWith("ls: cannot access")} : "Harbor base path is not empty: $basePath"
-
+        assert cmdOutput == ["0"] || cmdOutput.any { it.startsWith("ls: cannot access") }: "Harbor base path is not empty: $basePath"
 
 
         cmdOutput = runBashCommandInContainer("apt install -y wget; echo status: \$?", 100)
@@ -91,16 +91,27 @@ class HarborManagerContainer extends DoodContainer {
         log.info("\tFinished downloading and extracting Harbor")
 
 
-        assert modifyInstallYml() : "Error updating Harbor install config file: harbor.yml"
+        assert modifyInstallYml(): "Error updating Harbor install config file: harbor.yml"
 
         log.info("\tStarting  installation")
-        cmdOutput = runBashCommandInContainer(installPath + "/harbor/install.sh ; echo status: \$?", 400 )
-        assert cmdOutput.last().contains("status: 0"): "Error installing harbor:" + cmdOutput.join("\n")
+        cmdOutput = runBashCommandInContainer(installPath + "/harbor/install.sh ; echo status: \$?", 400)
+        if (!cmdOutput.last().contains("status: 0")) {
+            log.warn("\tThere where problems during setup of harbor, potentially because dockerCompose has yet to mo modified, will modify and try again")
+        }
 
+        cmdOutput = runBashCommandInContainer("cd " + installPath + "/harbor && docker-compose stop && echo status: \$?", 80)
+        assert cmdOutput.last().contains("status: 0"): "Error stopping harbor before modifying docker-compose file:" + cmdOutput.join("\n")
 
-        assert modifyDockerCompose() : "Error modifying Harbors docker-compose file"
+        assert modifyDockerCompose(): "Error modifying Harbors docker-compose file"
 
+
+        sleep(5000)
         cmdOutput = runBashCommandInContainer("cd " + installPath + "/harbor && docker-compose up -d && echo status: \$?", 80)
+        if (cmdOutput.last() != "status: 0" || cmdOutput.toString().contains("error")) {
+            log.warn("\tThere was an error starting harbor after docker compose modification, this is common and a second attempt will be made")
+            sleep(5000)
+            cmdOutput = runBashCommandInContainer("cd " + installPath + "/harbor && docker-compose up -d && echo status: \$?", 120)
+        }
         assert cmdOutput.last().contains("status: 0"): "Error applying the modified docker-compose file:" + cmdOutput.join("\n")
 
         return true
@@ -114,61 +125,60 @@ class HarborManagerContainer extends DoodContainer {
      */
     boolean modifyDockerCompose() {
 
-
-
-
         log.info("\tCustomizing Harbor docker compose")
-        Path tmpDir= Files.createTempDirectory("harbor-compose")
+        Path tmpDir = Files.createTempDirectory("harbor-compose")
         String tmpDirPath = tmpDir.toFile().absolutePath
 
 
-
         ArrayList<File> files = copyFilesFromContainer("${installPath}/harbor/docker-compose.yml", tmpDirPath + "/")
 
-        assert files.size() == 1 && files.first().name == "docker-compose.yml" : "Error, could not find docker-compose.yml file"
+        assert files.size() == 1 && files.first().name == "docker-compose.yml": "Error, could not find docker-compose.yml file"
         File yamlFile = files.first()
         log.debug("\t\tRetried docker compose file from container:" + yamlFile.absolutePath)
 
 
-        LazyMap originalYml = new YamlSlurper().parse(yamlFile) as LazyMap
-        LazyMap modifiedYml = new YamlSlurper().parse(yamlFile) as LazyMap
+        //LazyMap originalYml = new YamlSlurper().parse(yamlFile) as LazyMap
+        //LazyMap modifiedYml = new YamlSlurper().parse(yamlFile) as LazyMap
+        LazyMap originalYml = objectMapper.readValue(yamlFile, LazyMap.class)
+        LazyMap modifiedYml = objectMapper.readValue(yamlFile, LazyMap.class)
 
 
-
-        modifiedYml.services.each {Entry<String, LazyMap> service  ->
-            log.debug("\t"*3 + "Customising Docker Service:" + service.key)
+        modifiedYml.services.each { Entry<String, LazyMap> service ->
+            log.debug("\t" * 3 + "Customising Docker Service:" + service.key)
 
             service.value.networks = containerDefaultNetworks
-            log.trace("\t"*4 + "Set networks to:" + service.value.networks)
-            log.trace("\t"*4 + "Used to be:" + originalYml.services.get(service.key).networks)
+            log.trace("\t" * 4 + "Set networks to:" + service.value.networks)
+            log.trace("\t" * 4 + "Used to be:" + originalYml.services.get(service.key).networks)
 
         }
 
-        log.debug("\t"*3 + "Customising Docker Network")
+        log.debug("\t" * 3 + "Customising Docker Network")
         modifiedYml.remove("networks")
 
         Map<String, LazyMap> networks = [:]
-        containerDefaultNetworks.each {networkName ->
-            networks.put(networkName as String,["external": true, "name":networkName] as LazyMap )
+        containerDefaultNetworks.each { networkName ->
+            networks.put(networkName as String, ["external": true, "name": networkName] as LazyMap)
         }
         modifiedYml.put("networks", networks)
         //modifiedYml.put("networks", ["default": [external: [name: networkName]]])
 
-        log.trace("\t"*4 + "Set networks to:" + modifiedYml.networks)
-        log.trace("\t"*4 + "Used to be:" + originalYml.networks)
-
+        log.trace("\t" * 4 + "Set networks to:" + modifiedYml.networks)
+        log.trace("\t" * 4 + "Used to be:" + originalYml.networks)
 
 
-        YamlBuilder yamlBuilder = new YamlBuilder()
-        yamlBuilder(modifiedYml)
+        //Change the user of log container to root https://github.com/goharbor/harbor/issues/16669
+        (modifiedYml.services.find { Entry<String, LazyMap> service -> service.key == "log" } as Entry<String, Map>).value.put("user", "root")
 
+        //YamlBuilder yamlBuilder = new YamlBuilder()
+        //yamlBuilder(modifiedYml)
 
-        yamlFile.write(yamlBuilder.toString())
+        //yamlFile.write(yamlBuilder.toString())
+        yamlFile.write(  objectMapper.writeValueAsString(modifiedYml))
 
-        assert copyFileToContainer(yamlFile.absolutePath, installPath + "/harbor/") : "Error copying updated YAML file to container"
+        assert copyFileToContainer(yamlFile.absolutePath, installPath + "/harbor/"): "Error copying updated YAML file to container"
         tmpDir.deleteDir()
 
-        log.info("\tFinished customizing installation configuration")
+        log.info("\tFinished customizing docker-compose file")
 
         return true
 
@@ -180,17 +190,19 @@ class HarborManagerContainer extends DoodContainer {
     boolean modifyInstallYml() {
 
 
-        Path tmpDir= Files.createTempDirectory("harbor-conf")
+        Path tmpDir = Files.createTempDirectory("harbor-conf")
         String tmpDirPath = tmpDir.toFile().absolutePath
 
 
         ArrayList<File> files = copyFilesFromContainer("${installPath}/harbor/harbor.yml.tmpl", tmpDirPath + "/")
 
-        assert files.size() == 1 && files.first().name == "harbor.yml.tmpl" : "Error, could not find template config file"
+        assert files.size() == 1 && files.first().name == "harbor.yml.tmpl": "Error, could not find template config file"
         File yamlFile = files.first()
 
 
-        LazyMap originalYml = new YamlSlurper().parse(yamlFile) as LazyMap
+        //LazyMap originalYml = new YamlSlurper().parse(yamlFile) as LazyMap
+        LazyMap originalYml = objectMapper.readValue(yamlFile, LazyMap.class)
+
 
 
         LazyMap modifiedYml = originalYml
@@ -201,14 +213,15 @@ class HarborManagerContainer extends DoodContainer {
 
 
 
-        YamlBuilder yamlBuilder = new YamlBuilder()
-        yamlBuilder(modifiedYml)
+        //YamlBuilder yamlBuilder = new YamlBuilder()
+        //yamlBuilder(modifiedYml)
 
         File modifiedYamlFile = new File(tmpDirPath + "/harbor.yml")
         modifiedYamlFile.createNewFile()
-        modifiedYamlFile.write(yamlBuilder.toString().replaceAll("\"", ""))
+        //modifiedYamlFile.write(yamlBuilder.toString().replaceAll("\"", ""))
+        modifiedYamlFile.write(objectMapper.writeValueAsString(originalYml).replaceAll("\"", ""))
 
-        assert copyFileToContainer(modifiedYamlFile.absolutePath, installPath + "/harbor/") : "Error copying updated YAML file to container"
+        assert copyFileToContainer(modifiedYamlFile.absolutePath, installPath + "/harbor/"): "Error copying updated YAML file to container"
         tmpDir.deleteDir()
 
         log.info("\tFinished customizing installation configuration")

diff --git a/src/main/groovy/com/eficode/devstack/deployment/impl/HarborDeployment.groovy b/src/main/groovy/com/eficode/devstack/deployment/impl/HarborDeployment.groovy
@@ -74,9 +74,14 @@ class HarborDeployment implements Deployment {
 
         assert managerContainer.startContainer()
         sleep(5000)
-        ArrayList<String> cmdOutput = managerContainer.runBashCommandInContainer("cd ${managerContainer.installPath}/harbor ; docker-compose start ; echo status: \$?", 120)
+        ArrayList<String> cmdOutput = managerContainer.runBashCommandInContainer("cd ${managerContainer.installPath}/harbor && docker-compose start && echo status: \$?", 120)
+        if (cmdOutput.last() != "status: 0" || cmdOutput.toString().contains("error")) {
+            log.warn("\tThere was an error starting harbor deployment, this is common and a second attempt will be made")
+            sleep(5000)
+            cmdOutput = managerContainer.runBashCommandInContainer("cd ${managerContainer.installPath}/harbor && docker-compose start && echo status: \$?", 120)
+        }
         assert cmdOutput.last() == "status: 0": "Error starting harbor:" + cmdOutput.join("\n")
-
+        sleep(5000)
         return true
     }
 
@@ -86,7 +91,7 @@ class HarborDeployment implements Deployment {
 
 
         assert managerContainer.startContainer()
-        ArrayList<String> cmdOutput = managerContainer.runBashCommandInContainer("cd ${managerContainer.installPath}/harbor ; docker-compose stop ; echo status: \$?", 120)
+        ArrayList<String> cmdOutput = managerContainer.runBashCommandInContainer("cd ${managerContainer.installPath}/harbor && docker-compose stop && echo status: \$?", 120)
         assert cmdOutput.last() == "status: 0": "Error stopping harbor:" + cmdOutput.join("\n")
 
         assert managerContainer.stopContainer()

diff --git a/src/main/groovy/com/eficode/devstack/deployment/impl/JsmH2Deployment.groovy b/src/main/groovy/com/eficode/devstack/deployment/impl/JsmH2Deployment.groovy
@@ -118,7 +118,10 @@ class JsmH2Deployment implements Deployment{
             if (setupDeployment()) {
                 if (snapshotAfterCreation) {
                     log.info("\tSnapshotting the newly created container")
-                    return jsmContainer.snapshotJiraHome() != null
+                    assert jsmContainer.snapshotJiraHome() != null : "Error snapshotting container:" + jsmContainer.shortId
+                    log.info("\tWaiting for JIRA to start back up")
+                    assert jiraRest.waitForJiraToBeResponsive() : "Error waiting for JIRA to start up after creating snapshot"
+                    return true
                 }else {
                     return true
                 }

diff --git a/src/test/groovy/com/eficode/devstack/deployment/impl/HarborDeploymentTest.groovy b/src/test/groovy/com/eficode/devstack/deployment/impl/HarborDeploymentTest.groovy
@@ -37,13 +37,17 @@ class HarborDeploymentTest extends DevStackSpec {
     def "Test the basics"(String dockerHost, String certPath, String harborBaseUrl, String harborVersion, String harborBaseDir) {
 
         when:
+
+        new File(harborBaseDir).exists() && new File(harborBaseDir).deleteDir()
+        new File(harborBaseDir).mkdirs()
+
         HarborDeployment hd = new HarborDeployment(harborBaseUrl, harborVersion, harborBaseDir, dockerHost, certPath)
 
         if (dockerHost && certPath) {
             assert hd.managerContainer.dockerClient.dockerClientConfig.host == hd.managerContainer.extractDomainFromUrl(dockerHost): "Connection to remote Docker host was not setup"
 
-        }else {
-            assert hd.managerContainer.dockerClient.dockerClientConfig.host == "/var/run/docker.sock": "Connection to local Docker host was not setup"
+        } else {
+            assert hd.managerContainer.dockerClient.dockerClientConfig.host.endsWith("run/docker.sock"): "Connection to local Docker host was not setup as expected"
 
         }
 
@@ -56,10 +60,14 @@ class HarborDeploymentTest extends DevStackSpec {
         Unirest.get(harborBaseUrl).basicAuth("admin", "Harbor12345").asEmpty().status == 200
         hd.harborContainers.id.contains(hd.managerContainer.inspectContainer().id) //Make sure harborContainers returns manager container as well
         hd.harborContainers.every { it.state in [ContainerState.Status.Running.value, ContainerState.Status.Restarting.value] }
-        hd.harborContainers.every {it.networkSettings.networks.keySet().toList() == [hd.deploymentNetworkName]}
-        hd.harborContainers.collect {it.names.first()}.every {containerName ->
+        hd.harborContainers.every {
+            assert it.networkSettings.networks.keySet().toList() == [hd.deploymentNetworkName] : it.names.join(",") + " container has the wrong network: " + it.networkSettings.networks.keySet().toList().join(", ")
+            return true
+        }
+        hd.harborContainers.collect { it.names.first() }.every { containerName ->
             String hostname = containerName[1..-1]
-            hd.managerContainer.runBashCommandInContainer("ping ${hostname} -c 1 && echo Status: \$?", 5).last().contains("Status: 0")
+            assert hd.managerContainer.runBashCommandInContainer("ping ${hostname} -c 1 && echo Status: \$?", 5).last().contains("Status: 0") : "Management container can not ping $hostname"
+            return true
         }
 
         when: "Stopping the deployment"
@@ -85,9 +93,10 @@ class HarborDeploymentTest extends DevStackSpec {
         hd.getHarborContainers().size() == 0
 
         where:
-        dockerHost       | certPath       | harborBaseUrl             | harborVersion | harborBaseDir
-        ""               | ""             | "http://localhost"        | "v2.6.0"      | "/tmp"
-        dockerRemoteHost | dockerCertPath | "http://harbor.domain.se" | "v2.6.0"      | "/tmp"
+        dockerHost | certPath | harborBaseUrl      | harborVersion | harborBaseDir
+        ""         | ""       | "http://localhost" | "v2.7.3"      | "/tmp/harbor"
+        ""         | ""       | "http://localhost" | "v2.6.0"      | "/tmp/harbor"
+        //dockerRemoteHost | dockerCertPath | "http://harbor.domain.se" | "v2.6.0"      | "/tmp"
 
     }
 }