DirectorySyncer.groovy

 * Added and switched to using getPollBasedSyncScript()

DockerClientDS.groovy
 * Added foreNew parameter to getOrCreateVolume

Author: farthinder

src/main/groovy/com/eficode/devstack/util/DirectorySyncer.groovy

@@ -2,11 +2,8 @@ package com.eficode.devstack.util
 
 import com.eficode.devstack.container.Container
 import com.fasterxml.jackson.databind.ObjectMapper
-import de.gesellix.docker.client.EngineResponseContent
 import de.gesellix.docker.client.network.ManageNetworkClient
 import de.gesellix.docker.remote.api.ContainerSummary
-import de.gesellix.docker.remote.api.Mount
-import de.gesellix.docker.remote.api.MountPoint
 import de.gesellix.docker.remote.api.Volume
 import org.slf4j.Logger
 
@@ -41,7 +38,20 @@ class DirectorySyncer implements Container {
 
     }
 
-    static String getSyncScript(String rsyncOptions = "-avh", String rsyncSrc = " /mnt/src/", String rsyncDest = " /mnt/dest/") {
+    static String getPollBasedSyncScript(String rsyncOptions = "-avh", String rsyncSrc = "/mnt/src/", String rsyncDest = "/mnt/dest/", Double intervalS = 1.5) {
+
+        return """
+        apk update
+        apk add rsync
+        watch -n $intervalS "rsync $rsyncOptions ${rsyncSrc.replace(" ", "\\ ")} ${rsyncDest.replace(" ", "\\ ")}"
+        
+        """.stripIndent()
+
+    }
+
+
+    //Has problems with recursive dirs added after start
+    static String getEventBasedSyncScript(String rsyncOptions = "-avh", String rsyncSrc = " /mnt/src/", String rsyncDest = " /mnt/dest/") {
 
         return """
         
@@ -56,14 +66,15 @@ class DirectorySyncer implements Container {
             exit 1
         fi
         
+        echo inotifywait is installed
         
         function execute() {
             eval "\$@"
                 rsync $rsyncOptions $rsyncSrc $rsyncDest
         }
         
         execute""
-        
+
         inotifywait --recursive --monitor --format "%e %w%f" \\
         --event modify,create,delete,moved_from,close_write /mnt/src \\
         | while read changed; do
@@ -98,6 +109,13 @@ class DirectorySyncer implements Container {
 
     }
 
+    static DirectorySyncer getDuplicateContainer(DockerClientDS dockerClientDS, String containerName) {
+        DirectorySyncer syncer = new DirectorySyncer(dockerClientDS.host, dockerClientDS.certPath)
+        syncer.containerName = containerName
+
+        return syncer.getDuplicateContainer()
+    }
+
 
 
     /**
@@ -108,34 +126,15 @@ class DirectorySyncer implements Container {
     DirectorySyncer getDuplicateContainer() {
 
 
-        Map filterMap = [name: [this.containerName + ".*"], "volume": this.preparedMounts.collect { it.target }]
+        Map filterMap = [name: [this.containerName]]
         String filterString = new ObjectMapper().writeValueAsString(filterMap)
-        ArrayList<ContainerSummary> looselyMatchingContainers = dockerClient.ps(true, null, false, filterString).content
-        ArrayList<ContainerSummary> matchingContainers = []
-        ArrayList<String> myMounts = this.preparedMounts.target
-        myMounts += this.preparedMounts.findAll {it.type == Mount.Type.Volume}.source
-        if (looselyMatchingContainers) {
-            matchingContainers = looselyMatchingContainers.findAll { matchingContainer ->
-
-                ArrayList<String> matchingMounts = matchingContainer.mounts.destination
-                matchingMounts += matchingContainer.mounts.findAll {it.type == MountPoint.Type.Volume}.name
-                //Handles the fact the mount points arent always given with a trailing /
-                Boolean mountsMatch = myMounts.every { myMount ->
-                    matchingMounts.any { it.equalsIgnoreCase(myMount) } ||
-                            matchingMounts.collect { it + "/" }.any { it.equalsIgnoreCase(myMount) }
-                }
-
-                return mountsMatch
-
-            }
-        }
-
-        if (matchingContainers.size() > 1) {
-            throw new InputMismatchException("Found multiple potential duplicate DirectorySyncer´s: " + matchingContainers.collect { it.id }.join(","))
-        } else if (matchingContainers.size() == 1) {
-            return new DirectorySyncer(dockerClient, matchingContainers.first())
-        } else {
+        ArrayList<ContainerSummary> matchingContainers = dockerClient.ps(true, null, false, filterString).content
+        if (matchingContainers.size() > 1){
+            throw new InputMismatchException("Error determining duplicate container based on name:" + this.containerName)
+        }else if (matchingContainers.size() == 0) {
             return null
+        }else {
+            return new DirectorySyncer(dockerClient, matchingContainers.first())
         }
 
     }
@@ -184,7 +183,7 @@ class DirectorySyncer implements Container {
         Logger log = container.log
 
         container.containerName = containerName ?: container.getAvailableContainerName()
-        container.prepareCustomEnvVar(["syncScript=${getSyncScript(rsyncOptions, "/mnt/src/*/")}"])
+        container.prepareCustomEnvVar(["syncScript=${getPollBasedSyncScript(rsyncOptions, "/mnt/src/*/")}"])
 
         Volume volume = container.dockerClient.getOrCreateVolume(destVolumeName)
 
@@ -211,7 +210,8 @@ class DirectorySyncer implements Container {
             return duplicate
         }
 
-        container.createContainer(["/bin/sh", "-c", "echo \"\$syncScript\" > /syncScript.sh && /bin/sh syncScript.sh"], [])
+
+        container.createContainer([container.defaultShell, "-c", "echo \"\$syncScript\" > /syncScript.sh && ${container.defaultShell} syncScript.sh"], [])
         container.startContainer()
 
         return container
@@ -225,9 +225,9 @@ class DirectorySyncer implements Container {
      * @param destUser The destination user and group that the file owner will be changed to, ex: 1001:1001
      * @return
      */
-    static DirectorySyncer syncBetweenVolumesAndUsers(String srcVolumeName, String destVolumeName, String destUser) {
+    static DirectorySyncer syncBetweenVolumesAndUsers(String srcVolumeName, String destVolumeName, String destUser, String containerName = "") {
 
-        DirectorySyncer syncer =  createSyncVolumeToVolume(srcVolumeName, destVolumeName, "-avhog --chown $destUser")
+        DirectorySyncer syncer =  createSyncVolumeToVolume(srcVolumeName, destVolumeName, "-avhog --chown $destUser", containerName)
 
         return syncer
     }
@@ -249,7 +249,7 @@ class DirectorySyncer implements Container {
         Logger log = container.log
 
         container.containerName = containerName ?: container.getAvailableContainerName()
-        container.prepareCustomEnvVar(["syncScript=${getSyncScript(rsyncOptions)}"])
+        container.prepareCustomEnvVar(["syncScript=${getPollBasedSyncScript(rsyncOptions)}"])
 
         Volume destVolume = container.dockerClient.getOrCreateVolume(destVolumeName)
         Volume srcVolume = container.dockerClient.getOrCreateVolume(srcVolumeName)

src/main/groovy/com/eficode/devstack/util/DockerClientDS.groovy

@@ -76,19 +76,28 @@ class DockerClientDS extends DockerClientImpl {
     }
 
 
-    Volume getOrCreateVolume(String volumeName) {
+    Volume getOrCreateVolume(String volumeName, Boolean forceNew = false) {
         Volume volume = getVolumesWithName(volumeName).find { true }
 
         if (volume) {
             log.debug("\tFound existing volume:" + volume.name)
-        } else {
-            log.debug("\tCreating new volume $volumeName")
-            EngineResponseContent<Volume> volumeResponse = createVolume(volumeName)
-            volume = volumeResponse?.content
-            assert volume: "Error creating volume $volumeName, " + volumeResponse?.getStatus()?.text
-            log.debug("\t\tCreated destination volume:" + volume.name)
+            if (!forceNew) {
+                log.debug("\t"*2 + "Returning found volume")
+                return volume
+            }else {
+                log.debug("\t"*2 + "Deleting existing volume as forceNew == true")
+                rmVolume(volume.name)
+            }
         }
 
+
+        log.debug("\tCreating new volume $volumeName")
+        EngineResponseContent<Volume> volumeResponse = createVolume(volumeName)
+        volume = volumeResponse?.content
+        assert volume: "Error creating volume $volumeName, " + volumeResponse?.getStatus()?.text
+        log.debug("\t\tCreated destination volume:" + volume.name)
+
+
         return volume
     }
 
@@ -305,7 +314,7 @@ class DockerClientDS extends DockerClientImpl {
                 },
                 Duration.of(timeoutM, ChronoUnit.MINUTES),
                 tag,
-               new FileInputStream(buildContext)
+                new FileInputStream(buildContext)
         )
 
         return new ImageSummaryDS(this, this.images().content.find { it.repoTags.any { it == tag } })

src/test/groovy/com/eficode/devstack/container/impl/DirectorySyncerTest.groovy

@@ -44,27 +44,27 @@ class DirectorySyncerTest extends DevStackSpec {
         !volumeExists(uniqueVolumeName) ?: dockerClient.rmVolume(uniqueVolumeName)
         log.debug("\tWill use sync to Docker volume:" + uniqueVolumeName)
 
-        DirectorySyncer firstSyncer = DirectorySyncer.createSyncToVolume([srcDir1.canonicalPath, srcDir2.canonicalPath], uniqueVolumeName, "DirSyncer", "-avh --delete", dockerRemoteHost, dockerCertPath )
+        DirectorySyncer firstSyncer = DirectorySyncer.createSyncToVolume([srcDir1.canonicalPath, srcDir2.canonicalPath], uniqueVolumeName, "DirSyncer", "-avh --delete", dockerRemoteHost, dockerCertPath)
         log.info("\tCreated first sync container: ${firstSyncer.containerName} (${firstSyncer.shortId})")
         Integer containersAfterFirst = firstSyncer.dockerClient.ps(true).content.size()
         log.info("\t\tDocker engine now has a total of ${containersAfterFirst} contianers")
 
         when: "Creating second sync container"
-        DirectorySyncer secondSyncer = DirectorySyncer.createSyncToVolume([srcDir1.canonicalPath, srcDir2.canonicalPath], uniqueVolumeName, "DirSyncer","-avh --delete", dockerRemoteHost, dockerCertPath )
+        DirectorySyncer secondSyncer = DirectorySyncer.createSyncToVolume([srcDir1.canonicalPath, srcDir2.canonicalPath], uniqueVolumeName, "DirSyncer", "-avh --delete", dockerRemoteHost, dockerCertPath)
         log.info("\tCreated second sync container: ${secondSyncer.containerName} (${secondSyncer.shortId})")
 
         then: "They should have the same ID"
-        assert firstSyncer.id == secondSyncer.id : "The second container doesnt have the same id"
-        assert containersAfterFirst == secondSyncer.dockerClient.ps(true).content.size() : "The number of containers changed after creating the second one"
+        assert firstSyncer.id == secondSyncer.id: "The second container doesnt have the same id"
+        assert containersAfterFirst == secondSyncer.dockerClient.ps(true).content.size(): "The number of containers changed after creating the second one"
         assert secondSyncer.running
         log.info("\tA duplicate container was not created, instead the first one was returned")
 
         when: "Stopping the sync container, and creating another duplicate"
         firstSyncer.stopContainer()
         assert !firstSyncer.running
-        secondSyncer = DirectorySyncer.createSyncToVolume([srcDir1.canonicalPath, srcDir2.canonicalPath], uniqueVolumeName, "DirSyncer","-avh --delete", dockerRemoteHost, dockerCertPath )
+        secondSyncer = DirectorySyncer.createSyncToVolume([srcDir1.canonicalPath, srcDir2.canonicalPath], uniqueVolumeName, "DirSyncer", "-avh --delete", dockerRemoteHost, dockerCertPath)
 
-        then:"The duplicate should have been automatically started"
+        then: "The duplicate should have been automatically started"
         secondSyncer.running
 
 
@@ -75,7 +75,6 @@ class DirectorySyncerTest extends DevStackSpec {
         dockerClient.rmVolume(uniqueVolumeName)
 
 
-
     }
 
 
@@ -96,7 +95,7 @@ class DirectorySyncerTest extends DevStackSpec {
         srcContainer.user = "1001:1001"
         srcContainer.createSleepyContainer()
         srcContainer.startContainer()
-        srcContainer.runBashCommandInContainer(ImageSummaryDS.getReplaceUserScriptBody("ubuntu", "1000", "ubuntu", "1000", "ubuntusrc", "1001", "ubuntusrc", "1001"),10, "root" )
+        srcContainer.runBashCommandInContainer(ImageSummaryDS.getReplaceUserScriptBody("ubuntu", "1000", "ubuntu", "1000", "ubuntusrc", "1001", "ubuntusrc", "1001"), 10, "root")
         srcContainer.runBashCommandInContainer("chown 1001:1001 -R /mnt/volume", 5, "root")
 
 
@@ -106,7 +105,7 @@ class DirectorySyncerTest extends DevStackSpec {
         destContainer.user = "1002:1002"
         destContainer.createSleepyContainer()
         destContainer.startContainer()
-        destContainer.runBashCommandInContainer(ImageSummaryDS.getReplaceUserScriptBody("ubuntu", "1000", "ubuntu", "1000", "ubuntudest", "1002", "ubuntudest", "1002"),10, "root" )
+        destContainer.runBashCommandInContainer(ImageSummaryDS.getReplaceUserScriptBody("ubuntu", "1000", "ubuntu", "1000", "ubuntudest", "1002", "ubuntudest", "1002"), 10, "root")
         destContainer.runBashCommandInContainer("chown 1002:1002 -R /mnt/volume", 5, "root")
 
         then: "When checking user id, they should be different"
@@ -151,7 +150,7 @@ class DirectorySyncerTest extends DevStackSpec {
         when: "When creating syncer"
 
         assert !volumeExists(uniqueVolumeName): "Destination volume already exists"
-        DirectorySyncer syncer = DirectorySyncer.createSyncToVolume([srcDir1.canonicalPath, srcDir2.canonicalPath], uniqueVolumeName, "DirSyncer", "-avh --delete", dockerRemoteHost, dockerCertPath )
+        DirectorySyncer syncer = DirectorySyncer.createSyncToVolume([srcDir1.canonicalPath, srcDir2.canonicalPath], uniqueVolumeName, "DirSyncer", "-avh --delete", dockerRemoteHost, dockerCertPath)
         log.info("\tCreated sync container: ${syncer.containerName} (${syncer.shortId})")
         ContainerInspectResponse containerInspect = syncer.inspectContainer()
 
@@ -185,14 +184,16 @@ class DirectorySyncerTest extends DevStackSpec {
         log.debug("\tContainer successfully synced the files to destination dir")
 
         when: "Creating a recursive file"
-        File recursiveFile = new File(srcDir1.canonicalPath + "/subDir/subFile.temp").createParentDirectories()
+        File recursiveFile = new File(srcDir1.canonicalPath + "/subDir/subDir2/subDir3/subDir4/subDir5/subFile.temp")
+        recursiveFile.createParentDirectories()
+        sleep(2000) //Make sure that we dont trigger on the dir creation, but on the recursive file
         recursiveFile.createNewFile()
         recursiveFile.text = System.nanoTime()
         log.info("\tCreate recursive file:" + recursiveFile.canonicalPath)
 
         then: "The sync container should see the new source file, and sync it to a new recursive dir"
         sleep(2000)
-        syncer.runBashCommandInContainer("cat /mnt/dest/subDir/subFile.temp").toString().contains(recursiveFile.text)
+        syncer.runBashCommandInContainer("cat /mnt/dest/subDir/subDir2/subDir3/subDir4/subDir5/subFile.temp").toString().contains(recursiveFile.text)
         log.info("\t\tFile was successfully synced")
 
 
@@ -211,27 +212,27 @@ class DirectorySyncerTest extends DevStackSpec {
         syncer.runBashCommandInContainer("cat /mnt/dest/${srcFile2.name}").toString().containsIgnoreCase(srcFile2.text)
         log.debug("\t\tContainer successfully synced the changes")
 
-        when:"Creating a new container and attaching it to the synced volume"
+        when: "Creating a new container and attaching it to the synced volume"
         AlpineContainer secondContainer = new AlpineContainer(dockerRemoteHost, dockerCertPath)
-        secondContainer.containerName = syncer.containerName  + "-companion"
+        secondContainer.containerName = syncer.containerName + "-companion"
         secondContainer.prepareVolumeMount(uniqueVolumeName, "/mnt/syncDir", false)
         secondContainer.createSleepyContainer()
 
 
-        then:"The second container should see the still remaining synced files"
-        assert secondContainer.startContainer() : "Error creating/staring second container"
+        then: "The second container should see the still remaining synced files"
+        assert secondContainer.startContainer(): "Error creating/staring second container"
         log.info("\tCreated an started second container ${secondContainer.shortId}")
-        assert secondContainer.mounts.any { it.type == MountPoint.Type.Volume && it.RW == true} : "Second container did not mount the shared volume"
+        assert secondContainer.mounts.any { it.type == MountPoint.Type.Volume && it.RW == true }: "Second container did not mount the shared volume"
         log.info("\tSecond container was attached to volume:" + uniqueVolumeName)
-        log.info("\tChecking that second container can access synced file:" + " /mnt/syncDir/${srcFile2.name}" )
-        assert secondContainer.runBashCommandInContainer("cat /mnt/syncDir/${srcFile2.name}").toString().containsIgnoreCase(srcFile2.text) : "Error reading synced file in second container:" + " /mnt/syncDir/${srcFile2.name}"
+        log.info("\tChecking that second container can access synced file:" + " /mnt/syncDir/${srcFile2.name}")
+        assert secondContainer.runBashCommandInContainer("cat /mnt/syncDir/${srcFile2.name}").toString().containsIgnoreCase(srcFile2.text): "Error reading synced file in second container:" + " /mnt/syncDir/${srcFile2.name}"
 
         log.info("\tChecking that second container can access recursive synced file")
-        assert secondContainer.runBashCommandInContainer("cat /mnt/syncDir/subDir/subFile.temp").toString().contains(recursiveFile.text)
+        assert secondContainer.runBashCommandInContainer("cat /mnt/syncDir/subDir/subDir2/subDir3/subDir4/subDir5/subFile.temp").toString().contains(recursiveFile.text)
         log.info("\t\tContainer can access that file")
         cleanup:
-        assert syncer.stopAndRemoveContainer()
-        assert secondContainer?.stopAndRemoveContainer()
+        syncer.stopAndRemoveContainer()
+        secondContainer?.stopAndRemoveContainer()
         srcDir1.deleteDir()
         srcDir2.deleteDir()
         dockerClient.rmVolume(containerInspect.mounts.find { it.type == MountPoint.Type.Volume }.name)