Skip to content

Commit 0fd7d7e

Browse files
committed
DirectorySyncer.groovy
* Added and switched to using getPollBasedSyncScript() DockerClientDS.groovy * Added foreNew parameter to getOrCreateVolume
1 parent c847a3a commit 0fd7d7e

File tree

3 files changed

+76
-66
lines changed

3 files changed

+76
-66
lines changed

src/main/groovy/com/eficode/devstack/util/DirectorySyncer.groovy

Lines changed: 36 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,8 @@ package com.eficode.devstack.util
22

33
import com.eficode.devstack.container.Container
44
import com.fasterxml.jackson.databind.ObjectMapper
5-
import de.gesellix.docker.client.EngineResponseContent
65
import de.gesellix.docker.client.network.ManageNetworkClient
76
import de.gesellix.docker.remote.api.ContainerSummary
8-
import de.gesellix.docker.remote.api.Mount
9-
import de.gesellix.docker.remote.api.MountPoint
107
import de.gesellix.docker.remote.api.Volume
118
import org.slf4j.Logger
129

@@ -41,7 +38,20 @@ class DirectorySyncer implements Container {
4138

4239
}
4340

44-
static String getSyncScript(String rsyncOptions = "-avh", String rsyncSrc = " /mnt/src/", String rsyncDest = " /mnt/dest/") {
41+
static String getPollBasedSyncScript(String rsyncOptions = "-avh", String rsyncSrc = "/mnt/src/", String rsyncDest = "/mnt/dest/", Double intervalS = 1.5) {
42+
43+
return """
44+
apk update
45+
apk add rsync
46+
watch -n $intervalS "rsync $rsyncOptions ${rsyncSrc.replace(" ", "\\ ")} ${rsyncDest.replace(" ", "\\ ")}"
47+
48+
""".stripIndent()
49+
50+
}
51+
52+
53+
//Has problems with recursive dirs added after start
54+
static String getEventBasedSyncScript(String rsyncOptions = "-avh", String rsyncSrc = " /mnt/src/", String rsyncDest = " /mnt/dest/") {
4555

4656
return """
4757
@@ -56,14 +66,15 @@ class DirectorySyncer implements Container {
5666
exit 1
5767
fi
5868
69+
echo inotifywait is installed
5970
6071
function execute() {
6172
eval "\$@"
6273
rsync $rsyncOptions $rsyncSrc $rsyncDest
6374
}
6475
6576
execute""
66-
77+
6778
inotifywait --recursive --monitor --format "%e %w%f" \\
6879
--event modify,create,delete,moved_from,close_write /mnt/src \\
6980
| while read changed; do
@@ -98,6 +109,13 @@ class DirectorySyncer implements Container {
98109

99110
}
100111

112+
static DirectorySyncer getDuplicateContainer(DockerClientDS dockerClientDS, String containerName) {
113+
DirectorySyncer syncer = new DirectorySyncer(dockerClientDS.host, dockerClientDS.certPath)
114+
syncer.containerName = containerName
115+
116+
return syncer.getDuplicateContainer()
117+
}
118+
101119

102120

103121
/**
@@ -108,34 +126,15 @@ class DirectorySyncer implements Container {
108126
DirectorySyncer getDuplicateContainer() {
109127

110128

111-
Map filterMap = [name: [this.containerName + ".*"], "volume": this.preparedMounts.collect { it.target }]
129+
Map filterMap = [name: [this.containerName]]
112130
String filterString = new ObjectMapper().writeValueAsString(filterMap)
113-
ArrayList<ContainerSummary> looselyMatchingContainers = dockerClient.ps(true, null, false, filterString).content
114-
ArrayList<ContainerSummary> matchingContainers = []
115-
ArrayList<String> myMounts = this.preparedMounts.target
116-
myMounts += this.preparedMounts.findAll {it.type == Mount.Type.Volume}.source
117-
if (looselyMatchingContainers) {
118-
matchingContainers = looselyMatchingContainers.findAll { matchingContainer ->
119-
120-
ArrayList<String> matchingMounts = matchingContainer.mounts.destination
121-
matchingMounts += matchingContainer.mounts.findAll {it.type == MountPoint.Type.Volume}.name
122-
//Handles the fact the mount points arent always given with a trailing /
123-
Boolean mountsMatch = myMounts.every { myMount ->
124-
matchingMounts.any { it.equalsIgnoreCase(myMount) } ||
125-
matchingMounts.collect { it + "/" }.any { it.equalsIgnoreCase(myMount) }
126-
}
127-
128-
return mountsMatch
129-
130-
}
131-
}
132-
133-
if (matchingContainers.size() > 1) {
134-
throw new InputMismatchException("Found multiple potential duplicate DirectorySyncer´s: " + matchingContainers.collect { it.id }.join(","))
135-
} else if (matchingContainers.size() == 1) {
136-
return new DirectorySyncer(dockerClient, matchingContainers.first())
137-
} else {
131+
ArrayList<ContainerSummary> matchingContainers = dockerClient.ps(true, null, false, filterString).content
132+
if (matchingContainers.size() > 1){
133+
throw new InputMismatchException("Error determining duplicate container based on name:" + this.containerName)
134+
}else if (matchingContainers.size() == 0) {
138135
return null
136+
}else {
137+
return new DirectorySyncer(dockerClient, matchingContainers.first())
139138
}
140139

141140
}
@@ -184,7 +183,7 @@ class DirectorySyncer implements Container {
184183
Logger log = container.log
185184

186185
container.containerName = containerName ?: container.getAvailableContainerName()
187-
container.prepareCustomEnvVar(["syncScript=${getSyncScript(rsyncOptions, "/mnt/src/*/")}"])
186+
container.prepareCustomEnvVar(["syncScript=${getPollBasedSyncScript(rsyncOptions, "/mnt/src/*/")}"])
188187

189188
Volume volume = container.dockerClient.getOrCreateVolume(destVolumeName)
190189

@@ -211,7 +210,8 @@ class DirectorySyncer implements Container {
211210
return duplicate
212211
}
213212

214-
container.createContainer(["/bin/sh", "-c", "echo \"\$syncScript\" > /syncScript.sh && /bin/sh syncScript.sh"], [])
213+
214+
container.createContainer([container.defaultShell, "-c", "echo \"\$syncScript\" > /syncScript.sh && ${container.defaultShell} syncScript.sh"], [])
215215
container.startContainer()
216216

217217
return container
@@ -225,9 +225,9 @@ class DirectorySyncer implements Container {
225225
* @param destUser The destination user and group that the file owner will be changed to, ex: 1001:1001
226226
* @return
227227
*/
228-
static DirectorySyncer syncBetweenVolumesAndUsers(String srcVolumeName, String destVolumeName, String destUser) {
228+
static DirectorySyncer syncBetweenVolumesAndUsers(String srcVolumeName, String destVolumeName, String destUser, String containerName = "") {
229229

230-
DirectorySyncer syncer = createSyncVolumeToVolume(srcVolumeName, destVolumeName, "-avhog --chown $destUser")
230+
DirectorySyncer syncer = createSyncVolumeToVolume(srcVolumeName, destVolumeName, "-avhog --chown $destUser", containerName)
231231

232232
return syncer
233233
}
@@ -249,7 +249,7 @@ class DirectorySyncer implements Container {
249249
Logger log = container.log
250250

251251
container.containerName = containerName ?: container.getAvailableContainerName()
252-
container.prepareCustomEnvVar(["syncScript=${getSyncScript(rsyncOptions)}"])
252+
container.prepareCustomEnvVar(["syncScript=${getPollBasedSyncScript(rsyncOptions)}"])
253253

254254
Volume destVolume = container.dockerClient.getOrCreateVolume(destVolumeName)
255255
Volume srcVolume = container.dockerClient.getOrCreateVolume(srcVolumeName)

src/main/groovy/com/eficode/devstack/util/DockerClientDS.groovy

Lines changed: 17 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -76,19 +76,28 @@ class DockerClientDS extends DockerClientImpl {
7676
}
7777

7878

79-
Volume getOrCreateVolume(String volumeName) {
79+
Volume getOrCreateVolume(String volumeName, Boolean forceNew = false) {
8080
Volume volume = getVolumesWithName(volumeName).find { true }
8181

8282
if (volume) {
8383
log.debug("\tFound existing volume:" + volume.name)
84-
} else {
85-
log.debug("\tCreating new volume $volumeName")
86-
EngineResponseContent<Volume> volumeResponse = createVolume(volumeName)
87-
volume = volumeResponse?.content
88-
assert volume: "Error creating volume $volumeName, " + volumeResponse?.getStatus()?.text
89-
log.debug("\t\tCreated destination volume:" + volume.name)
84+
if (!forceNew) {
85+
log.debug("\t"*2 + "Returning found volume")
86+
return volume
87+
}else {
88+
log.debug("\t"*2 + "Deleting existing volume as forceNew == true")
89+
rmVolume(volume.name)
90+
}
9091
}
9192

93+
94+
log.debug("\tCreating new volume $volumeName")
95+
EngineResponseContent<Volume> volumeResponse = createVolume(volumeName)
96+
volume = volumeResponse?.content
97+
assert volume: "Error creating volume $volumeName, " + volumeResponse?.getStatus()?.text
98+
log.debug("\t\tCreated destination volume:" + volume.name)
99+
100+
92101
return volume
93102
}
94103

@@ -305,7 +314,7 @@ class DockerClientDS extends DockerClientImpl {
305314
},
306315
Duration.of(timeoutM, ChronoUnit.MINUTES),
307316
tag,
308-
new FileInputStream(buildContext)
317+
new FileInputStream(buildContext)
309318
)
310319

311320
return new ImageSummaryDS(this, this.images().content.find { it.repoTags.any { it == tag } })

src/test/groovy/com/eficode/devstack/container/impl/DirectorySyncerTest.groovy

Lines changed: 23 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -44,27 +44,27 @@ class DirectorySyncerTest extends DevStackSpec {
4444
!volumeExists(uniqueVolumeName) ?: dockerClient.rmVolume(uniqueVolumeName)
4545
log.debug("\tWill use sync to Docker volume:" + uniqueVolumeName)
4646

47-
DirectorySyncer firstSyncer = DirectorySyncer.createSyncToVolume([srcDir1.canonicalPath, srcDir2.canonicalPath], uniqueVolumeName, "DirSyncer", "-avh --delete", dockerRemoteHost, dockerCertPath )
47+
DirectorySyncer firstSyncer = DirectorySyncer.createSyncToVolume([srcDir1.canonicalPath, srcDir2.canonicalPath], uniqueVolumeName, "DirSyncer", "-avh --delete", dockerRemoteHost, dockerCertPath)
4848
log.info("\tCreated first sync container: ${firstSyncer.containerName} (${firstSyncer.shortId})")
4949
Integer containersAfterFirst = firstSyncer.dockerClient.ps(true).content.size()
5050
log.info("\t\tDocker engine now has a total of ${containersAfterFirst} contianers")
5151

5252
when: "Creating second sync container"
53-
DirectorySyncer secondSyncer = DirectorySyncer.createSyncToVolume([srcDir1.canonicalPath, srcDir2.canonicalPath], uniqueVolumeName, "DirSyncer","-avh --delete", dockerRemoteHost, dockerCertPath )
53+
DirectorySyncer secondSyncer = DirectorySyncer.createSyncToVolume([srcDir1.canonicalPath, srcDir2.canonicalPath], uniqueVolumeName, "DirSyncer", "-avh --delete", dockerRemoteHost, dockerCertPath)
5454
log.info("\tCreated second sync container: ${secondSyncer.containerName} (${secondSyncer.shortId})")
5555

5656
then: "They should have the same ID"
57-
assert firstSyncer.id == secondSyncer.id : "The second container doesnt have the same id"
58-
assert containersAfterFirst == secondSyncer.dockerClient.ps(true).content.size() : "The number of containers changed after creating the second one"
57+
assert firstSyncer.id == secondSyncer.id: "The second container doesnt have the same id"
58+
assert containersAfterFirst == secondSyncer.dockerClient.ps(true).content.size(): "The number of containers changed after creating the second one"
5959
assert secondSyncer.running
6060
log.info("\tA duplicate container was not created, instead the first one was returned")
6161

6262
when: "Stopping the sync container, and creating another duplicate"
6363
firstSyncer.stopContainer()
6464
assert !firstSyncer.running
65-
secondSyncer = DirectorySyncer.createSyncToVolume([srcDir1.canonicalPath, srcDir2.canonicalPath], uniqueVolumeName, "DirSyncer","-avh --delete", dockerRemoteHost, dockerCertPath )
65+
secondSyncer = DirectorySyncer.createSyncToVolume([srcDir1.canonicalPath, srcDir2.canonicalPath], uniqueVolumeName, "DirSyncer", "-avh --delete", dockerRemoteHost, dockerCertPath)
6666

67-
then:"The duplicate should have been automatically started"
67+
then: "The duplicate should have been automatically started"
6868
secondSyncer.running
6969

7070

@@ -75,7 +75,6 @@ class DirectorySyncerTest extends DevStackSpec {
7575
dockerClient.rmVolume(uniqueVolumeName)
7676

7777

78-
7978
}
8079

8180

@@ -96,7 +95,7 @@ class DirectorySyncerTest extends DevStackSpec {
9695
srcContainer.user = "1001:1001"
9796
srcContainer.createSleepyContainer()
9897
srcContainer.startContainer()
99-
srcContainer.runBashCommandInContainer(ImageSummaryDS.getReplaceUserScriptBody("ubuntu", "1000", "ubuntu", "1000", "ubuntusrc", "1001", "ubuntusrc", "1001"),10, "root" )
98+
srcContainer.runBashCommandInContainer(ImageSummaryDS.getReplaceUserScriptBody("ubuntu", "1000", "ubuntu", "1000", "ubuntusrc", "1001", "ubuntusrc", "1001"), 10, "root")
10099
srcContainer.runBashCommandInContainer("chown 1001:1001 -R /mnt/volume", 5, "root")
101100

102101

@@ -106,7 +105,7 @@ class DirectorySyncerTest extends DevStackSpec {
106105
destContainer.user = "1002:1002"
107106
destContainer.createSleepyContainer()
108107
destContainer.startContainer()
109-
destContainer.runBashCommandInContainer(ImageSummaryDS.getReplaceUserScriptBody("ubuntu", "1000", "ubuntu", "1000", "ubuntudest", "1002", "ubuntudest", "1002"),10, "root" )
108+
destContainer.runBashCommandInContainer(ImageSummaryDS.getReplaceUserScriptBody("ubuntu", "1000", "ubuntu", "1000", "ubuntudest", "1002", "ubuntudest", "1002"), 10, "root")
110109
destContainer.runBashCommandInContainer("chown 1002:1002 -R /mnt/volume", 5, "root")
111110

112111
then: "When checking user id, they should be different"
@@ -151,7 +150,7 @@ class DirectorySyncerTest extends DevStackSpec {
151150
when: "When creating syncer"
152151

153152
assert !volumeExists(uniqueVolumeName): "Destination volume already exists"
154-
DirectorySyncer syncer = DirectorySyncer.createSyncToVolume([srcDir1.canonicalPath, srcDir2.canonicalPath], uniqueVolumeName, "DirSyncer", "-avh --delete", dockerRemoteHost, dockerCertPath )
153+
DirectorySyncer syncer = DirectorySyncer.createSyncToVolume([srcDir1.canonicalPath, srcDir2.canonicalPath], uniqueVolumeName, "DirSyncer", "-avh --delete", dockerRemoteHost, dockerCertPath)
155154
log.info("\tCreated sync container: ${syncer.containerName} (${syncer.shortId})")
156155
ContainerInspectResponse containerInspect = syncer.inspectContainer()
157156

@@ -185,14 +184,16 @@ class DirectorySyncerTest extends DevStackSpec {
185184
log.debug("\tContainer successfully synced the files to destination dir")
186185

187186
when: "Creating a recursive file"
188-
File recursiveFile = new File(srcDir1.canonicalPath + "/subDir/subFile.temp").createParentDirectories()
187+
File recursiveFile = new File(srcDir1.canonicalPath + "/subDir/subDir2/subDir3/subDir4/subDir5/subFile.temp")
188+
recursiveFile.createParentDirectories()
189+
sleep(2000) //Make sure that we dont trigger on the dir creation, but on the recursive file
189190
recursiveFile.createNewFile()
190191
recursiveFile.text = System.nanoTime()
191192
log.info("\tCreate recursive file:" + recursiveFile.canonicalPath)
192193

193194
then: "The sync container should see the new source file, and sync it to a new recursive dir"
194195
sleep(2000)
195-
syncer.runBashCommandInContainer("cat /mnt/dest/subDir/subFile.temp").toString().contains(recursiveFile.text)
196+
syncer.runBashCommandInContainer("cat /mnt/dest/subDir/subDir2/subDir3/subDir4/subDir5/subFile.temp").toString().contains(recursiveFile.text)
196197
log.info("\t\tFile was successfully synced")
197198

198199

@@ -211,27 +212,27 @@ class DirectorySyncerTest extends DevStackSpec {
211212
syncer.runBashCommandInContainer("cat /mnt/dest/${srcFile2.name}").toString().containsIgnoreCase(srcFile2.text)
212213
log.debug("\t\tContainer successfully synced the changes")
213214

214-
when:"Creating a new container and attaching it to the synced volume"
215+
when: "Creating a new container and attaching it to the synced volume"
215216
AlpineContainer secondContainer = new AlpineContainer(dockerRemoteHost, dockerCertPath)
216-
secondContainer.containerName = syncer.containerName + "-companion"
217+
secondContainer.containerName = syncer.containerName + "-companion"
217218
secondContainer.prepareVolumeMount(uniqueVolumeName, "/mnt/syncDir", false)
218219
secondContainer.createSleepyContainer()
219220

220221

221-
then:"The second container should see the still remaining synced files"
222-
assert secondContainer.startContainer() : "Error creating/staring second container"
222+
then: "The second container should see the still remaining synced files"
223+
assert secondContainer.startContainer(): "Error creating/staring second container"
223224
log.info("\tCreated an started second container ${secondContainer.shortId}")
224-
assert secondContainer.mounts.any { it.type == MountPoint.Type.Volume && it.RW == true} : "Second container did not mount the shared volume"
225+
assert secondContainer.mounts.any { it.type == MountPoint.Type.Volume && it.RW == true }: "Second container did not mount the shared volume"
225226
log.info("\tSecond container was attached to volume:" + uniqueVolumeName)
226-
log.info("\tChecking that second container can access synced file:" + " /mnt/syncDir/${srcFile2.name}" )
227-
assert secondContainer.runBashCommandInContainer("cat /mnt/syncDir/${srcFile2.name}").toString().containsIgnoreCase(srcFile2.text) : "Error reading synced file in second container:" + " /mnt/syncDir/${srcFile2.name}"
227+
log.info("\tChecking that second container can access synced file:" + " /mnt/syncDir/${srcFile2.name}")
228+
assert secondContainer.runBashCommandInContainer("cat /mnt/syncDir/${srcFile2.name}").toString().containsIgnoreCase(srcFile2.text): "Error reading synced file in second container:" + " /mnt/syncDir/${srcFile2.name}"
228229

229230
log.info("\tChecking that second container can access recursive synced file")
230-
assert secondContainer.runBashCommandInContainer("cat /mnt/syncDir/subDir/subFile.temp").toString().contains(recursiveFile.text)
231+
assert secondContainer.runBashCommandInContainer("cat /mnt/syncDir/subDir/subDir2/subDir3/subDir4/subDir5/subFile.temp").toString().contains(recursiveFile.text)
231232
log.info("\t\tContainer can access that file")
232233
cleanup:
233-
assert syncer.stopAndRemoveContainer()
234-
assert secondContainer?.stopAndRemoveContainer()
234+
syncer.stopAndRemoveContainer()
235+
secondContainer?.stopAndRemoveContainer()
235236
srcDir1.deleteDir()
236237
srcDir2.deleteDir()
237238
dockerClient.rmVolume(containerInspect.mounts.find { it.type == MountPoint.Type.Volume }.name)

0 commit comments

Comments
 (0)