Orphan volume on delete if we get a conflict

cattle/plugins/docker/storage.py

@@ -242,6 +242,9 @@ def _is_managed_volume(self, volume):
         return True
 
     def _do_volume_remove(self, volume, storage_pool, progress):
+        if self._is_volume_removed(volume, storage_pool):
+            return
+
         if volume.deviceNumber == 0:
             container = get_compute().get_container(docker_client(),
                                                     volume.instance)
@@ -250,7 +253,14 @@ def _do_volume_remove(self, volume, storage_pool, progress):
             remove_container(docker_client(), container)
         elif self._is_managed_volume(volume):
             v = DockerConfig.storage_api_version()
-            docker_client(version=v).remove_volume(volume.name)
+            try:
+                docker_client(version=v).remove_volume(volume.name)
+            except APIError as e:
+                if e.message.response.status_code == 409:
+                    log.warn('Encountered conflict (%s) while deleting '
+                             'volume. Orphaning volume.', e)
+                else:
+                    raise e
         else:
             path = self._path_to_volume(volume)
             if not volume.data.fields['isHostPath']:

cattle/storage/__init__.py

@@ -63,7 +63,6 @@ def volume_remove(self, req=None, volumeStoragePoolMap=None, **kw):
 
         return self._do(
             req=req,
-            check=lambda: self._is_volume_removed(volume, storage_pool),
             result=lambda: self._get_response_data(req, volumeStoragePoolMap),
             lock_obj=volume,
             action=lambda: self._do_volume_remove(volume, storage_pool,

tests/docker_common.py

@@ -12,6 +12,7 @@
 from cattle import CONFIG_OVERRIDE, Config
 from .common_fixtures import TEST_DIR
 from docker.utils import compare_version
+from cattle.plugins.docker import DockerConfig
 
 CONFIG_OVERRIDE['DOCKER_REQUIRED'] = 'false'  # NOQA
 CONFIG_OVERRIDE['DOCKER_HOST_IP'] = '1.2.3.4'  # NOQA
@@ -125,6 +126,14 @@ def remove_state_file(container):
             pass
 
 
+def delete_volume(name):
+    client = docker_client(version=DockerConfig.storage_api_version())
+    try:
+        client.remove_volume(name)
+    except:
+        pass
+
+
 def delete_container(name):
     client = docker_client()
     for c in client.containers(all=True):

tests/test_docker.py

@@ -1559,6 +1559,39 @@ def post(req, resp):
                       pre_func=pre, post_func=post, diff=False)
 
 
+@if_docker
+def test_volume_delete_orphaning(agent, responses, request):
+    # This test emulates the situatoin we've seen in docker 1.10 where we need
+    # to delete a volume but docker's ref count is off and it won't let us
+    # delete it. In this scenario, we'll now just orphan the volume and return
+    # success
+    delete_container('/orphan_test')
+    vol_name = 'orphan_test_vol'
+    delete_volume(vol_name)
+
+    v = DockerConfig.storage_api_version()
+    docker_client(version=v).create_volume(vol_name, 'local')
+
+    client = docker_client()
+    c = client.create_container('ibuildthecloud/helloworld',
+                                name='orphan_test',
+                                host_config=client.create_host_config(
+                                    binds=['%s:/tmp/1' % vol_name]))
+    client.start(c)
+
+    def pre(req):
+        vol = req['data']['volumeStoragePoolMap']['volume']
+        vol['name'] = vol_name
+        vol['data'] = {'fields': {'driver': 'local'}}
+        vol['uri'] = 'local:///%s' % vol_name
+
+    def post(req, resp):
+        found_vol = docker_client(version=v).inspect_volume(vol_name)
+        assert found_vol is not None
+
+    event_test(agent, 'docker/volume_remove', pre_func=pre, post_func=post)
+
+
 @if_docker
 def test_volume_from_data_volume_mounts_with_opt(agent, responses, request):
     driver_opts = JsonObject(