Skip to content

Commit

Permalink
Do not scale down deployment before resource deletion. Removing watch…
Browse files Browse the repository at this point in the history
… pod logic (seems to be also broken after 3.6 osio update)

Signed-off-by: Ilya Buziuk <ibuziuk@redhat.com>
  • Loading branch information
ibuziuk committed Aug 16, 2017
1 parent 7ecf17b commit 52b878e
Showing 1 changed file with 17 additions and 62 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,6 @@

import java.io.IOException;
import java.util.List;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;

import javax.inject.Singleton;

Expand All @@ -23,16 +21,9 @@
import org.slf4j.LoggerFactory;

import io.fabric8.kubernetes.api.model.Pod;
import io.fabric8.kubernetes.api.model.PodList;
import io.fabric8.kubernetes.api.model.Service;
import io.fabric8.kubernetes.api.model.extensions.Deployment;
import io.fabric8.kubernetes.api.model.extensions.DoneableDeployment;
import io.fabric8.kubernetes.api.model.extensions.ReplicaSet;
import io.fabric8.kubernetes.client.KubernetesClientException;
import io.fabric8.kubernetes.client.Watch;
import io.fabric8.kubernetes.client.Watcher;
import io.fabric8.kubernetes.client.dsl.FilterWatchListDeletable;
import io.fabric8.kubernetes.client.dsl.ScalableResource;
import io.fabric8.openshift.api.model.Route;
import io.fabric8.openshift.client.DefaultOpenShiftClient;
import io.fabric8.openshift.client.OpenShiftClient;
Expand All @@ -41,26 +32,14 @@
public class OpenShiftDeploymentCleaner {
private static final Logger LOG = LoggerFactory.getLogger(OpenShiftDeploymentCleaner.class);
private static final int OPENSHIFT_POD_DELETION_TIMEOUT = 120;
private static final int OPENSHIFT_WAIT_POD_DELAY = 1000;


public void cleanDeploymentResources(final String deploymentName, final String namespace) throws IOException {
scaleDownDeployment(deploymentName, namespace);
cleanUpWorkspaceResources(deploymentName, namespace);
waitUntilWorkspacePodIsDeleted(deploymentName, namespace);
}

private void scaleDownDeployment(String deploymentName, final String namespace) throws OpenShiftException {
try (OpenShiftClient openShiftClient = new DefaultOpenShiftClient()) {
ScalableResource<Deployment, DoneableDeployment> deployment = openShiftClient.extensions()
.deployments()
.inNamespace(namespace)
.withName(deploymentName);

if (deployment != null) {
deployment.scale(0, true);
}
}
}

private void cleanUpWorkspaceResources(final String deploymentName, final String namespace) throws IOException {
Deployment deployment = KubernetesResourceUtil.getDeploymentByName(deploymentName, namespace);
Service service = KubernetesResourceUtil.getServiceBySelector(OpenShiftConnector.OPENSHIFT_DEPLOYMENT_LABEL, deploymentName, namespace);
Expand Down Expand Up @@ -92,50 +71,26 @@ private void cleanUpWorkspaceResources(final String deploymentName, final String
}
}
}

private void waitUntilWorkspacePodIsDeleted(final String deploymentName, final String namespace) throws OpenShiftException {
try (OpenShiftClient client = new DefaultOpenShiftClient()) {
FilterWatchListDeletable<Pod, PodList, Boolean, Watch, Watcher<Pod>> pods = client.pods()
.inNamespace(namespace)
.withLabel(OpenShiftConnector.OPENSHIFT_DEPLOYMENT_LABEL, deploymentName);

int numberOfPodsToStop = pods.list().getItems().size();
LOG.info("Number of workspace pods to stop {}", numberOfPodsToStop);
if (numberOfPodsToStop > 0) {
final CountDownLatch podCount = new CountDownLatch(numberOfPodsToStop);
pods.watch(new Watcher<Pod>() {
@Override
public void eventReceived(Action action, Pod pod) {
try {
switch (action) {
case ADDED:
case MODIFIED:
case ERROR:
break;
case DELETED:
LOG.info("Pod {} deleted", pod.getMetadata().getName());
podCount.countDown();
break;
}
} catch (Exception e) {
LOG.error("Failed to process {} on Pod {}. Error: ", action, pod, e);
}
}

@Override
public void onClose(KubernetesClientException ex) {
}
});

try {
LOG.info("Waiting for all pods to be deleted for deployment '{}'", deploymentName);
podCount.await(OPENSHIFT_POD_DELETION_TIMEOUT, TimeUnit.SECONDS);
} catch (InterruptedException e) {
LOG.error("Exception while waiting for pods to be deleted", e);
throw new OpenShiftException("Timeout while waiting for pods to terminate", e);
for (int waitCount = 0; waitCount < OPENSHIFT_POD_DELETION_TIMEOUT; waitCount++) {
List<Pod> pods = client.pods().inNamespace(namespace)
.withLabel(OpenShiftConnector.OPENSHIFT_DEPLOYMENT_LABEL, deploymentName)
.list()
.getItems();

if (pods.size() == 0) {
return;
}
Thread.sleep(OPENSHIFT_WAIT_POD_DELAY);
}
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
LOG.info("Thread interrupted while cleaning up workspace");
}

throw new OpenShiftException("Timeout while waiting for pods to terminate");
}

}

0 comments on commit 52b878e

Please sign in to comment.