Fix port already bound error in integration tests (#368)

* clean up socket and increase port range

* format code

* revert format change

* add shutdown argument

* Removed incorrect shutdown call

* more tries

* Catch Docker port-bound exception and start clipper in a retry loop

* format code

* Removed commented out code

integration-tests/test_utils.py

@@ -32,7 +32,7 @@ def __str__(self):
 
 
 # range of ports where available ports can be found
-PORT_RANGE = [34256, 40000]
+PORT_RANGE = [34256, 50000]
 
 
 def get_docker_client():
@@ -51,9 +51,12 @@ def find_unbound_port():
         sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
         try:
             sock.bind(("127.0.0.1", port))
+            # Make sure we clean up after binding
+            del sock
             return port
-        except socket.error:
-            logger.debug(
+        except socket.error as e:
+            logger.info("Socket error: {}".format(e))
+            logger.info(
                 "randomly generated port %d is bound. Trying again." % port)
 
 
@@ -70,9 +73,24 @@ def create_docker_connection(cleanup=True, start_clipper=True):
         docker_client = get_docker_client()
         docker_client.containers.prune(filters={"label": CLIPPER_DOCKER_LABEL})
     if start_clipper:
-        logging.info("Starting Clipper")
-        cl.start_clipper()
-        time.sleep(1)
+        # Try to start Clipper in a retry loop here to address flaky tests
+        # as described in https://github.com/ucbrise/clipper/issues/352
+        while True:
+            try:
+                logging.info("Starting Clipper")
+                cl.start_clipper()
+                time.sleep(1)
+                break
+            except docker.errors.APIError as e:
+                logging.info(
+                    "Problem starting Clipper: {}\nTrying again.".format(e))
+                cl.stop_all()
+                cm = DockerContainerManager(
+                    clipper_query_port=find_unbound_port(),
+                    clipper_management_port=find_unbound_port(),
+                    clipper_rpc_port=find_unbound_port(),
+                    redis_port=find_unbound_port())
+                cl = ClipperConnection(cm)
     else:
         cl.connect()
     return cl