Sre 2645 fix race condition

.circleci/config.yml

@@ -19,7 +19,7 @@ jobs:
       # - image: circleci/postgres:9.4
 
         environment: 
-          PYTHON_BIN: /usr/bin/python2.7
+          PYTHON_BIN: /usr/local/bin/python2.7
     working_directory: ~/repo
 
     steps:
@@ -35,17 +35,20 @@ jobs:
       - run:
           name: set version and build the rpm
           command: |
+            set -x
             version=$(cat VERSION)
             echo "${version}.${CIRCLE_BUILD_NUM}" > VERSION
             make rpm
 
             git config user.email "dev@objectrocket.com"
             git config user.name "objectrocketdev"
             git tag -a $(cat VERSION) -m "Tagged by ${CIRCLE_BUILD_URL}" HEAD
+            rpm -qa | grep ssh
+            rpm -ql libssh2-1.4.2-2.el6_7.1.x86_64
             git push --tags origin
 
             mkdir mongodb_consistent_backup_rpm
-            cp /root/repo/build/rpm/RPMS/x86_64/mongodb_consistent_backup-$(cat VERSION)-1.el7.x86_64.rpm mongodb_consistent_backup_rpm/
+            cp /root/repo/build/rpm/RPMS/x86_64/mongodb_consistent_backup-$(cat VERSION)-1.el6.x86_64.rpm mongodb_consistent_backup_rpm/
 
       - save_cache:
           paths:

mongodb_consistent_backup/Common/DB.py

@@ -22,7 +22,7 @@ def parse_read_pref_tags(tags_str):
 
 class DB:
     def __init__(self, uri, config, do_replset=False, read_pref='primaryPreferred', do_rp_tags=False,
-                 do_connect=True, conn_timeout=5000, retries=5):
+                 do_connect=True, conn_timeout=60000, retries=5):
         self.uri            = uri
         self.config         = config
         self.do_replset     = do_replset

mongodb_consistent_backup/Oplog/Tailer/TailThread.py

@@ -146,7 +146,7 @@ def run(self):
                     sleep(1)
                 finally:
                     if self._cursor:
-                        logging.debug("Stopping oplog cursor on %s" % self.uri)
+                        logging.info("Stopping oplog cursor on %s" % self.uri)
                         self._cursor.close()
         except OperationError, e:
             logging.error("Tailer %s encountered error: %s" % (self.uri, e))

mongodb_consistent_backup/Oplog/Tailer/Tailer.py

@@ -104,7 +104,11 @@ def stop(self, kill=False, sleep_secs=3):
 
                 # wait for replication to get in sync
                 while state.get('last_ts') and state.get('last_ts') < timestamp:
-                    logging.info('Waiting for %s tailer to reach ts: %s, currrent: %s' % (uri, timestamp, state.get('last_ts')))
+                    if self.shards[shard]['thread'].exitcode:
+                      logging.info('Not waiting for %s tailer to reach ts: %s, currrent: %s. Exit code: %i' % (uri, timestamp, state.get('last_ts'),self.shards[shard]['thread'].exitcode))
+                      break
+                    else:  
+                      logging.info('Waiting for %s tailer to reach ts: %s, currrent: %s' % (uri, timestamp, state.get('last_ts')))
                     sleep(sleep_secs)
 
             # set thread stop event

scripts/mongodb_consistent_backup.spec

@@ -21,8 +21,8 @@ Prefix:		/usr
 
 # Use CentOS SCL python27 (https://www.softwarecollections.org/en/scls/rhscl/python27/) on CentOS 6 (RHEL6 untested)
 # On build host: 'yum install python27-python python27-python-devel python27-python-virtualenv gcc make libffi-devel openssl-devel'
-%{?el6:Requires: python27-python openssl-libs}
-%{?el6:BuildRequires: python27-python python27-python-devel python27-python-virtualenv gcc make libffi-devel openssl-devel}
+#%{?el6:Requires: openssl-libs}
+%{?el6:BuildRequires: gcc make libffi-devel openssl-devel}
 
 # Use base python/virtualenv, which should be 2.7 on CentOS/RHEL 7
 # On build host: 'yum install python python-devel python-virtualenv gcc make libffi-devel openssl-devel'