Fix admin logout issue when overseer is dead (NVIDIA#628)

Add NVFL_OVERSEER_HEARTBEAT_TIMEOUT env var

nvflare/ha/overseer/overseer.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import os
 from datetime import datetime
 
 from flask import jsonify, request
@@ -29,6 +30,11 @@
 )
 
 privilege_dict = load_privilege()
+heartbeat_timeout = os.environ.get("NVFL_OVERSEER_HEARTBEAT_TIMEOUT", "10")
+try:
+    heartbeat_timeout = int(heartbeat_timeout)
+except BaseException:
+    heartbeat_timeout = 10
 
 
 @app.route("/api/v1/heartbeat", methods=["GET", "POST"])
@@ -40,7 +46,7 @@ def heartbeat():
         if project is None or role is None:
             return jsonify({"Error": "project and role must be provided"})
         now = datetime.utcnow()
-        update_sp_state(project, now)
+        update_sp_state(project, now, heartbeat_timeout=heartbeat_timeout)
         if role == "server":
             sp_end_point = req.get("sp_end_point")
             if sp_end_point is None:

nvflare/ha/overseer_agent.py

@@ -102,6 +102,7 @@ def resume(self):
         self._flag.set()
 
     def end(self):
+        self._asked_to_stop_retrying = True
         self._flag.set()
         self._asked_to_exit = True
         self._report_and_query.join()

nvflare/lighter/impl/master_template.yml

@@ -236,7 +236,7 @@ log_config: |
 start_ovsr_sh: |
   #!/usr/bin/env bash
   DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-  AUTHZ_FILE=$DIR/privilege.yml gunicorn -c $DIR/gunicorn.conf.py --keyfile $DIR/overseer.key --certfile $DIR/overseer.crt --ca-certs $DIR/rootCA.pem
+  NVFL_OVERSEER_HEARTBEAT_TIMEOUT=10 AUTHZ_FILE=$DIR/privilege.yml gunicorn -c $DIR/gunicorn.conf.py --keyfile $DIR/overseer.key --certfile $DIR/overseer.crt --ca-certs $DIR/rootCA.pem
 
 start_cln_sh: |
   #!/usr/bin/env bash