Implemented graceful shutdown for HTTP requests (close #2698) (#2717)

* Listens for SIGTERM as the termination signal
* Stops accepting new connections once the signal is received
* Waits for all connections to be drained, before shutting down
* Forcefully kills all pending connections after 30 seconds

Currently this does not send a close message to websocket clients, I'd
like to submit that change as a separate pull request, but at least this
solve my biggest concern which is not getting confirmation for mutations
while restarting the server.

.circleci/test-cli-with-last-release.sh

@@ -36,7 +36,7 @@ wait_for_port 8080
 HASURA_GRAPHQL_TEST_ENDPOINT="http://localhost:8080" make test
 
 # kill the running server
-kill $PID
+kill -s INT $PID
 
 # start graphql-engine with admin secret
 psql -U gql_test -h localhost -c 'CREATE DATABASE "gql_test_with_admin_secret";'
@@ -48,4 +48,4 @@ wait_for_port 8080
 
 # test cli
 GOCACHE=off HASURA_GRAPHQL_TEST_ENDPOINT="http://localhost:8080" HASURA_GRAPHQL_TEST_ADMIN_SECRET="abcd" make test
-kill $PID
+kill -s INT $PID

.circleci/test-cli.sh

@@ -30,7 +30,7 @@ wait_for_port 8080
 
 # test cli
 HASURA_GRAPHQL_TEST_ENDPOINT="http://localhost:8080" make test
-kill $PID
+kill -s INT $PID
 
 # start graphql-engine with admin secret
 psql -U gql_test -h localhost -c 'CREATE DATABASE "gql_test_with_admin_secret";'
@@ -42,4 +42,4 @@ wait_for_port 8080
 
 # test cli
 GOCACHE=off HASURA_GRAPHQL_TEST_ENDPOINT="http://localhost:8080" HASURA_GRAPHQL_TEST_ADMIN_SECRET="abcd" make test
-kill $PID
+kill -s INT $PID

.circleci/test-deprecated-server-flags.sh

@@ -66,7 +66,7 @@ run_hge_with_flags() {
 }
 
 kill_hge() {
-  kill $HGE_PID || true
+  kill -s INT $HGE_PID || true
   wait $HGE_PID || true
 }
 

.circleci/test-server-flags.sh

@@ -66,7 +66,7 @@ run_hge_with_flags() {
 }
 
 kill_hge() {
-  kill $HGE_PID || true
+  kill -s INT $HGE_PID || true
   wait $HGE_PID || true
 }
 

.circleci/test-server.sh

@@ -4,9 +4,9 @@ set -euo pipefail
 ### Functions
 
 stop_services() {
-   kill -INT $HGE_PIDS || true
+   kill -s INT $HGE_PIDS || true
    kill $WH_PID || true
-   kill -INT $WHC_PID || true
+   kill -s INT $WHC_PID || true
 }
 
 time_elapsed(){
@@ -94,7 +94,7 @@ combine_all_hpc_reports() {
 }
 
 kill_hge_servers() {
-	kill -INT $HGE_PIDS || true
+	kill -s INT $HGE_PIDS || true
 	wait $HGE_PIDS || true
 	HGE_PIDS=""
 }

.gitignore

@@ -7,3 +7,6 @@ test-server-output
 test-server-flags-output
 .vscode
 .idea
+
+# Test artifacts
+*.tix

server/graphql-engine.cabal

@@ -359,6 +359,7 @@ executable graphql-engine
                      , wreq
                      , string-conversions
                      , uuid
+                     , unix
 
   other-modules:       Ops
                      , Migrate

server/packaging/Dockerfile

@@ -23,5 +23,4 @@ RUN upx /packaging/build/rootfs/bin/${project}
 
 # Stage 3: copy the rootfs into a scratch container
 FROM scratch
-STOPSIGNAL SIGINT
 COPY --from=packager /packaging/build/rootfs /

server/src-exec/Main.hs

@@ -21,6 +21,7 @@ import qualified Data.Yaml                  as Y
 import qualified Network.HTTP.Client        as HTTP
 import qualified Network.HTTP.Client.TLS    as HTTP
 import qualified Network.Wai.Handler.Warp   as Warp
+import qualified System.Posix.Signals       as Signals
 
 import           Hasura.Db
 import           Hasura.Events.Lib
@@ -161,7 +162,11 @@ main =  do
       startSchemaSync sqlGenCtx pool logger httpManager
                       cacheRef instanceId cacheInitTime
 
-      let warpSettings = Warp.setPort port $ Warp.setHost host Warp.defaultSettings
+      let warpSettings = Warp.setPort port
+                       . Warp.setHost host
+                       . Warp.setGracefulShutdownTimeout (Just 30) -- 30s graceful shutdown
+                       . Warp.setInstallShutdownHandler (shutdownHandler logger)
+                       $ Warp.defaultSettings
 
       maxEvThrds <- getFromEnv defaultMaxEventThreads "HASURA_GRAPHQL_EVENTS_HTTP_POOL_SIZE"
       evFetchMilliSec  <- getFromEnv defaultFetchIntervalMilliSec "HASURA_GRAPHQL_EVENTS_FETCH_INTERVAL"
@@ -273,6 +278,16 @@ main =  do
     cleanSuccess =
       putStrLn "successfully cleaned graphql-engine related data"
 
+    -- | Catches the SIGTERM signal and initiates a graceful shutdown. Graceful shutdown for regular HTTP
+    -- requests is already implemented in Warp, and is triggered by invoking the 'closeSocket' callback.
+    -- We only catch the SIGTERM signal once, that is, if the user hits CTRL-C once again, we terminate
+    -- the process immediately.
+    shutdownHandler :: Logger -> IO () -> IO ()
+    shutdownHandler  (Logger logger) closeSocket =
+      void $ Signals.installHandler Signals.sigTERM (Signals.CatchOnce $ closeSocket >> logShutdown) Nothing
+     where
+      logShutdown = logger $
+        mkGenericStrLog LevelInfo "server" "gracefully shutting down server"
 
 telemetryNotice :: String
 telemetryNotice =