Implemented graceful shutdown for HTTP requests (close #2698) (#2717)

* Listens for SIGTERM as the termination signal
* Stops accepting new connections once the signal is received
* Waits for all connections to be drained, before shutting down
* Forcefully kills all pending connections after 30 seconds

Currently this does not send a close message to websocket clients, I'd
like to submit that change as a separate pull request, but at least this
solve my biggest concern which is not getting confirmation for mutations
while restarting the server.
This commit is contained in:
José Lorenzo Rodríguez 2019-08-26 07:31:27 +02:00 committed by Alexis King
parent 4bdf965d87
commit c7a2320456
9 changed files with 29 additions and 11 deletions

View File

@ -36,7 +36,7 @@ wait_for_port 8080
HASURA_GRAPHQL_TEST_ENDPOINT="http://localhost:8080" make test
# kill the running server
kill $PID
kill -s INT $PID
# start graphql-engine with admin secret
psql -U gql_test -h localhost -c 'CREATE DATABASE "gql_test_with_admin_secret";'
@ -48,4 +48,4 @@ wait_for_port 8080
# test cli
GOCACHE=off HASURA_GRAPHQL_TEST_ENDPOINT="http://localhost:8080" HASURA_GRAPHQL_TEST_ADMIN_SECRET="abcd" make test
kill $PID
kill -s INT $PID

View File

@ -30,7 +30,7 @@ wait_for_port 8080
# test cli
HASURA_GRAPHQL_TEST_ENDPOINT="http://localhost:8080" make test
kill $PID
kill -s INT $PID
# start graphql-engine with admin secret
psql -U gql_test -h localhost -c 'CREATE DATABASE "gql_test_with_admin_secret";'
@ -42,4 +42,4 @@ wait_for_port 8080
# test cli
GOCACHE=off HASURA_GRAPHQL_TEST_ENDPOINT="http://localhost:8080" HASURA_GRAPHQL_TEST_ADMIN_SECRET="abcd" make test
kill $PID
kill -s INT $PID

View File

@ -66,7 +66,7 @@ run_hge_with_flags() {
}
kill_hge() {
kill $HGE_PID || true
kill -s INT $HGE_PID || true
wait $HGE_PID || true
}

View File

@ -66,7 +66,7 @@ run_hge_with_flags() {
}
kill_hge() {
kill $HGE_PID || true
kill -s INT $HGE_PID || true
wait $HGE_PID || true
}

View File

@ -4,9 +4,9 @@ set -euo pipefail
### Functions
stop_services() {
kill -INT $HGE_PIDS || true
kill -s INT $HGE_PIDS || true
kill $WH_PID || true
kill -INT $WHC_PID || true
kill -s INT $WHC_PID || true
}
time_elapsed(){
@ -94,7 +94,7 @@ combine_all_hpc_reports() {
}
kill_hge_servers() {
kill -INT $HGE_PIDS || true
kill -s INT $HGE_PIDS || true
wait $HGE_PIDS || true
HGE_PIDS=""
}

3
.gitignore vendored
View File

@ -7,3 +7,6 @@ test-server-output
test-server-flags-output
.vscode
.idea
# Test artifacts
*.tix

View File

@ -359,6 +359,7 @@ executable graphql-engine
, wreq
, string-conversions
, uuid
, unix
other-modules: Ops
, Migrate

View File

@ -23,5 +23,4 @@ RUN upx /packaging/build/rootfs/bin/${project}
# Stage 3: copy the rootfs into a scratch container
FROM scratch
STOPSIGNAL SIGINT
COPY --from=packager /packaging/build/rootfs /

View File

@ -21,6 +21,7 @@ import qualified Data.Yaml as Y
import qualified Network.HTTP.Client as HTTP
import qualified Network.HTTP.Client.TLS as HTTP
import qualified Network.Wai.Handler.Warp as Warp
import qualified System.Posix.Signals as Signals
import Hasura.Db
import Hasura.Events.Lib
@ -161,7 +162,11 @@ main = do
startSchemaSync sqlGenCtx pool logger httpManager
cacheRef instanceId cacheInitTime
let warpSettings = Warp.setPort port $ Warp.setHost host Warp.defaultSettings
let warpSettings = Warp.setPort port
. Warp.setHost host
. Warp.setGracefulShutdownTimeout (Just 30) -- 30s graceful shutdown
. Warp.setInstallShutdownHandler (shutdownHandler logger)
$ Warp.defaultSettings
maxEvThrds <- getFromEnv defaultMaxEventThreads "HASURA_GRAPHQL_EVENTS_HTTP_POOL_SIZE"
evFetchMilliSec <- getFromEnv defaultFetchIntervalMilliSec "HASURA_GRAPHQL_EVENTS_FETCH_INTERVAL"
@ -273,6 +278,16 @@ main = do
cleanSuccess =
putStrLn "successfully cleaned graphql-engine related data"
-- | Catches the SIGTERM signal and initiates a graceful shutdown. Graceful shutdown for regular HTTP
-- requests is already implemented in Warp, and is triggered by invoking the 'closeSocket' callback.
-- We only catch the SIGTERM signal once, that is, if the user hits CTRL-C once again, we terminate
-- the process immediately.
shutdownHandler :: Logger -> IO () -> IO ()
shutdownHandler (Logger logger) closeSocket =
void $ Signals.installHandler Signals.sigTERM (Signals.CatchOnce $ closeSocket >> logShutdown) Nothing
where
logShutdown = logger $
mkGenericStrLog LevelInfo "server" "gracefully shutting down server"
telemetryNotice :: String
telemetryNotice =