From 43547a572fe58420221281bb9fc204be5dae4c98 Mon Sep 17 00:00:00 2001 From: Andrea Rosa Date: Mon, 12 Mar 2018 09:38:27 +0000 Subject: [PATCH] Check runner connection before sending requests (#831) If a runner disconnect not gracefully it could happen that the connection gets stuck in connecting mode, this change verifies the state of the connection before starting to execute a call, if the client connection is not ready we fail fast to give a change to the next runner (if any) to execute the call. --- api/agent/nodepool/grpc/grpc_pool.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/api/agent/nodepool/grpc/grpc_pool.go b/api/agent/nodepool/grpc/grpc_pool.go index 664ee71c0..f9fdcb1a6 100644 --- a/api/agent/nodepool/grpc/grpc_pool.go +++ b/api/agent/nodepool/grpc/grpc_pool.go @@ -9,6 +9,7 @@ import ( "time" "google.golang.org/grpc" + "google.golang.org/grpc/connectivity" "google.golang.org/grpc/credentials" "github.com/fnproject/fn/api/agent" @@ -281,6 +282,15 @@ func (r *gRPCRunner) TryExec(ctx context.Context, call agent.Call) (bool, error) logrus.WithField("runner_addr", r.address).Debug("Attempting to place call") r.wg.Add(1) defer r.wg.Done() + // If the connection is not READY, we want to fail-fast. + // There are some cases where the connection stays in CONNECTING, for example if the + // runner dies not gracefully (e.g. unplug the cable), the connection get stuck until + // the context times out. + // https://github.com/grpc/grpc/blob/master/doc/connectivity-semantics-and-api.md + if r.conn.GetState() != connectivity.Ready { + logrus.WithField("runner_address", r.address).Debug("Runner connection is not ready") + return false, nil + } // Get app and route information // Construct model.Call with CONFIG in it already