Check runner connection before sending requests (#831)

If a runner disconnect not gracefully it could happen that the
connection gets stuck in connecting mode, this change verifies the state
of the connection before starting to execute a call, if the client
connection is not ready we fail fast to give a change to the next runner
(if any) to execute the call.

api/agent/nodepool/grpc/grpc_pool.go

@@ -9,6 +9,7 @@ import (
 	"time"
 
 	"google.golang.org/grpc"
+	"google.golang.org/grpc/connectivity"
 	"google.golang.org/grpc/credentials"
 
 	"github.com/fnproject/fn/api/agent"
@@ -281,6 +282,15 @@ func (r *gRPCRunner) TryExec(ctx context.Context, call agent.Call) (bool, error)
 	logrus.WithField("runner_addr", r.address).Debug("Attempting to place call")
 	r.wg.Add(1)
 	defer r.wg.Done()
+	// If the connection is not READY, we want to fail-fast.
+	// There are some cases where the connection stays in CONNECTING, for example if the
+	// runner dies not gracefully (e.g. unplug the cable), the connection get stuck until
+	// the context times out.
+	// https://github.com/grpc/grpc/blob/master/doc/connectivity-semantics-and-api.md
+	if r.conn.GetState() != connectivity.Ready {
+		logrus.WithField("runner_address", r.address).Debug("Runner connection is not ready")
+		return false, nil
+	}
 
 	// Get app and route information
 	// Construct model.Call with CONFIG in it already