fn: pure-runner time out while waiting TryCall (#1006)

This should return a retriable error code 503.

api/agent/pure_runner.go

@@ -23,9 +23,11 @@ import (
 	"github.com/golang/protobuf/ptypes/empty"
 	"github.com/sirupsen/logrus"
 	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/credentials"
 	"google.golang.org/grpc/metadata"
 	"google.golang.org/grpc/peer"
+	"google.golang.org/grpc/status"
 )
 
 /*
@@ -420,6 +422,10 @@ func (ch *callHandle) getTryMsg() *runner.TryCall {
 	select {
 	case <-ch.doneQueue:
 	case <-ch.ctx.Done():
+		// if ctx timed out while waiting, then this is a 503 (retriable)
+		err := status.Errorf(codes.Code(models.ErrCallTimeoutServerBusy.Code()), models.ErrCallTimeoutServerBusy.Error())
+		ch.shutdown(err)
+		return nil
 	case item := <-ch.inQueue:
 		if item != nil {
 			msg = item.GetTry()

api/agent/runner_client.go

@@ -11,6 +11,7 @@ import (
 
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials"
+	"google.golang.org/grpc/status"
 
 	pb "github.com/fnproject/fn/api/agent/grpc"
 	"github.com/fnproject/fn/api/common"
@@ -86,7 +87,18 @@ func (r *gRPCRunner) Address() string {
 }
 
 func isRetriable(err error) bool {
-	return models.GetAPIErrorCode(err) == models.GetAPIErrorCode(models.ErrCallTimeoutServerBusy)
+	// A formal API error returned from pure-runner
+	if models.GetAPIErrorCode(err) == models.GetAPIErrorCode(models.ErrCallTimeoutServerBusy) {
+		return true
+	}
+	if err != nil {
+		// engagement/recv errors could also be a 503.
+		st := status.Convert(err)
+		if int(st.Code()) == models.GetAPIErrorCode(models.ErrCallTimeoutServerBusy) {
+			return true
+		}
+	}
+	return false
 }
 
 func (r *gRPCRunner) TryExec(ctx context.Context, call pool.RunnerCall) (bool, error) {