fn: LB agent: reduce 'Too Busy' error logs (#1033)

With this PR, runner client translates too busy errors
from gRPC session and runner itself into Fn error type.
Placers now ignore this error message to reduce unnecessary
logging.

api/agent/runner_client.go

@@ -85,7 +85,7 @@ func (r *gRPCRunner) Address() string {
 	return r.address
 }
 
-func isRetriable(err error) bool {
+func isTooBusy(err error) bool {
 	// A formal API error returned from pure-runner
 	if models.GetAPIErrorCode(err) == models.GetAPIErrorCode(models.ErrCallTimeoutServerBusy) {
 		return true
@@ -144,7 +144,11 @@ func (r *gRPCRunner) TryExec(ctx context.Context, call pool.RunnerCall) (bool, e
 		logrus.Infof("Engagement Context ended ctxErr=%v", ctx.Err())
 		return true, ctx.Err()
 	case recvErr := <-recvDone:
-		return !isRetriable(recvErr), recvErr
+		if isTooBusy(recvErr) {
+			// Try on next runner
+			return false, models.ErrCallTimeoutServerBusy
+		}
+		return true, recvErr
 	}
 }
 

api/runnerpool/ch_placer.go

@@ -52,7 +52,7 @@ func (p *chPlacer) PlaceCall(rp RunnerPool, ctx context.Context, call RunnerCall
 				placed, err := r.TryExec(tryCtx, call)
 				tryCancel()
 
-				if err != nil {
+				if err != nil && err != models.ErrCallTimeoutServerBusy {
 					logrus.WithError(err).Error("Failed during call placement")
 				}
 				if placed {

api/runnerpool/naive_placer.go

@@ -46,7 +46,7 @@ func (sp *naivePlacer) PlaceCall(rp RunnerPool, ctx context.Context, call Runner
 				placed, err := r.TryExec(tryCtx, call)
 				tryCancel()
 
-				if err != nil {
+				if err != nil && err != models.ErrCallTimeoutServerBusy {
 					logrus.WithError(err).Error("Failed during call placement")
 				}
 				if placed {