From 1cd5894f419974e4d5c9852e78c69844ebf7777b Mon Sep 17 00:00:00 2001
From: Tolga Ceylan <tolga.ceylan@gmail.com>
Date: Mon, 4 Jun 2018 12:16:00 -0700
Subject: [PATCH] fn: LB agent: reduce 'Too Busy' error logs (#1033)

With this PR, runner client translates too busy errors
from gRPC session and runner itself into Fn error type.
Placers now ignore this error message to reduce unnecessary
logging.
---
 api/agent/runner_client.go     | 8 ++++++--
 api/runnerpool/ch_placer.go    | 2 +-
 api/runnerpool/naive_placer.go | 2 +-
 3 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/api/agent/runner_client.go b/api/agent/runner_client.go
index 6c567305b..6e54744f6 100644
--- a/api/agent/runner_client.go
+++ b/api/agent/runner_client.go
@@ -85,7 +85,7 @@ func (r *gRPCRunner) Address() string {
 	return r.address
 }
 
-func isRetriable(err error) bool {
+func isTooBusy(err error) bool {
 	// A formal API error returned from pure-runner
 	if models.GetAPIErrorCode(err) == models.GetAPIErrorCode(models.ErrCallTimeoutServerBusy) {
 		return true
@@ -144,7 +144,11 @@ func (r *gRPCRunner) TryExec(ctx context.Context, call pool.RunnerCall) (bool, e
 		logrus.Infof("Engagement Context ended ctxErr=%v", ctx.Err())
 		return true, ctx.Err()
 	case recvErr := <-recvDone:
-		return !isRetriable(recvErr), recvErr
+		if isTooBusy(recvErr) {
+			// Try on next runner
+			return false, models.ErrCallTimeoutServerBusy
+		}
+		return true, recvErr
 	}
 }
 
diff --git a/api/runnerpool/ch_placer.go b/api/runnerpool/ch_placer.go
index 7ed197754..3f4ac002a 100644
--- a/api/runnerpool/ch_placer.go
+++ b/api/runnerpool/ch_placer.go
@@ -52,7 +52,7 @@ func (p *chPlacer) PlaceCall(rp RunnerPool, ctx context.Context, call RunnerCall
 				placed, err := r.TryExec(tryCtx, call)
 				tryCancel()
 
-				if err != nil {
+				if err != nil && err != models.ErrCallTimeoutServerBusy {
 					logrus.WithError(err).Error("Failed during call placement")
 				}
 				if placed {
diff --git a/api/runnerpool/naive_placer.go b/api/runnerpool/naive_placer.go
index 42bca9d35..9550b4c40 100644
--- a/api/runnerpool/naive_placer.go
+++ b/api/runnerpool/naive_placer.go
@@ -46,7 +46,7 @@ func (sp *naivePlacer) PlaceCall(rp RunnerPool, ctx context.Context, call Runner
 				placed, err := r.TryExec(tryCtx, call)
 				tryCancel()
 
-				if err != nil {
+				if err != nil && err != models.ErrCallTimeoutServerBusy {
 					logrus.WithError(err).Error("Failed during call placement")
 				}
 				if placed {