fn: added server too busy stats (#717)

api/agent/agent.go

@@ -227,7 +227,7 @@ func transformTimeout(e error, isRetriable bool) error {
 func (a *agent) handleStatsDequeue(ctx context.Context, call *call, err error) {
 	if err == context.DeadlineExceeded {
 		a.stats.Dequeue(ctx, call.AppName, call.Path)
-		// note that this is not a timeout from the perspective of the caller, so don't increment the timeout count
+		a.stats.IncrementTooBusy(ctx)
 	} else {
 		a.stats.DequeueAndFail(ctx, call.AppName, call.Path)
 		a.stats.IncrementErrors(ctx)

api/agent/stats.go

@@ -149,6 +149,10 @@ func (s *stats) IncrementErrors(ctx context.Context) {
 	common.IncrementCounter(ctx, errorsMetricName)
 }
 
+func (s *stats) IncrementTooBusy(ctx context.Context) {
+	common.IncrementCounter(ctx, serverBusyMetricName)
+}
+
 func (s *stats) Stats() Stats {
 	var stats Stats
 	s.mu.Lock()
@@ -166,11 +170,12 @@ func (s *stats) Stats() Stats {
 }
 
 const (
-	queuedMetricName    = "queued"
-	callsMetricName     = "calls"
-	runningMetricName   = "running"
-	completedMetricName = "completed"
-	failedMetricName    = "failed"
-	timedoutMetricName  = "timeouts"
-	errorsMetricName    = "errors"
+	queuedMetricName     = "queued"
+	callsMetricName      = "calls"
+	runningMetricName    = "running"
+	completedMetricName  = "completed"
+	failedMetricName     = "failed"
+	timedoutMetricName   = "timeouts"
+	errorsMetricName     = "errors"
+	serverBusyMetricName = "server_busy"
 )