diff --git a/api/agent/agent.go b/api/agent/agent.go index 910be422f..2e1f24358 100644 --- a/api/agent/agent.go +++ b/api/agent/agent.go @@ -227,8 +227,10 @@ func transformTimeout(e error, isRetriable bool) error { func (a *agent) handleStatsDequeue(ctx context.Context, call *call, err error) { if err == context.DeadlineExceeded { a.stats.Dequeue(ctx, call.AppName, call.Path) + // note that this is not a timeout from the perspective of the caller, so don't increment the timeout count } else { a.stats.DequeueAndFail(ctx, call.AppName, call.Path) + a.stats.IncrementErrors(ctx) } } @@ -240,6 +242,12 @@ func (a *agent) handleStatsEnd(ctx context.Context, call *call, err error) { } else { // decrement running count, increment failed count a.stats.Failed(ctx, call.AppName, call.Path) + // increment the timeout or errors count, as appropriate + if err == context.DeadlineExceeded { + a.stats.IncrementTimedout(ctx) + } else { + a.stats.IncrementErrors(ctx) + } } } diff --git a/api/agent/stats.go b/api/agent/stats.go index 645a574f1..c1074d285 100644 --- a/api/agent/stats.go +++ b/api/agent/stats.go @@ -94,7 +94,7 @@ func (s *stats) DequeueAndStart(ctx context.Context, app string, path string) { s.running++ s.getStatsForFunction(path).running++ - common.IncrementGauge(ctx, runningSuffix) + common.IncrementGauge(ctx, runningMetricName) s.mu.Unlock() } @@ -104,7 +104,7 @@ func (s *stats) Complete(ctx context.Context, app string, path string) { s.running-- s.getStatsForFunction(path).running-- - common.DecrementGauge(ctx, runningSuffix) + common.DecrementGauge(ctx, runningMetricName) s.complete++ s.getStatsForFunction(path).complete++ @@ -118,7 +118,7 @@ func (s *stats) Failed(ctx context.Context, app string, path string) { s.running-- s.getStatsForFunction(path).running-- - common.DecrementGauge(ctx, runningSuffix) + common.DecrementGauge(ctx, runningMetricName) s.failed++ s.getStatsForFunction(path).failed++ @@ -141,6 +141,14 @@ func (s *stats) DequeueAndFail(ctx context.Context, app string, path string) { s.mu.Unlock() } +func (s *stats) IncrementTimedout(ctx context.Context) { + common.IncrementCounter(ctx, timedoutMetricName) +} + +func (s *stats) IncrementErrors(ctx context.Context) { + common.IncrementCounter(ctx, errorsMetricName) +} + func (s *stats) Stats() Stats { var stats Stats s.mu.Lock() @@ -160,7 +168,9 @@ func (s *stats) Stats() Stats { const ( queuedMetricName = "queued" callsMetricName = "calls" - runningSuffix = "running" + runningMetricName = "running" completedMetricName = "completed" failedMetricName = "failed" + timedoutMetricName = "timedout" + errorsMetricName = "errors" )