Add new Prom metrics fn_timeout and fn_errors (#679)

* Add new Prom metric fn_timedout

* Add new Prometheus metric fn_errors

* Tidy up variable name

* Add new Prometheus metric fn_errors

* gofmt

api/agent/agent.go

@@ -227,8 +227,10 @@ func transformTimeout(e error, isRetriable bool) error {
 func (a *agent) handleStatsDequeue(ctx context.Context, call *call, err error) {
 	if err == context.DeadlineExceeded {
 		a.stats.Dequeue(ctx, call.AppName, call.Path)
+		// note that this is not a timeout from the perspective of the caller, so don't increment the timeout count
 	} else {
 		a.stats.DequeueAndFail(ctx, call.AppName, call.Path)
+		a.stats.IncrementErrors(ctx)
 	}
 }
 
@@ -240,6 +242,12 @@ func (a *agent) handleStatsEnd(ctx context.Context, call *call, err error) {
 	} else {
 		// decrement running count, increment failed count
 		a.stats.Failed(ctx, call.AppName, call.Path)
+		// increment the timeout or errors count, as appropriate
+		if err == context.DeadlineExceeded {
+			a.stats.IncrementTimedout(ctx)
+		} else {
+			a.stats.IncrementErrors(ctx)
+		}
 	}
 }