fn: added server too busy stats (#717)

api/agent/agent.go

@@ -227,7 +227,7 @@ func transformTimeout(e error, isRetriable bool) error {
 func (a *agent) handleStatsDequeue(ctx context.Context, call *call, err error) {
 	if err == context.DeadlineExceeded {
 		a.stats.Dequeue(ctx, call.AppName, call.Path)
-		// note that this is not a timeout from the perspective of the caller, so don't increment the timeout count
+		a.stats.IncrementTooBusy(ctx)
 	} else {
 		a.stats.DequeueAndFail(ctx, call.AppName, call.Path)
 		a.stats.IncrementErrors(ctx)

api/agent/stats.go

@@ -149,6 +149,10 @@ func (s *stats) IncrementErrors(ctx context.Context) {
 	common.IncrementCounter(ctx, errorsMetricName)
 }
 
+func (s *stats) IncrementTooBusy(ctx context.Context) {
+	common.IncrementCounter(ctx, serverBusyMetricName)
+}
+
 func (s *stats) Stats() Stats {
 	var stats Stats
 	s.mu.Lock()
@@ -173,4 +177,5 @@ const (
 	failedMetricName     = "failed"
 	timedoutMetricName   = "timeouts"
 	errorsMetricName     = "errors"
+	serverBusyMetricName = "server_busy"
 )

examples/grafana/fn_grafana_dashboard.json

@@ -14,7 +14,7 @@
       "type": "grafana",
       "id": "grafana",
       "name": "Grafana",
-      "version": "4.6.2"
+      "version": "4.6.3"
     },
     {
       "type": "panel",
@@ -309,7 +309,7 @@
             "thresholdLabels": false,
             "thresholdMarkers": true
           },
-          "id": 13,
+          "id": 14,
           "interval": null,
           "links": [],
           "mappingType": 1,
@@ -328,7 +328,7 @@
           "nullText": null,
           "postfix": "",
           "postfixFontSize": "50%",
-          "prefix": "Failed:",
+          "prefix": "Incomplete:",
           "prefixFontSize": "50%",
           "rangeMaps": [
             {
@@ -347,7 +347,7 @@
           "tableColumn": "",
           "targets": [
             {
-              "expr": "sum(fn_failed)",
+              "expr": "sum(fn_failed) + sum(fn_timeouts) + sum(fn_server_busy)",
               "format": "time_series",
               "intervalFactor": 2,
               "legendFormat": "",
@@ -650,12 +650,26 @@
               "legendFormat": "Total failed",
               "refId": "A",
               "step": 2
+            },
+            {
+              "expr": "sum(fn_timeouts)",
+              "format": "time_series",
+              "intervalFactor": 2,
+              "legendFormat": "Total timeouts",
+              "refId": "B"
+            },
+            {
+              "expr": "sum(fn_server_busy)",
+              "format": "time_series",
+              "intervalFactor": 2,
+              "legendFormat": "Total server too busy",
+              "refId": "C"
             }
           ],
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "Total failed",
+          "title": "Total incomplete",
           "tooltip": {
             "shared": true,
             "sort": 0,
@@ -989,15 +1003,29 @@
               "expr": "fn_failed",
               "format": "time_series",
               "intervalFactor": 2,
-              "legendFormat": "{{fn_appname}} {{fn_path}}",
+              "legendFormat": "{{fn_appname}} {{fn_path}} failed",
               "refId": "A",
               "step": 2
+            },
+            {
+              "expr": "fn_timeouts",
+              "format": "time_series",
+              "intervalFactor": 2,
+              "legendFormat": "{{fn_appname}} {{fn_path}} timeouts",
+              "refId": "B"
+            },
+            {
+              "expr": "fn_server_busy",
+              "format": "time_series",
+              "intervalFactor": 2,
+              "legendFormat": "{{fn_appname}} {{fn_path}} server busy",
+              "refId": "C"
             }
           ],
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "Failed",
+          "title": "Incomplete",
           "tooltip": {
             "shared": true,
             "sort": 0,
@@ -1076,5 +1104,5 @@
   },
   "timezone": "",
   "title": "Fn usage",
-  "version": 5
+  "version": 1
 }