From ee59361bdaa0db4f061bc8f56a7e47fc5fac243c Mon Sep 17 00:00:00 2001
From: Tolga Ceylan <tolga.ceylan@gmail.com>
Date: Tue, 23 Jan 2018 19:30:01 -0800
Subject: [PATCH] fn: added server too busy stats (#717)

---
 api/agent/agent.go                         |  2 +-
 api/agent/stats.go                         | 19 ++++++----
 examples/grafana/fn_grafana_dashboard.json | 44 ++++++++++++++++++----
 3 files changed, 49 insertions(+), 16 deletions(-)

diff --git a/api/agent/agent.go b/api/agent/agent.go
index 22dbce5d3..3c65874f4 100644
--- a/api/agent/agent.go
+++ b/api/agent/agent.go
@@ -227,7 +227,7 @@ func transformTimeout(e error, isRetriable bool) error {
 func (a *agent) handleStatsDequeue(ctx context.Context, call *call, err error) {
 	if err == context.DeadlineExceeded {
 		a.stats.Dequeue(ctx, call.AppName, call.Path)
-		// note that this is not a timeout from the perspective of the caller, so don't increment the timeout count
+		a.stats.IncrementTooBusy(ctx)
 	} else {
 		a.stats.DequeueAndFail(ctx, call.AppName, call.Path)
 		a.stats.IncrementErrors(ctx)
diff --git a/api/agent/stats.go b/api/agent/stats.go
index ed22acf28..3a7e2155c 100644
--- a/api/agent/stats.go
+++ b/api/agent/stats.go
@@ -149,6 +149,10 @@ func (s *stats) IncrementErrors(ctx context.Context) {
 	common.IncrementCounter(ctx, errorsMetricName)
 }
 
+func (s *stats) IncrementTooBusy(ctx context.Context) {
+	common.IncrementCounter(ctx, serverBusyMetricName)
+}
+
 func (s *stats) Stats() Stats {
 	var stats Stats
 	s.mu.Lock()
@@ -166,11 +170,12 @@ func (s *stats) Stats() Stats {
 }
 
 const (
-	queuedMetricName    = "queued"
-	callsMetricName     = "calls"
-	runningMetricName   = "running"
-	completedMetricName = "completed"
-	failedMetricName    = "failed"
-	timedoutMetricName  = "timeouts"
-	errorsMetricName    = "errors"
+	queuedMetricName     = "queued"
+	callsMetricName      = "calls"
+	runningMetricName    = "running"
+	completedMetricName  = "completed"
+	failedMetricName     = "failed"
+	timedoutMetricName   = "timeouts"
+	errorsMetricName     = "errors"
+	serverBusyMetricName = "server_busy"
 )
diff --git a/examples/grafana/fn_grafana_dashboard.json b/examples/grafana/fn_grafana_dashboard.json
index 37c93c508..ee3bd2d3c 100644
--- a/examples/grafana/fn_grafana_dashboard.json
+++ b/examples/grafana/fn_grafana_dashboard.json
@@ -14,7 +14,7 @@
       "type": "grafana",
       "id": "grafana",
       "name": "Grafana",
-      "version": "4.6.2"
+      "version": "4.6.3"
     },
     {
       "type": "panel",
@@ -309,7 +309,7 @@
             "thresholdLabels": false,
             "thresholdMarkers": true
           },
-          "id": 13,
+          "id": 14,
           "interval": null,
           "links": [],
           "mappingType": 1,
@@ -328,7 +328,7 @@
           "nullText": null,
           "postfix": "",
           "postfixFontSize": "50%",
-          "prefix": "Failed:",
+          "prefix": "Incomplete:",
           "prefixFontSize": "50%",
           "rangeMaps": [
             {
@@ -347,7 +347,7 @@
           "tableColumn": "",
           "targets": [
             {
-              "expr": "sum(fn_failed)",
+              "expr": "sum(fn_failed) + sum(fn_timeouts) + sum(fn_server_busy)",
               "format": "time_series",
               "intervalFactor": 2,
               "legendFormat": "",
@@ -650,12 +650,26 @@
               "legendFormat": "Total failed",
               "refId": "A",
               "step": 2
+            },
+            {
+              "expr": "sum(fn_timeouts)",
+              "format": "time_series",
+              "intervalFactor": 2,
+              "legendFormat": "Total timeouts",
+              "refId": "B"
+            },
+            {
+              "expr": "sum(fn_server_busy)",
+              "format": "time_series",
+              "intervalFactor": 2,
+              "legendFormat": "Total server too busy",
+              "refId": "C"
             }
           ],
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "Total failed",
+          "title": "Total incomplete",
           "tooltip": {
             "shared": true,
             "sort": 0,
@@ -989,15 +1003,29 @@
               "expr": "fn_failed",
               "format": "time_series",
               "intervalFactor": 2,
-              "legendFormat": "{{fn_appname}} {{fn_path}}",
+              "legendFormat": "{{fn_appname}} {{fn_path}} failed",
               "refId": "A",
               "step": 2
+            },
+            {
+              "expr": "fn_timeouts",
+              "format": "time_series",
+              "intervalFactor": 2,
+              "legendFormat": "{{fn_appname}} {{fn_path}} timeouts",
+              "refId": "B"
+            },
+            {
+              "expr": "fn_server_busy",
+              "format": "time_series",
+              "intervalFactor": 2,
+              "legendFormat": "{{fn_appname}} {{fn_path}} server busy",
+              "refId": "C"
             }
           ],
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "Failed",
+          "title": "Incomplete",
           "tooltip": {
             "shared": true,
             "sort": 0,
@@ -1076,5 +1104,5 @@
   },
   "timezone": "",
   "title": "Fn usage",
-  "version": 5
+  "version": 1
 }
\ No newline at end of file