diff --git a/docs/assets/GrafanaDashboard2.png b/docs/assets/GrafanaDashboard2.png index d0f10deca..a4780a49a 100755 Binary files a/docs/assets/GrafanaDashboard2.png and b/docs/assets/GrafanaDashboard2.png differ diff --git a/examples/grafana/README.md b/examples/grafana/README.md index 230a1a5a9..eefe1ca8b 100644 --- a/examples/grafana/README.md +++ b/examples/grafana/README.md @@ -99,11 +99,12 @@ You should then see the dashboard shown above. Now execute some functions and se ## Tracing metrics -Tracing spans from the Fn server are available as Prometheus metrics. Each span has a name that describes the operation being performed (for example `docker_wait_container`), and its duration in seconds. Each span name is represented by a separate histogram metric, which has a name of the form `fn_span__duration_seconds`. +Tracing spans from the Fn server are available as Prometheus metrics. Each span represents a timed internal operation such as a function call, and has +two main attributes: a name that describes the operation being performed (for example `docker_wait_container`), and its duration in seconds. Each span name is represented by a separate histogram metric, which has a name of the form `fn_span__duration_seconds`. If the span is associated with a specific function invocation, the corresponding metric is given the labels `fn_app` and `fn_path` which are set to the application name and function path respectively. -A second example dashboard `fn_grafana_dashboard2.json` in this example's directory extends the first dashboard to show rate and duration data for a selection of tracing spans. +A second example dashboard `fn_grafana_dashboard2.json` in this example's directory shows rate and duration data for some significant operations. diff --git a/examples/grafana/fn_grafana_dashboard2.json b/examples/grafana/fn_grafana_dashboard2.json index 405c80a1f..20240059f 100755 --- a/examples/grafana/fn_grafana_dashboard2.json +++ b/examples/grafana/fn_grafana_dashboard2.json @@ -16,990 +16,10 @@ "gnetId": null, "graphTooltip": 0, "hideControls": false, - "id": 2, + "id": 1, "links": [], "refresh": "30s", "rows": [ - { - "collapse": false, - "height": 86, - "panels": [ - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "rgba(50, 172, 45, 0.97)", - "rgba(234, 128, 41, 0.89)", - "rgba(245, 54, 54, 0.9)" - ], - "datasource": "PromDS", - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "id": 10, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "Queued:", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "span": 3, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "sum(fn_api_queued)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "", - "refId": "A", - "step": 4 - } - ], - "thresholds": "", - "title": "", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "0", - "value": "null" - } - ], - "valueName": "current" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "rgba(50, 172, 45, 0.97)", - "rgba(234, 128, 41, 0.89)", - "rgba(245, 54, 54, 0.9)" - ], - "datasource": "PromDS", - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "id": 11, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "Running:", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "span": 3, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "sum(fn_api_running)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "", - "refId": "A", - "step": 4 - } - ], - "thresholds": "-1,0", - "title": "", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "0", - "value": "null" - } - ], - "valueName": "current" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "rgba(50, 172, 45, 0.97)", - "rgba(234, 128, 41, 0.89)", - "rgba(245, 54, 54, 0.9)" - ], - "datasource": "PromDS", - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "id": 12, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "Completed:", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "span": 3, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "sum(fn_api_completed)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "", - "refId": "A", - "step": 4 - } - ], - "thresholds": "", - "title": "", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "0", - "value": "null" - } - ], - "valueName": "current" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": true, - "colors": [ - "rgba(50, 172, 45, 0.97)", - "rgba(234, 128, 41, 0.89)", - "rgba(245, 54, 54, 0.9)" - ], - "datasource": "PromDS", - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "id": 13, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "Failed:", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "span": 3, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "sum(fn_api_failed)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "", - "refId": "A", - "step": 4 - } - ], - "thresholds": "0,0", - "title": "", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "0", - "value": "null" - } - ], - "valueName": "current" - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": false, - "title": "Dashboard Row", - "titleSize": "h6" - }, - { - "collapse": false, - "height": 250, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "PromDS", - "fill": 1, - "id": 6, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": false, - "min": false, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "span": 3, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(fn_api_queued)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Total queued", - "refId": "A", - "step": 2 - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Total queued", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "PromDS", - "fill": 1, - "id": 7, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": false, - "min": false, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "span": 3, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(fn_api_running)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Total running", - "refId": "A", - "step": 2 - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Total running", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "PromDS", - "fill": 1, - "id": 8, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": false, - "min": false, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "span": 3, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(fn_api_completed)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Total completed", - "refId": "A", - "step": 2 - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Total completed", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "PromDS", - "fill": 1, - "id": 9, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": false, - "min": false, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "span": 3, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(fn_api_failed)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Total failed", - "refId": "A", - "step": 2 - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Total queued", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": false, - "title": "Dashboard Row", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "350", - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "PromDS", - "fill": 1, - "hideTimeOverride": false, - "id": 19, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "sideWidth": null, - "sort": "current", - "sortDesc": false, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "span": 3, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "fn_api_queued", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{path}}", - "refId": "A", - "step": 2 - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Queued", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "PromDS", - "fill": 1, - "hideTimeOverride": false, - "id": 2, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "sideWidth": null, - "sort": "current", - "sortDesc": false, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "span": 3, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "fn_api_running", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{path}}", - "refId": "A", - "step": 2 - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Running", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "PromDS", - "fill": 1, - "hideTimeOverride": false, - "id": 4, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "sideWidth": null, - "sort": null, - "sortDesc": null, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "span": 3, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "fn_api_completed", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{path}}", - "refId": "A", - "step": 2 - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Completed", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "PromDS", - "fill": 1, - "hideTimeOverride": false, - "id": 5, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "sideWidth": null, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "span": 3, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "fn_api_failed", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{path}}", - "refId": "A", - "step": 2 - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Failed", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": false, - "title": "General metrics", - "titleSize": "h6" - }, { "collapse": false, "height": 267, @@ -1010,6 +30,7 @@ "dashLength": 10, "dashes": false, "datasource": "PromDS", + "description": "", "fill": 1, "id": 20, "legend": { @@ -1048,7 +69,7 @@ "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "agent_submit rate (spans/sec, 1min rolling avg)", + "title": "agent_submit rate (1 min rolling avg)", "tooltip": { "shared": true, "sort": 0, @@ -1064,8 +85,8 @@ }, "yaxes": [ { - "format": "short", - "label": null, + "format": "none", + "label": "spans/sec", "logBase": 1, "max": null, "min": null, @@ -1125,7 +146,7 @@ "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "agent submit duration (secs, 1min rolling avg)", + "title": "agent submit duration (1min rolling avg)", "tooltip": { "shared": true, "sort": 0, @@ -1142,7 +163,7 @@ "yaxes": [ { "format": "short", - "label": null, + "label": "Seconds", "logBase": 1, "max": null, "min": null, @@ -1202,7 +223,7 @@ "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "agent_get_slot duration (secs, 1min rolling avg)", + "title": "agent_get_slot duration (1min rolling avg)", "tooltip": { "shared": true, "sort": 0, @@ -1219,7 +240,7 @@ "yaxes": [ { "format": "short", - "label": null, + "label": "Seconds", "logBase": 1, "max": null, "min": null, @@ -1280,7 +301,7 @@ "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "agent_cold_exec duration (secs, 1min rolling average)", + "title": "agent_cold_exec duration (1min rolling avg)", "tooltip": { "shared": true, "sort": 0, @@ -1297,7 +318,7 @@ "yaxes": [ { "format": "short", - "label": null, + "label": "Seconds", "logBase": 1, "max": null, "min": null, @@ -1317,7 +338,7 @@ "repeat": null, "repeatIteration": null, "repeatRowId": null, - "showTitle": true, + "showTitle": false, "title": "Agent", "titleSize": "h6" }, @@ -1370,7 +391,7 @@ "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "docker_create_container span duration (secs, 1min rolling average)", + "title": "docker_create_container duration (1min rolling average)", "tooltip": { "shared": true, "sort": 0, @@ -1387,7 +408,7 @@ "yaxes": [ { "format": "short", - "label": null, + "label": "Seconds", "logBase": 1, "max": null, "min": null, @@ -1448,7 +469,7 @@ "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "docker_start_container span duration (secs, 1min rolling average)", + "title": "docker_start_container duration (1min rolling average)", "tooltip": { "shared": true, "sort": 0, @@ -1465,7 +486,7 @@ "yaxes": [ { "format": "short", - "label": null, + "label": "Seconds", "logBase": 1, "max": null, "min": null, @@ -1526,7 +547,7 @@ "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "docker_wait_container span duration (secs, 1min rolling average)", + "title": "docker_wait_container duration (1min rolling average)", "tooltip": { "shared": true, "sort": 0, @@ -1543,7 +564,7 @@ "yaxes": [ { "format": "short", - "label": null, + "label": "Seconds", "logBase": 1, "max": null, "min": null, @@ -1563,7 +584,7 @@ "repeat": null, "repeatIteration": null, "repeatRowId": null, - "showTitle": true, + "showTitle": false, "title": "Docker", "titleSize": "h6" } @@ -1604,6 +625,6 @@ ] }, "timezone": "", - "title": "Fn usage", - "version": 14 + "title": "Fn tracing span data", + "version": 8 } \ No newline at end of file