diff --git a/api/agent/agent.go b/api/agent/agent.go index 2f6ecf681..5936bfa1f 100644 --- a/api/agent/agent.go +++ b/api/agent/agent.go @@ -188,6 +188,8 @@ func (a *agent) Submit(callI Call) error { ctx := call.req.Context() span, ctx := opentracing.StartSpanFromContext(ctx, "agent_submit") + span.SetBaggageItem("fn_appname", callI.Model().AppName) + span.SetBaggageItem("fn_path", callI.Model().Path) defer span.Finish() // start the timer STAT! TODO add some wiggle room diff --git a/api/server/fntracer.go b/api/server/fntracer.go new file mode 100644 index 000000000..6abe92044 --- /dev/null +++ b/api/server/fntracer.go @@ -0,0 +1,130 @@ +package server + +import ( + "github.com/opentracing/opentracing-go" + "github.com/opentracing/opentracing-go/log" + "strings" +) + +// FnTracer is a custom Tracer which wraps another another tracer +// its main purpose is to wrap the underlying Span in a FnSpan, +// which adds some extra behaviour required for sending tracing spans to prometheus +type FnTracer struct { + wrappedTracer opentracing.Tracer +} + +// NewFnTracer returns a new FnTracer which wraps the specified Tracer +func NewFnTracer(tracerToWrap opentracing.Tracer) opentracing.Tracer { + newTracer := &FnTracer{} + newTracer.wrappedTracer = tracerToWrap + return newTracer +} + +// FnTracer implements opentracing.Tracer +func (thisFnTracer FnTracer) StartSpan(operationName string, opts ...opentracing.StartSpanOption) opentracing.Span { + return NewFnSpan(thisFnTracer.wrappedTracer.StartSpan(operationName, opts...)) +} + +// FnTracer implements opentracing.Tracer +func (thisFnTracer FnTracer) Inject(sm opentracing.SpanContext, format interface{}, carrier interface{}) error { + return thisFnTracer.wrappedTracer.Inject(sm, format, carrier) + +} + +// FnTracer implements opentracing.Tracer +func (thisFnTracer FnTracer) Extract(format interface{}, carrier interface{}) (opentracing.SpanContext, error) { + return thisFnTracer.wrappedTracer.Extract(format, carrier) +} + +// FnSpan is a custom Span that wraps another span +// which adds some extra behaviour required for sending tracing spans to prometheus +type FnSpan struct { + wrappedSpan opentracing.Span +} + +// NewFnSpan returns a new FnSpan which wraps the specified Span +func NewFnSpan(spanToWrap opentracing.Span) opentracing.Span { + newSpan := &FnSpan{} + newSpan.wrappedSpan = spanToWrap + return newSpan +} + +// FnSpan implements opentracing.Span +func (thisFnSpan FnSpan) Finish() { + thisFnSpan.copyBaggageItemsToTags() + thisFnSpan.wrappedSpan.Finish() +} + +// FnSpan implements opentracing.Span +func (thisFnSpan FnSpan) FinishWithOptions(opts opentracing.FinishOptions) { + thisFnSpan.copyBaggageItemsToTags() + thisFnSpan.wrappedSpan.FinishWithOptions(opts) +} + +func (thisFnSpan FnSpan) copyBaggageItemsToTags() { + // copy baggage items (which are inherited from the parent) with keys starting with "fn" to tags + // the PrometheusCollector will send these to Prometheus + // need to do this because the collector can't access baggage items, but it can access tags + // whereas here we can access the parent's baggage items, but not its tags + thisFnSpan.Context().ForeachBaggageItem(func(k, v string) bool { + if strings.HasPrefix(k, "fn") { + thisFnSpan.SetTag(k, v) + } + return true + }) +} + +// FnSpan implements opentracing.Span +func (thisFnSpan FnSpan) Context() opentracing.SpanContext { + return thisFnSpan.wrappedSpan.Context() +} + +// FnSpan implements opentracing.Span +func (thisFnSpan FnSpan) SetOperationName(operationName string) opentracing.Span { + return thisFnSpan.wrappedSpan.SetOperationName(operationName) +} + +// FnSpan implements opentracing.Span +func (thisFnSpan FnSpan) SetTag(key string, value interface{}) opentracing.Span { + return thisFnSpan.wrappedSpan.SetTag(key, value) +} + +// FnSpan implements opentracing.Span +func (thisFnSpan FnSpan) LogFields(fields ...log.Field) { + thisFnSpan.wrappedSpan.LogFields(fields...) +} + +// FnSpan implements opentracing.Span +func (thisFnSpan FnSpan) LogKV(alternatingKeyValues ...interface{}) { + thisFnSpan.wrappedSpan.LogKV(alternatingKeyValues...) +} + +// FnSpan implements opentracing.Span +func (thisFnSpan FnSpan) SetBaggageItem(restrictedKey, value string) opentracing.Span { + return thisFnSpan.wrappedSpan.SetBaggageItem(restrictedKey, value) +} + +// FnSpan implements opentracing.Span +func (thisFnSpan FnSpan) BaggageItem(restrictedKey string) string { + return thisFnSpan.wrappedSpan.BaggageItem(restrictedKey) +} + +// FnSpan implements opentracing.Span +func (thisFnSpan FnSpan) Tracer() opentracing.Tracer { + return thisFnSpan.wrappedSpan.Tracer() +} + +// FnSpan implements opentracing.Span +func (thisFnSpan FnSpan) LogEvent(event string) { + thisFnSpan.wrappedSpan.LogEvent(event) +} + +// FnSpan implements opentracing.Span +func (thisFnSpan FnSpan) LogEventWithPayload(event string, payload interface{}) { + thisFnSpan.wrappedSpan.LogEventWithPayload(event, payload) +} + +// FnSpan implements opentracing.Span +func (thisFnSpan FnSpan) Log(data opentracing.LogData) { + thisFnSpan.wrappedSpan.Log(data) +} diff --git a/api/server/prom_zip_collector.go b/api/server/prom_zip_collector.go new file mode 100644 index 000000000..e2d4fcd9d --- /dev/null +++ b/api/server/prom_zip_collector.go @@ -0,0 +1,80 @@ +package server + +import ( + "github.com/fnproject/fn/api/agent" + "github.com/openzipkin/zipkin-go-opentracing" + "github.com/openzipkin/zipkin-go-opentracing/thrift/gen-go/zipkincore" + "github.com/prometheus/client_golang/prometheus" + "strings" +) + +// Each span name is published as a separate Histogram metric +// Using metric names of the form fn_span__duration_seconds +var histogramVecMap = make(map[string]*prometheus.HistogramVec) + +// Return the HistogramVec corresponding to the specified spanName. +// If a HistogramVec does not already exist for specified spanName then one is created and configured with the specified labels +// otherwise the labels parameter is ignored. +func getHistogramVecForSpanName(spanName string, labels []string) *prometheus.HistogramVec { + thisHistogramVec, found := histogramVecMap[spanName] + if !found { + thisHistogramVec = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "fn_span_" + spanName + "_duration_seconds", + Help: "Span " + spanName + " duration, by span name", + }, + labels, + ) + histogramVecMap[spanName] = thisHistogramVec + prometheus.MustRegister(thisHistogramVec) + } + return thisHistogramVec +} + +// PrometheusCollector is a custom Collector +// which sends ZipKin traces to Prometheus +type PrometheusCollector struct { + a agent.Agent +} + +// NewPrometheusCollector returns a new PrometheusCollector +func NewPrometheusCollector(agent agent.Agent) (zipkintracer.Collector, error) { + pc := &PrometheusCollector{} + pc.a = agent + return pc, nil +} + +// PrometheusCollector implements Collector. +func (pc PrometheusCollector) Collect(span *zipkincore.Span) error { + + // extract any label values from the span + labelKeys, labelValueMap := getLabels(span) + + getHistogramVecForSpanName(span.GetName(), labelKeys).With(labelValueMap).Observe(float64(span.GetDuration()) / 1000000) + + return nil +} + +// extract from the specified span the key/value pairs that we want to add as labels to the Prometheus metric for this span +// returns an array of keys, and a map of key-value pairs +func getLabels(span *zipkincore.Span) ([]string, map[string]string) { + + var keys []string + labelMap := make(map[string]string) + + // extract any tags whose key starts with "fn" from the span + binaryAnnotations := span.GetBinaryAnnotations() + for _, thisBinaryAnnotation := range binaryAnnotations { + key := thisBinaryAnnotation.GetKey() + if thisBinaryAnnotation.GetAnnotationType() == zipkincore.AnnotationType_STRING && strings.HasPrefix(key, "fn") { + keys = append(keys, key) + value := string(thisBinaryAnnotation.GetValue()[:]) + labelMap[key] = value + } + } + + return keys, labelMap +} + +// PrometheusCollector implements Collector. +func (PrometheusCollector) Close() error { return nil } diff --git a/api/server/server.go b/api/server/server.go index 6ea1432b7..8399c50ab 100644 --- a/api/server/server.go +++ b/api/server/server.go @@ -86,7 +86,7 @@ func New(ctx context.Context, ds models.Datastore, mq models.MessageQueue, logDB } setMachineId() - setTracer() + s.setTracer() s.Router.Use(loggerWrap, traceWrap, panicWrap) s.bindHandlers(ctx) @@ -117,7 +117,7 @@ func traceWrap(c *gin.Context) { c.Next() } -func setTracer() { +func (s *Server) setTracer() { var ( debugMode = false serviceName = "fn-server" @@ -126,25 +126,40 @@ func setTracer() { // ex: "http://zipkin:9411/api/v1/spans" ) - if zipkinHTTPEndpoint == "" { - return + var collector zipkintracer.Collector + + // custom Zipkin collector to send tracing spans to Prometheus + promCollector, promErr := NewPrometheusCollector(s.Agent) + if promErr != nil { + logrus.WithError(promErr).Fatalln("couldn't start Prometheus trace collector") } logger := zipkintracer.LoggerFunc(func(i ...interface{}) error { logrus.Error(i...); return nil }) - collector, err := zipkintracer.NewHTTPCollector(zipkinHTTPEndpoint, zipkintracer.HTTPLogger(logger)) - if err != nil { - logrus.WithError(err).Fatalln("couldn't start trace collector") + if zipkinHTTPEndpoint != "" { + // Custom PrometheusCollector and Zipkin HTTPCollector + httpCollector, zipErr := zipkintracer.NewHTTPCollector(zipkinHTTPEndpoint, zipkintracer.HTTPLogger(logger)) + if zipErr != nil { + logrus.WithError(zipErr).Fatalln("couldn't start Zipkin trace collector") + } + collector = zipkintracer.MultiCollector{httpCollector, promCollector} + } else { + // Custom PrometheusCollector only + collector = promCollector } - tracer, err := zipkintracer.NewTracer(zipkintracer.NewRecorder(collector, debugMode, serviceHostPort, serviceName), + + ziptracer, err := zipkintracer.NewTracer(zipkintracer.NewRecorder(collector, debugMode, serviceHostPort, serviceName), zipkintracer.ClientServerSameSpan(true), zipkintracer.TraceID128Bit(true), ) if err != nil { logrus.WithError(err).Fatalln("couldn't start tracer") } + + // wrap the Zipkin tracer in a FnTracer which will also send spans to Prometheus + fntracer := NewFnTracer(ziptracer) - opentracing.SetGlobalTracer(tracer) + opentracing.SetGlobalTracer(fntracer) logrus.WithFields(logrus.Fields{"url": zipkinHTTPEndpoint}).Info("started tracer") } diff --git a/docs/assets/GrafanaDashboard2.png b/docs/assets/GrafanaDashboard2.png new file mode 100755 index 000000000..d0f10deca Binary files /dev/null and b/docs/assets/GrafanaDashboard2.png differ diff --git a/examples/grafana/README.md b/examples/grafana/README.md index b84044d14..8b7674202 100644 --- a/examples/grafana/README.md +++ b/examples/grafana/README.md @@ -90,11 +90,20 @@ Create a datasource to obtain metrics from Promethesus: * Click **Add** and then **Save and test** Import the example dashboard that displays metrics from the Fn server: -* Click on the main menu at the top left and choose **Dashboards** and then **Home** -* Click on **Home** at the top and then **Import dashboard** +* Click on the main menu at the top left and choose **Dashboards** and then **Import** * In the dialog that opens, click **Upload .json file** and specify `fn_grafana_dashboard.json` in this example's directory. * Specify the Prometheus data source that you just created * Click **Import** You should then see the dashboard shown above. Now execute some functions and see the graphs update. +## Tracing metrics + +Tracing spans from the Fn server are available as Prometheus metrics. Each span has a name that describes the operation being performed (for example `docker_wait_container`), and its duration in seconds. Each span name is represented by a separate histogram metric, which has a name of the form `fn_span__duration_seconds`. + +If the span is associated with a specific function invocation, the corresponding metric is given the labels `fn_app` and `fn_path` which are set to the application name and function path respectively. + +A second example dashboard shows rate and duration data for a selection of tracing spans. This is `fn_grafana_dashboard2.json` in this example's directory. + + + diff --git a/examples/grafana/fn_grafana_dashboard2.json b/examples/grafana/fn_grafana_dashboard2.json new file mode 100755 index 000000000..405c80a1f --- /dev/null +++ b/examples/grafana/fn_grafana_dashboard2.json @@ -0,0 +1,1609 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": 2, + "links": [], + "refresh": "30s", + "rows": [ + { + "collapse": false, + "height": 86, + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(234, 128, 41, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "PromDS", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "id": 10, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "Queued:", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 3, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(fn_api_queued)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 4 + } + ], + "thresholds": "", + "title": "", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(234, 128, 41, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "PromDS", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "id": 11, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "Running:", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 3, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(fn_api_running)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 4 + } + ], + "thresholds": "-1,0", + "title": "", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(234, 128, 41, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "PromDS", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "id": 12, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "Completed:", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 3, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(fn_api_completed)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 4 + } + ], + "thresholds": "", + "title": "", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(234, 128, 41, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "PromDS", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "id": 13, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "Failed:", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 3, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(fn_api_failed)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 4 + } + ], + "thresholds": "0,0", + "title": "", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6" + }, + { + "collapse": false, + "height": 250, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "PromDS", + "fill": 1, + "id": 6, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(fn_api_queued)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Total queued", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Total queued", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "PromDS", + "fill": 1, + "id": 7, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(fn_api_running)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Total running", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Total running", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "PromDS", + "fill": 1, + "id": 8, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(fn_api_completed)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Total completed", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Total completed", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "PromDS", + "fill": 1, + "id": 9, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(fn_api_failed)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Total failed", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Total queued", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "350", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "PromDS", + "fill": 1, + "hideTimeOverride": false, + "id": 19, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "fn_api_queued", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{path}}", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Queued", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "PromDS", + "fill": 1, + "hideTimeOverride": false, + "id": 2, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "fn_api_running", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{path}}", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Running", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "PromDS", + "fill": 1, + "hideTimeOverride": false, + "id": 4, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "fn_api_completed", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{path}}", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Completed", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "PromDS", + "fill": 1, + "hideTimeOverride": false, + "id": 5, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "fn_api_failed", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{path}}", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Failed", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "General metrics", + "titleSize": "h6" + }, + { + "collapse": false, + "height": 267, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "PromDS", + "fill": 1, + "id": 20, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(fn_span_agent_submit_duration_seconds_count[1m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{fn_path}} ({{fn_appname}})", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "agent_submit rate (spans/sec, 1min rolling avg)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "PromDS", + "fill": 1, + "id": 17, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(fn_span_agent_submit_duration_seconds_sum[1m]) / rate(fn_span_agent_submit_duration_seconds_count[1m])\n\n\n\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{fn_path}} ({{fn_appname}})", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "agent submit duration (secs, 1min rolling avg)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "PromDS", + "fill": 1, + "id": 21, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(fn_span_agent_get_slot_duration_seconds_sum[1m]) / rate(fn_span_agent_get_slot_duration_seconds_count[1m])\n\n\n\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{fn_path}} ({{fn_appname}})", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "agent_get_slot duration (secs, 1min rolling avg)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "PromDS", + "fill": 1, + "id": 42, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(fn_span_agent_cold_exec_duration_seconds_sum[1m]) / rate(fn_span_agent_cold_exec_duration_seconds_count[1m])\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{fn_path}} ({{fn_appname}})", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "agent_cold_exec duration (secs, 1min rolling average)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Agent", + "titleSize": "h6" + }, + { + "collapse": false, + "height": 228, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "PromDS", + "fill": 1, + "id": 43, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(fn_span_docker_create_container_duration_seconds_sum[1m]) / rate(fn_span_docker_create_container_duration_seconds_count[1m])\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{fn_path}} ({{fn_appname}})", + "refId": "A", + "step": 1 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "docker_create_container span duration (secs, 1min rolling average)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "PromDS", + "fill": 1, + "id": 33, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(fn_span_docker_start_container_duration_seconds_sum[1m]) / rate(fn_span_docker_start_container_duration_seconds_count[1m])\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{fn_path}} ({{fn_appname}})", + "refId": "A", + "step": 1 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "docker_start_container span duration (secs, 1min rolling average)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "PromDS", + "fill": 1, + "id": 35, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(fn_span_docker_wait_container_duration_seconds_sum[1m]) / rate(fn_span_docker_start_container_duration_seconds_count[1m])\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{fn_path}} ({{fn_appname}})", + "refId": "A", + "step": 1 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "docker_wait_container span duration (secs, 1min rolling average)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Docker", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Fn usage", + "version": 14 +} \ No newline at end of file