fn-serverless/api/agent/stats.go

package agent

import (
	"context"
	"github.com/fnproject/fn/api/common"
	"sync"
)

// TODO this should expose:
// * hot containers active
// * memory used / available

// stats is the top-level struct containing composite and individial statistics all routes in all apps
// an instance of this struc is maintained in memory to keep a record of the current stats since the server was started
// access must be synchronized using the Mutex
type stats struct {
	mu sync.Mutex
	// composite statistics for all routes in all apps
	queue    uint64
	running  uint64
	complete uint64
	failed   uint64
	// statistics for individual apps, keyed by appname
	apps map[string]appStats
}

// appStats holds statistics for the routes in an individual app, keyed by the path of the route
// instances of this struc are used to maintain an in-memory record of the current stats
// access must be synchronized using the Mutex on the parent stats
type appStats struct {
	routes map[string]*routeStats
}

// routeStats holds statistics for an individual route
// instances of this struc are used to maintain an in-memory record of the current stats
// access must be synchronized using the Mutex on the parent stats
type routeStats struct {
	queue    uint64
	running  uint64
	complete uint64
	failed   uint64
}

// Stats is the top-level struct containing composite and individial statistics all routes in all apps
// an instance of this struc is created when converting the current stats to JSON
type Stats struct {
	Queue    uint64
	Running  uint64
	Complete uint64
	Failed   uint64
	// statistics for individual apps, keyed by appname
	Apps map[string]AppStats
}

// AppStats holds statistics for the routes in an individual app, keyed by the path of the route
// instances of this struc are used when converting the current stats to JSON
type AppStats struct {
	Routes map[string]*RouteStats
}

// RouteStats holds statistics for an individual route
// instances of this struc are used when converting the current stats to JSON
type RouteStats struct {
	Queue    uint64
	Running  uint64
	Complete uint64
	Failed   uint64
}

// return the stats corresponding to the specified app name and route path, creating a new stats if one does not already exist
func (s *stats) getStatsForRoute(app string, path string) *routeStats {
	if s.apps == nil {
		s.apps = make(map[string]appStats)
	}
	thisAppStats, appFound := s.apps[app]
	if !appFound {
		thisAppStats = appStats{routes: make(map[string]*routeStats)}
		s.apps[app] = thisAppStats
	}
	thisRouteStats, pathFound := thisAppStats.routes[path]
	if !pathFound {
		thisRouteStats = &routeStats{}
		thisAppStats.routes[path] = thisRouteStats
	}
	return thisRouteStats
}

func (s *stats) Enqueue(ctx context.Context, app string, path string) {
	s.mu.Lock()

	fstats := s.getStatsForRoute(app, path)
	s.queue++
	fstats.queue++

	s.mu.Unlock()

	common.IncrementGauge(ctx, queuedMetricName)
	common.IncrementCounter(ctx, callsMetricName)
}

// Call when a function has been queued but cannot be started because of an error
func (s *stats) Dequeue(ctx context.Context, app string, path string) {
	s.mu.Lock()

	fstats := s.getStatsForRoute(app, path)
	s.queue--
	fstats.queue--

	s.mu.Unlock()

	common.DecrementGauge(ctx, queuedMetricName)
}

func (s *stats) DequeueAndStart(ctx context.Context, app string, path string) {
	s.mu.Lock()

	fstats := s.getStatsForRoute(app, path)
	s.queue--
	s.running++
	fstats.queue--
	fstats.running++

	s.mu.Unlock()

	common.DecrementGauge(ctx, queuedMetricName)
	common.IncrementGauge(ctx, runningMetricName)
}

func (s *stats) Complete(ctx context.Context, app string, path string) {
	s.mu.Lock()

	fstats := s.getStatsForRoute(app, path)
	s.running--
	s.complete++
	fstats.running--
	fstats.complete++

	s.mu.Unlock()

	common.DecrementGauge(ctx, runningMetricName)
	common.IncrementCounter(ctx, completedMetricName)
}

func (s *stats) Failed(ctx context.Context, app string, path string) {
	s.mu.Lock()

	fstats := s.getStatsForRoute(app, path)
	s.running--
	s.failed++
	fstats.running--
	fstats.failed++

	s.mu.Unlock()

	common.DecrementGauge(ctx, runningMetricName)
	common.IncrementCounter(ctx, failedMetricName)
}

func (s *stats) DequeueAndFail(ctx context.Context, app string, path string) {
	s.mu.Lock()

	fstats := s.getStatsForRoute(app, path)
	s.queue--
	s.failed++
	fstats.queue--
	fstats.failed++

	s.mu.Unlock()

	common.DecrementGauge(ctx, queuedMetricName)
	common.IncrementCounter(ctx, failedMetricName)
}

func IncrementTimedout(ctx context.Context) {
	common.IncrementCounter(ctx, timedoutMetricName)
}

func IncrementErrors(ctx context.Context) {
	common.IncrementCounter(ctx, errorsMetricName)
}

func IncrementTooBusy(ctx context.Context) {
	common.IncrementCounter(ctx, serverBusyMetricName)
}

func (s *stats) Stats() Stats {
	// this creates a Stats from a stats
	// stats is the internal struc which is continuously updated, and access is controlled using its Mutex
	// Stats is a deep copy for external use, and can be converted to JSON
	var stats Stats
	s.mu.Lock()
	stats.Running = s.running
	stats.Complete = s.complete
	stats.Queue = s.queue
	stats.Failed = s.failed
	stats.Apps = make(map[string]AppStats)
	for appname, thisAppStats := range s.apps {
		newAppStats := AppStats{Routes: make(map[string]*RouteStats)}
		stats.Apps[appname] = newAppStats
		for path, thisRouteStats := range thisAppStats.routes {
			newAppStats.Routes[path] = &RouteStats{Queue: thisRouteStats.queue, Running: thisRouteStats.running, Complete: thisRouteStats.complete, Failed: thisRouteStats.failed}
		}
	}
	s.mu.Unlock()
	return stats
}

const (
	queuedMetricName     = "queued"
	callsMetricName      = "calls"
	runningMetricName    = "running"
	completedMetricName  = "completed"
	failedMetricName     = "failed"
	timedoutMetricName   = "timeouts"
	errorsMetricName     = "errors"
	serverBusyMetricName = "server_busy"
)