mirror of
https://github.com/fnproject/fn.git
synced 2022-10-28 21:29:17 +03:00
@@ -88,10 +88,6 @@ type Agent interface {
|
|||||||
// Close is not safe to be called from multiple threads.
|
// Close is not safe to be called from multiple threads.
|
||||||
io.Closer
|
io.Closer
|
||||||
|
|
||||||
// Stats should be burned at the stake. adding so as to not ruffle feathers.
|
|
||||||
// TODO this should be derived from our metrics
|
|
||||||
Stats() Stats
|
|
||||||
|
|
||||||
// Return the http.Handler used to handle Prometheus metric requests
|
// Return the http.Handler used to handle Prometheus metric requests
|
||||||
PromHandler() http.Handler
|
PromHandler() http.Handler
|
||||||
AddCallListener(fnext.CallListener)
|
AddCallListener(fnext.CallListener)
|
||||||
@@ -119,8 +115,6 @@ type agent struct {
|
|||||||
freezeIdleMsecs time.Duration
|
freezeIdleMsecs time.Duration
|
||||||
ejectIdleMsecs time.Duration
|
ejectIdleMsecs time.Duration
|
||||||
|
|
||||||
stats // TODO kill me
|
|
||||||
|
|
||||||
// Prometheus HTTP handler
|
// Prometheus HTTP handler
|
||||||
promHandler http.Handler
|
promHandler http.Handler
|
||||||
}
|
}
|
||||||
@@ -242,7 +236,7 @@ func (a *agent) endStateTrackers(ctx context.Context, call *call) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (a *agent) submit(ctx context.Context, call *call) error {
|
func (a *agent) submit(ctx context.Context, call *call) error {
|
||||||
a.stats.Enqueue(ctx, call.AppName, call.Path)
|
StatsEnqueue(ctx)
|
||||||
|
|
||||||
a.startStateTrackers(ctx, call)
|
a.startStateTrackers(ctx, call)
|
||||||
defer a.endStateTrackers(ctx, call)
|
defer a.endStateTrackers(ctx, call)
|
||||||
@@ -262,7 +256,7 @@ func (a *agent) submit(ctx context.Context, call *call) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// decrement queued count, increment running count
|
// decrement queued count, increment running count
|
||||||
a.stats.DequeueAndStart(ctx, call.AppName, call.Path)
|
StatsDequeueAndStart(ctx)
|
||||||
|
|
||||||
// pass this error (nil or otherwise) to end directly, to store status, etc
|
// pass this error (nil or otherwise) to end directly, to store status, etc
|
||||||
err = slot.exec(ctx, call)
|
err = slot.exec(ctx, call)
|
||||||
@@ -289,11 +283,11 @@ func transformTimeout(e error, isRetriable bool) error {
|
|||||||
// cases. Only timeouts can be a simple dequeue while other cases are actual errors.
|
// cases. Only timeouts can be a simple dequeue while other cases are actual errors.
|
||||||
func (a *agent) handleStatsDequeue(ctx context.Context, call *call, err error) {
|
func (a *agent) handleStatsDequeue(ctx context.Context, call *call, err error) {
|
||||||
if err == context.DeadlineExceeded {
|
if err == context.DeadlineExceeded {
|
||||||
a.stats.Dequeue(ctx, call.AppName, call.Path)
|
StatsDequeue(ctx)
|
||||||
IncrementTooBusy(ctx)
|
StatsIncrementTooBusy(ctx)
|
||||||
} else {
|
} else {
|
||||||
a.stats.DequeueAndFail(ctx, call.AppName, call.Path)
|
StatsDequeueAndFail(ctx)
|
||||||
IncrementErrors(ctx)
|
StatsIncrementErrors(ctx)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -301,15 +295,15 @@ func (a *agent) handleStatsDequeue(ctx context.Context, call *call, err error) {
|
|||||||
func (a *agent) handleStatsEnd(ctx context.Context, call *call, err error) {
|
func (a *agent) handleStatsEnd(ctx context.Context, call *call, err error) {
|
||||||
if err == nil {
|
if err == nil {
|
||||||
// decrement running count, increment completed count
|
// decrement running count, increment completed count
|
||||||
a.stats.Complete(ctx, call.AppName, call.Path)
|
StatsComplete(ctx)
|
||||||
} else {
|
} else {
|
||||||
// decrement running count, increment failed count
|
// decrement running count, increment failed count
|
||||||
a.stats.Failed(ctx, call.AppName, call.Path)
|
StatsFailed(ctx)
|
||||||
// increment the timeout or errors count, as appropriate
|
// increment the timeout or errors count, as appropriate
|
||||||
if err == context.DeadlineExceeded {
|
if err == context.DeadlineExceeded {
|
||||||
IncrementTimedout(ctx)
|
StatsIncrementTimedout(ctx)
|
||||||
} else {
|
} else {
|
||||||
IncrementErrors(ctx)
|
StatsIncrementErrors(ctx)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,208 +3,50 @@ package agent
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"github.com/fnproject/fn/api/common"
|
"github.com/fnproject/fn/api/common"
|
||||||
"sync"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// TODO this should expose:
|
func StatsEnqueue(ctx context.Context) {
|
||||||
// * hot containers active
|
|
||||||
// * memory used / available
|
|
||||||
|
|
||||||
// stats is the top-level struct containing composite and individial statistics all routes in all apps
|
|
||||||
// an instance of this struc is maintained in memory to keep a record of the current stats since the server was started
|
|
||||||
// access must be synchronized using the Mutex
|
|
||||||
type stats struct {
|
|
||||||
mu sync.Mutex
|
|
||||||
// composite statistics for all routes in all apps
|
|
||||||
queue uint64
|
|
||||||
running uint64
|
|
||||||
complete uint64
|
|
||||||
failed uint64
|
|
||||||
// statistics for individual apps, keyed by appname
|
|
||||||
apps map[string]appStats
|
|
||||||
}
|
|
||||||
|
|
||||||
// appStats holds statistics for the routes in an individual app, keyed by the path of the route
|
|
||||||
// instances of this struc are used to maintain an in-memory record of the current stats
|
|
||||||
// access must be synchronized using the Mutex on the parent stats
|
|
||||||
type appStats struct {
|
|
||||||
routes map[string]*routeStats
|
|
||||||
}
|
|
||||||
|
|
||||||
// routeStats holds statistics for an individual route
|
|
||||||
// instances of this struc are used to maintain an in-memory record of the current stats
|
|
||||||
// access must be synchronized using the Mutex on the parent stats
|
|
||||||
type routeStats struct {
|
|
||||||
queue uint64
|
|
||||||
running uint64
|
|
||||||
complete uint64
|
|
||||||
failed uint64
|
|
||||||
}
|
|
||||||
|
|
||||||
// Stats is the top-level struct containing composite and individial statistics all routes in all apps
|
|
||||||
// an instance of this struc is created when converting the current stats to JSON
|
|
||||||
type Stats struct {
|
|
||||||
Queue uint64
|
|
||||||
Running uint64
|
|
||||||
Complete uint64
|
|
||||||
Failed uint64
|
|
||||||
// statistics for individual apps, keyed by appname
|
|
||||||
Apps map[string]AppStats
|
|
||||||
}
|
|
||||||
|
|
||||||
// AppStats holds statistics for the routes in an individual app, keyed by the path of the route
|
|
||||||
// instances of this struc are used when converting the current stats to JSON
|
|
||||||
type AppStats struct {
|
|
||||||
Routes map[string]*RouteStats
|
|
||||||
}
|
|
||||||
|
|
||||||
// RouteStats holds statistics for an individual route
|
|
||||||
// instances of this struc are used when converting the current stats to JSON
|
|
||||||
type RouteStats struct {
|
|
||||||
Queue uint64
|
|
||||||
Running uint64
|
|
||||||
Complete uint64
|
|
||||||
Failed uint64
|
|
||||||
}
|
|
||||||
|
|
||||||
// return the stats corresponding to the specified app name and route path, creating a new stats if one does not already exist
|
|
||||||
func (s *stats) getStatsForRoute(app string, path string) *routeStats {
|
|
||||||
if s.apps == nil {
|
|
||||||
s.apps = make(map[string]appStats)
|
|
||||||
}
|
|
||||||
thisAppStats, appFound := s.apps[app]
|
|
||||||
if !appFound {
|
|
||||||
thisAppStats = appStats{routes: make(map[string]*routeStats)}
|
|
||||||
s.apps[app] = thisAppStats
|
|
||||||
}
|
|
||||||
thisRouteStats, pathFound := thisAppStats.routes[path]
|
|
||||||
if !pathFound {
|
|
||||||
thisRouteStats = &routeStats{}
|
|
||||||
thisAppStats.routes[path] = thisRouteStats
|
|
||||||
}
|
|
||||||
return thisRouteStats
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *stats) Enqueue(ctx context.Context, app string, path string) {
|
|
||||||
s.mu.Lock()
|
|
||||||
|
|
||||||
fstats := s.getStatsForRoute(app, path)
|
|
||||||
s.queue++
|
|
||||||
fstats.queue++
|
|
||||||
|
|
||||||
s.mu.Unlock()
|
|
||||||
|
|
||||||
common.IncrementGauge(ctx, queuedMetricName)
|
common.IncrementGauge(ctx, queuedMetricName)
|
||||||
common.IncrementCounter(ctx, callsMetricName)
|
common.IncrementCounter(ctx, callsMetricName)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Call when a function has been queued but cannot be started because of an error
|
// Call when a function has been queued but cannot be started because of an error
|
||||||
func (s *stats) Dequeue(ctx context.Context, app string, path string) {
|
func StatsDequeue(ctx context.Context) {
|
||||||
s.mu.Lock()
|
|
||||||
|
|
||||||
fstats := s.getStatsForRoute(app, path)
|
|
||||||
s.queue--
|
|
||||||
fstats.queue--
|
|
||||||
|
|
||||||
s.mu.Unlock()
|
|
||||||
|
|
||||||
common.DecrementGauge(ctx, queuedMetricName)
|
common.DecrementGauge(ctx, queuedMetricName)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *stats) DequeueAndStart(ctx context.Context, app string, path string) {
|
func StatsDequeueAndStart(ctx context.Context) {
|
||||||
s.mu.Lock()
|
|
||||||
|
|
||||||
fstats := s.getStatsForRoute(app, path)
|
|
||||||
s.queue--
|
|
||||||
s.running++
|
|
||||||
fstats.queue--
|
|
||||||
fstats.running++
|
|
||||||
|
|
||||||
s.mu.Unlock()
|
|
||||||
|
|
||||||
common.DecrementGauge(ctx, queuedMetricName)
|
common.DecrementGauge(ctx, queuedMetricName)
|
||||||
common.IncrementGauge(ctx, runningMetricName)
|
common.IncrementGauge(ctx, runningMetricName)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *stats) Complete(ctx context.Context, app string, path string) {
|
func StatsComplete(ctx context.Context) {
|
||||||
s.mu.Lock()
|
|
||||||
|
|
||||||
fstats := s.getStatsForRoute(app, path)
|
|
||||||
s.running--
|
|
||||||
s.complete++
|
|
||||||
fstats.running--
|
|
||||||
fstats.complete++
|
|
||||||
|
|
||||||
s.mu.Unlock()
|
|
||||||
|
|
||||||
common.DecrementGauge(ctx, runningMetricName)
|
common.DecrementGauge(ctx, runningMetricName)
|
||||||
common.IncrementCounter(ctx, completedMetricName)
|
common.IncrementCounter(ctx, completedMetricName)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *stats) Failed(ctx context.Context, app string, path string) {
|
func StatsFailed(ctx context.Context) {
|
||||||
s.mu.Lock()
|
|
||||||
|
|
||||||
fstats := s.getStatsForRoute(app, path)
|
|
||||||
s.running--
|
|
||||||
s.failed++
|
|
||||||
fstats.running--
|
|
||||||
fstats.failed++
|
|
||||||
|
|
||||||
s.mu.Unlock()
|
|
||||||
|
|
||||||
common.DecrementGauge(ctx, runningMetricName)
|
common.DecrementGauge(ctx, runningMetricName)
|
||||||
common.IncrementCounter(ctx, failedMetricName)
|
common.IncrementCounter(ctx, failedMetricName)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *stats) DequeueAndFail(ctx context.Context, app string, path string) {
|
func StatsDequeueAndFail(ctx context.Context) {
|
||||||
s.mu.Lock()
|
|
||||||
|
|
||||||
fstats := s.getStatsForRoute(app, path)
|
|
||||||
s.queue--
|
|
||||||
s.failed++
|
|
||||||
fstats.queue--
|
|
||||||
fstats.failed++
|
|
||||||
|
|
||||||
s.mu.Unlock()
|
|
||||||
|
|
||||||
common.DecrementGauge(ctx, queuedMetricName)
|
common.DecrementGauge(ctx, queuedMetricName)
|
||||||
common.IncrementCounter(ctx, failedMetricName)
|
common.IncrementCounter(ctx, failedMetricName)
|
||||||
}
|
}
|
||||||
|
|
||||||
func IncrementTimedout(ctx context.Context) {
|
func StatsIncrementTimedout(ctx context.Context) {
|
||||||
common.IncrementCounter(ctx, timedoutMetricName)
|
common.IncrementCounter(ctx, timedoutMetricName)
|
||||||
}
|
}
|
||||||
|
|
||||||
func IncrementErrors(ctx context.Context) {
|
func StatsIncrementErrors(ctx context.Context) {
|
||||||
common.IncrementCounter(ctx, errorsMetricName)
|
common.IncrementCounter(ctx, errorsMetricName)
|
||||||
}
|
}
|
||||||
|
|
||||||
func IncrementTooBusy(ctx context.Context) {
|
func StatsIncrementTooBusy(ctx context.Context) {
|
||||||
common.IncrementCounter(ctx, serverBusyMetricName)
|
common.IncrementCounter(ctx, serverBusyMetricName)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *stats) Stats() Stats {
|
|
||||||
// this creates a Stats from a stats
|
|
||||||
// stats is the internal struc which is continuously updated, and access is controlled using its Mutex
|
|
||||||
// Stats is a deep copy for external use, and can be converted to JSON
|
|
||||||
var stats Stats
|
|
||||||
s.mu.Lock()
|
|
||||||
stats.Running = s.running
|
|
||||||
stats.Complete = s.complete
|
|
||||||
stats.Queue = s.queue
|
|
||||||
stats.Failed = s.failed
|
|
||||||
stats.Apps = make(map[string]AppStats)
|
|
||||||
for appname, thisAppStats := range s.apps {
|
|
||||||
newAppStats := AppStats{Routes: make(map[string]*RouteStats)}
|
|
||||||
stats.Apps[appname] = newAppStats
|
|
||||||
for path, thisRouteStats := range thisAppStats.routes {
|
|
||||||
newAppStats.Routes[path] = &RouteStats{Queue: thisRouteStats.queue, Running: thisRouteStats.running, Complete: thisRouteStats.complete, Failed: thisRouteStats.failed}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
s.mu.Unlock()
|
|
||||||
return stats
|
|
||||||
}
|
|
||||||
|
|
||||||
const (
|
const (
|
||||||
queuedMetricName = "queued"
|
queuedMetricName = "queued"
|
||||||
callsMetricName = "calls"
|
callsMetricName = "calls"
|
||||||
|
|||||||
@@ -435,7 +435,6 @@ func (s *Server) bindHandlers(ctx context.Context) {
|
|||||||
engine.GET("/", handlePing)
|
engine.GET("/", handlePing)
|
||||||
engine.GET("/version", handleVersion)
|
engine.GET("/version", handleVersion)
|
||||||
// TODO: move the following under v1
|
// TODO: move the following under v1
|
||||||
engine.GET("/stats", s.handleStats)
|
|
||||||
engine.GET("/metrics", s.handlePrometheusMetrics)
|
engine.GET("/metrics", s.handlePrometheusMetrics)
|
||||||
|
|
||||||
profilerSetup(engine, "/debug")
|
profilerSetup(engine, "/debug")
|
||||||
|
|||||||
@@ -1,11 +0,0 @@
|
|||||||
package server
|
|
||||||
|
|
||||||
import (
|
|
||||||
"net/http"
|
|
||||||
|
|
||||||
"github.com/gin-gonic/gin"
|
|
||||||
)
|
|
||||||
|
|
||||||
func (s *Server) handleStats(c *gin.Context) {
|
|
||||||
c.JSON(http.StatusOK, s.agent.Stats())
|
|
||||||
}
|
|
||||||
Reference in New Issue
Block a user