fn-serverless/api/agent/state_trackers.go

package agent

import (
	"context"
	"sync"
	"time"

	"go.opencensus.io/stats"
	"go.opencensus.io/stats/view"
)

type RequestStateType int
type ContainerStateType int

type containerState struct {
	lock  sync.Mutex
	state ContainerStateType
	start time.Time
}

type requestState struct {
	lock  sync.Mutex
	state RequestStateType
	start time.Time
}

type ContainerState interface {
	UpdateState(ctx context.Context, newState ContainerStateType, slots *slotQueue)
}
type RequestState interface {
	UpdateState(ctx context.Context, newState RequestStateType, slots *slotQueue)
}

func NewRequestState() RequestState {
	return &requestState{}
}

func NewContainerState() ContainerState {
	return &containerState{}
}

const (
	RequestStateNone RequestStateType = iota // uninitialized
	RequestStateWait                         // request is waiting
	RequestStateExec                         // request is executing
	RequestStateDone                         // request is done
	RequestStateMax
)

const (
	ContainerStateNone  ContainerStateType = iota // uninitialized
	ContainerStateWait                            // resource (cpu + mem) waiting
	ContainerStateStart                           // launching
	ContainerStateIdle                            // running idle
	ContainerStateBusy                            // running busy
	ContainerStateDone                            // exited/failed/done
	ContainerStateMax
)

var containerGaugeKeys = [ContainerStateMax]string{
	"",
	"container_wait_total",
	"container_start_total",
	"container_idle_total",
	"container_busy_total",
	"container_done_total",
}
var containerTimeKeys = [ContainerStateMax]string{
	"",
	"container_wait_duration_seconds",
	"container_start_duration_seconds",
	"container_idle_duration_seconds",
	"container_busy_duration_seconds",
}

func (c *requestState) UpdateState(ctx context.Context, newState RequestStateType, slots *slotQueue) {

	var now time.Time
	var oldState RequestStateType

	c.lock.Lock()

	// we can only advance our state forward
	if c.state < newState {

		now = time.Now()
		oldState = c.state
		c.state = newState
		c.start = now
	}

	c.lock.Unlock()

	if now.IsZero() {
		return
	}

	// reflect this change to slot mgr if defined (AKA hot)
	if slots != nil {
		slots.enterRequestState(newState)
		slots.exitRequestState(oldState)
	}
}

func (c *containerState) UpdateState(ctx context.Context, newState ContainerStateType, slots *slotQueue) {

	var now time.Time
	var oldState ContainerStateType
	var before time.Time

	c.lock.Lock()

	// except for 1) switching back to idle from busy (hot containers) or 2)
	// to waiting from done, otherwise we can only move forward in states
	if c.state < newState ||
		(c.state == ContainerStateBusy && newState == ContainerStateIdle) ||
		(c.state == ContainerStateDone && newState == ContainerStateIdle) {

		now = time.Now()
		oldState = c.state
		before = c.start
		c.state = newState
		c.start = now
	}

	c.lock.Unlock()

	if now.IsZero() {
		return
	}

	// reflect this change to slot mgr if defined (AKA hot)
	if slots != nil {
		slots.enterContainerState(newState)
		slots.exitContainerState(oldState)
	}

	// update old state stats
	gaugeKey := containerGaugeKeys[oldState]
	if gaugeKey != "" {
		stats.Record(ctx, containerGaugeMeasures[oldState].M(-1))
	}

	timeKey := containerTimeKeys[oldState]
	if timeKey != "" {
		stats.Record(ctx, containerTimeMeasures[oldState].M(int64(now.Sub(before).Round(time.Millisecond))))
	}

	// update new state stats
	gaugeKey = containerGaugeKeys[newState]
	if gaugeKey != "" {
		stats.Record(ctx, containerGaugeMeasures[newState].M(1))
	}
}

var (
	containerGaugeMeasures []*stats.Int64Measure
	containerTimeMeasures  []*stats.Int64Measure
)

func init() {
	// TODO(reed): do we have to do this? the measurements will be tagged on the context, will they be propagated
	// or we have to white list them in the view for them to show up? test...

	containerGaugeMeasures = make([]*stats.Int64Measure, len(containerGaugeKeys))
	for i, key := range containerGaugeKeys {
		if key == "" { // leave nil intentionally, let it panic
			continue
		}
		containerGaugeMeasures[i] = makeMeasure(key, "containers in state "+key, "", view.Count())
	}

	containerTimeMeasures = make([]*stats.Int64Measure, len(containerTimeKeys))

	for i, key := range containerTimeKeys {
		if key == "" {
			continue
		}
		containerTimeMeasures[i] = makeMeasure(key, "time spent in container state "+key, "ms", view.Distribution())
	}
}