fn: add container state to eviction stats (#1296)

api/agent/agent.go

@@ -1010,7 +1010,7 @@ func (a *agent) runHotReq(ctx context.Context, call *call, state ContainerState,
 	if call.slots.acquireSlot(s) {
 		slot.Close()
 		if isEvictEvent {
-			statsContainerEvicted(ctx)
+			statsContainerEvicted(ctx, state.GetState())
 		}
 		return false
 	}

api/agent/state_trackers.go

@@ -25,6 +25,7 @@ type requestState struct {
 
 type ContainerState interface {
 	UpdateState(ctx context.Context, newState ContainerStateType, slots *slotQueue)
+	GetState() string
 }
 type RequestState interface {
 	UpdateState(ctx context.Context, newState RequestStateType, slots *slotQueue)
@@ -57,6 +58,16 @@ const (
 	ContainerStateMax
 )
 
+var containerStateKeys = [ContainerStateMax]string{
+	"none",
+	"wait",
+	"start",
+	"idle",
+	"paused",
+	"busy",
+	"done",
+}
+
 var containerGaugeKeys = [ContainerStateMax]string{
 	"",
 	"container_wait_total",
@@ -108,6 +119,16 @@ func isIdleState(state ContainerStateType) bool {
 	return state == ContainerStateIdle || state == ContainerStatePaused
 }
 
+func (c *containerState) GetState() string {
+	var res ContainerStateType
+
+	c.lock.Lock()
+	res = c.state
+	c.lock.Unlock()
+
+	return containerStateKeys[res]
+}
+
 func (c *containerState) UpdateState(ctx context.Context, newState ContainerStateType, slots *slotQueue) {
 
 	var now time.Time

api/agent/stats.go

@@ -10,6 +10,11 @@ import (
 	"github.com/sirupsen/logrus"
 	"go.opencensus.io/stats"
 	"go.opencensus.io/stats/view"
+	"go.opencensus.io/tag"
+)
+
+var (
+	containerStateKey = common.MakeKey("container_state")
 )
 
 func statsCalls(ctx context.Context) {
@@ -60,7 +65,14 @@ func statsLBAgentRunnerExecLatency(ctx context.Context, dur time.Duration) {
 	stats.Record(ctx, runnerExecLatencyMeasure.M(int64(dur/time.Millisecond)))
 }
 
-func statsContainerEvicted(ctx context.Context) {
+func statsContainerEvicted(ctx context.Context, containerState string) {
+	ctx, err := tag.New(ctx,
+		tag.Upsert(containerStateKey, containerState),
+	)
+	if err != nil {
+		logrus.Fatal(err)
+	}
+
 	stats.Record(ctx, containerEvictedMeasure.M(0))
 }
 
@@ -226,8 +238,17 @@ func RegisterContainerViews(tagKeys []string, latencyDist []float64) {
 		}
 	}
 
+	// add container state tag for evictions
+	evictTags := make([]string, 0, len(tagKeys)+1)
+	evictTags = append(evictTags, "container_state")
+	for _, key := range tagKeys {
+		if key != "container_state" {
+			evictTags = append(evictTags, key)
+		}
+	}
+
 	err := view.Register(
-		common.CreateView(containerEvictedMeasure, view.Count(), tagKeys),
+		common.CreateView(containerEvictedMeasure, view.Count(), evictTags),
 	)
 	if err != nil {
 		logrus.WithError(err).Fatal("cannot register view")