fn: container initialization monitoring (#1288)

Container initialization phase consumes resource tracker
resources (token), during lengthy operations.
In order for agent stability/liveness, this phase has
to be evictable/cancelable and time bounded.

With this change, introducing a new system wide environment setting
to bound the time spent in container initialization phase. This phase
includes docker-pull, docker-create, docker-attach, docker-start
and UDS wait operations. This initialization period is also now
considered evictable.

api/agent/slots.go

@@ -43,6 +43,11 @@ type slotToken struct {
 	isBusy  uint32
 }
 
+type slotCaller struct {
+	notify chan error      // notification to caller
+	done   <-chan struct{} // caller done
+}
+
 // LIFO queue that exposes input/output channels along
 // with runner/waiter tracking for agent
 type slotQueue struct {
@@ -50,7 +55,7 @@ type slotQueue struct {
 	cond      *sync.Cond
 	slots     []*slotToken
 	nextId    uint64
-	signaller chan chan error
+	signaller chan *slotCaller
 	statsLock sync.Mutex // protects stats below
 	stats     slotQueueStats
 }
@@ -67,7 +72,7 @@ func NewSlotQueue(key string) *slotQueue {
 		key:       key,
 		cond:      sync.NewCond(new(sync.Mutex)),
 		slots:     make([]*slotToken, 0),
-		signaller: make(chan chan error, 1),
+		signaller: make(chan *slotCaller, 1),
 	}
 
 	return obj