Idle Hot Container Freeze/Preempt Support (#733)

* fn: freeze/unfreeze and eject idle under resource contention

api/agent/resource.go

@@ -40,6 +40,9 @@ type ResourceTracker interface {
 	// machine. It must be called before GetResourceToken or GetResourceToken may hang.
 	// Memory is expected to be provided in MB units.
 	IsResourcePossible(memory, cpuQuota uint64, isAsync bool) bool
+
+	// returns number of waiters waiting for a resource token blocked on condition variable
+	GetResourceTokenWaiterCount() uint64
 }
 
 type resourceTracker struct {
@@ -66,6 +69,8 @@ type resourceTracker struct {
 	cpuAsyncUsed uint64
 	// cpu in use for async area in which agent stops dequeuing async jobs
 	cpuAsyncHWMark uint64
+	// number of waiters waiting for a token blocked on the condition variable
+	tokenWaiterCount uint64
 }
 
 func NewResourceTracker() ResourceTracker {
@@ -123,6 +128,17 @@ func (a *resourceTracker) IsResourcePossible(memory uint64, cpuQuota uint64, isA
 	}
 }
 
+// returns number of waiters waiting for a resource token blocked on condition variable
+func (a *resourceTracker) GetResourceTokenWaiterCount() uint64 {
+	var waiters uint64
+
+	a.cond.L.Lock()
+	waiters = a.tokenWaiterCount
+	a.cond.L.Unlock()
+
+	return waiters
+}
+
 // the received token should be passed directly to launch (unconditionally), launch
 // will close this token (i.e. the receiver should not call Close)
 func (a *resourceTracker) GetResourceToken(ctx context.Context, memory uint64, cpuQuota uint64, isAsync bool) <-chan ResourceToken {
@@ -158,7 +174,9 @@ func (a *resourceTracker) GetResourceToken(ctx context.Context, memory uint64, c
 
 		isWaiting = true
 		for !a.isResourceAvailableLocked(memory, cpuQuota, isAsync) && ctx.Err() == nil {
+			a.tokenWaiterCount++
 			c.Wait()
+			a.tokenWaiterCount--
 		}
 		isWaiting = false