additional ctx spans / maid service (#716)

* add spans to async

* clean up / add spans to agent

* there were a few methods which had multiple contexts which existed in the same
scope (this doesn't end well, usually), flattened those out.
* loop bound context cancels now rely on defer (also was brittle)
* runHot had a lot of ctx shuffling, flattened that.
* added some additional spans in certain paths for added granularity
* linked up the hot launcher / run hot / wait hot to _a_ root span, the first
2 are follows from spans, but at least we can see the source of these and also
can see containers launched over a hot launcher's lifetime

I left TODO around the FollowsFrom because OpenCensus doesn't, at least at the
moment, appear to have any idea of FollowsFrom and it was an extra OpenTracing
method (we have to get the span out, start a new span with the option, then
add it to the context... some shuffling required). anyway, was on the fence
about adding at least.

* resource waiters need to manage their own goroutine lifecycle

* if we get an impossible memory request, bail instead of infinite loop

* handle timeout slippery case

* still sucks, but hotLauncher doesn't leak anything. even the time.After timer goroutines

* simplify GetResourceToken

GetCall can guard against the impossible to allocate resource tasks entering
the system by erroring instead of doling them out. this makes GetResourceToken
logic more straightforward for callers, who now simply have the contract that
they won't ever get a token if they let tasks into the agent that can't run
(but GetCall guards this, and there's a test for it).

sorry, I was going to make this only do that, but when I went to fix up the
tests, my last patch went haywire so I fixed that too. this also at least
tries to simplify the hotLaunch loop, which will now no longer leak time.After
timers (which were long, and with signaller, they were many -- I got a stack
trace :) -- this breaks out the bottom half of the logic to check to see if we
need to launch into its own function, and handles the cleaning duties only in
the caller instead of in 2 different select statements. played with this a
bit, no doubt further cleaning could be done, but this _seems_ better.

* fix vet

* add units to exported method contract docs

* oops

api/agent/resource.go

@@ -13,6 +13,7 @@ import (
 	"strings"
 	"sync"
 
+	opentracing "github.com/opentracing/opentracing-go"
 	"github.com/sirupsen/logrus"
 )
 
@@ -24,9 +25,21 @@ const (
 // A simple resource (memory, cpu, disk, etc.) tracker for scheduling.
 // TODO: add cpu, disk, network IO for future
 type ResourceTracker interface {
+	// WaitAsyncResource returns a channel that will send once when there seem to be sufficient
+	// resource levels to run an async task, it is up to the implementer to create policy here.
 	WaitAsyncResource(ctx context.Context) chan struct{}
-	// returns a closed channel if the resource can never me met.
-	GetResourceToken(ctx context.Context, memory uint64, cpuQuota uint64, isAsync bool) <-chan ResourceToken
+
+	// GetResourceToken returns a channel to wait for a resource token on. If the provided context is canceled,
+	// the channel will never receive anything. If it is not possible to fulfill this resource, the channel
+	// will never receive anything (use IsResourcePossible). If a resource token is available for the provided
+	// resource parameters, it will otherwise be sent once on the returned channel. The channel is never closed.
+	// Memory is expected to be provided in MB units.
+	GetResourceToken(ctx context.Context, memory, cpuQuota uint64, isAsync bool) <-chan ResourceToken
+
+	// IsResourcePossible returns whether it's possible to fulfill the requested resources on this
+	// machine. It must be called before GetResourceToken or GetResourceToken may hang.
+	// Memory is expected to be provided in MB units.
+	IsResourcePossible(memory, cpuQuota uint64, isAsync bool) bool
 }
 
 type resourceTracker struct {
@@ -100,7 +113,9 @@ func (a *resourceTracker) isResourceAvailableLocked(memory uint64, cpuQuota uint
 }
 
 // is this request possible to meet? If no, fail quick
-func (a *resourceTracker) isResourcePossible(memory uint64, cpuQuota uint64, isAsync bool) bool {
+func (a *resourceTracker) IsResourcePossible(memory uint64, cpuQuota uint64, isAsync bool) bool {
+	memory = memory * Mem1MB
+
 	if isAsync {
 		return memory <= a.ramAsyncTotal && cpuQuota <= a.cpuAsyncTotal
 	} else {
@@ -111,17 +126,20 @@ func (a *resourceTracker) isResourcePossible(memory uint64, cpuQuota uint64, isA
 // the received token should be passed directly to launch (unconditionally), launch
 // will close this token (i.e. the receiver should not call Close)
 func (a *resourceTracker) GetResourceToken(ctx context.Context, memory uint64, cpuQuota uint64, isAsync bool) <-chan ResourceToken {
-
-	memory = memory * Mem1MB
+	ch := make(chan ResourceToken)
+	if !a.IsResourcePossible(memory, cpuQuota, isAsync) {
+		// return the channel, but never send anything.
+		return ch
+	}
 
 	c := a.cond
 	isWaiting := false
-	ch := make(chan ResourceToken)
 
-	if !a.isResourcePossible(memory, cpuQuota, isAsync) {
-		close(ch)
-		return ch
-	}
+	memory = memory * Mem1MB
+
+	// if we find a resource token, shut down the thread waiting on ctx finish.
+	// alternatively, if the ctx is done, wake up the cond loop.
+	ctx, cancel := context.WithCancel(ctx)
 
 	go func() {
 		<-ctx.Done()
@@ -132,7 +150,10 @@ func (a *resourceTracker) GetResourceToken(ctx context.Context, memory uint64, c
 		c.L.Unlock()
 	}()
 
+	span, ctx := opentracing.StartSpanFromContext(ctx, "agent_get_resource_token")
 	go func() {
+		defer span.Finish()
+		defer cancel()
 		c.L.Lock()
 
 		isWaiting = true
@@ -202,6 +223,10 @@ func (a *resourceTracker) WaitAsyncResource(ctx context.Context) chan struct{} {
 	isWaiting := false
 	c := a.cond
 
+	// if we find a resource token, shut down the thread waiting on ctx finish.
+	// alternatively, if the ctx is done, wake up the cond loop.
+	ctx, cancel := context.WithCancel(ctx)
+
 	go func() {
 		<-ctx.Done()
 		c.L.Lock()
@@ -211,7 +236,10 @@ func (a *resourceTracker) WaitAsyncResource(ctx context.Context) chan struct{} {
 		c.L.Unlock()
 	}()
 
+	span, ctx := opentracing.StartSpanFromContext(ctx, "agent_wait_async_resource")
 	go func() {
+		defer span.Finish()
+		defer cancel()
 		c.L.Lock()
 		isWaiting = true
 		for (a.ramAsyncUsed >= a.ramAsyncHWMark || a.cpuAsyncUsed >= a.cpuAsyncHWMark) && ctx.Err() == nil {