fn: locked mutex while blocked on I/O considered harmful (#935)

* fn: mutex while waiting I/O considered harmful

*) Removed hold mutex while wait I/O cases these
included possible disk I/O and network I/O.

*) Error/Context Close/Shutdown semantics changed since
the context timeout and comments were misleading. Close
always waits for pending gRPC session to complete.
Context usage here was merely 'wait up to x secs to
report an error' which only logs the error anyway.
Instead, the runner can log the error. And context
still can be passed around perhaps for future opencensus
instrumentation.

api/agent/lb_agent.go

@@ -22,8 +22,7 @@ const (
 	// sleep time when scaling from 0 to 1 runners
 	noCapacityWaitInterval = 1 * time.Second
 	// amount of time to wait to place a request on a runner
-	placementTimeout          = 15 * time.Second
-	runnerPoolShutdownTimeout = 5 * time.Second
+	placementTimeout = 15 * time.Second
 )
 
 type lbAgent struct {
@@ -118,9 +117,7 @@ func (a *lbAgent) Close() error {
 	ch := a.shutWg.CloseGroupNB()
 
 	// finally shutdown the runner pool
-	ctx, cancel := context.WithTimeout(context.Background(), runnerPoolShutdownTimeout)
-	defer cancel()
-	err := a.rp.Shutdown(ctx)
+	err := a.rp.Shutdown(context.Background())
 	if err != nil {
 		logrus.WithError(err).Warn("Runner pool shutdown error")
 	}