fn: tests for hung and bad docker repo during docker-pull (#1298)

* fn: tests for hung and bad docker repo during docker-pull

api/agent/drivers/docker/cookie.go

@@ -24,6 +24,9 @@ type cookie struct {
 	task drivers.ContainerTask
 	// pointer to docker driver
 	drv *DockerDriver
+
+	// do we need to remove container at exit?
+	isCreated bool
 }
 
 func (c *cookie) configureLogger(log logrus.FieldLogger) {
@@ -199,7 +202,10 @@ func (c *cookie) configureEnv(log logrus.FieldLogger) {
 
 // implements Cookie
 func (c *cookie) Close(ctx context.Context) error {
-	err := c.drv.removeContainer(ctx, c.task.Id())
+	var err error
+	if c.isCreated {
+		err = c.drv.removeContainer(ctx, c.task.Id())
+	}
 	c.drv.unpickPool(c)
 	c.drv.unpickNetwork(c)
 	return err

api/agent/drivers/docker/docker.go

@@ -80,7 +80,7 @@ func NewDocker(conf drivers.Config) *DockerDriver {
 	driver := &DockerDriver{
 		cancel:   cancel,
 		conf:     conf,
-		docker:   newClient(ctx),
+		docker:   newClient(ctx, conf.MaxRetries),
 		hostname: hostname,
 		auths:    auths,
 	}
@@ -275,6 +275,11 @@ func (drv *DockerDriver) PrepareCookie(ctx context.Context, c drivers.Cookie) er
 		return err
 	}
 
+	// here let's assume we have created container, logically this should be after 'CreateContainer', but we
+	// are not 100% sure that *any* failure to CreateContainer does not ever leave a container around especially
+	// going through fsouza+docker-api.
+	cookie.isCreated = true
+
 	cookie.opts.Context = ctx
 	_, err = drv.docker.CreateContainer(cookie.opts)
 	if err != nil {

api/agent/drivers/docker/docker_client.go

@@ -47,7 +47,7 @@ type dockerClient interface {
 }
 
 // TODO: switch to github.com/docker/engine-api
-func newClient(ctx context.Context) dockerClient {
+func newClient(ctx context.Context, maxRetries uint64) dockerClient {
 	// TODO this was much easier, don't need special settings at the moment
 	// docker, err := docker.NewClient(conf.Docker)
 	client, err := docker.NewClientFromEnv()
@@ -59,12 +59,18 @@ func newClient(ctx context.Context) dockerClient {
 		logrus.WithError(err).Fatal("couldn't connect to docker daemon")
 	}
 
+	// punch in default if not set
+	if maxRetries == 0 {
+		maxRetries = 10
+	}
+
 	go listenEventLoop(ctx, client)
-	return &dockerWrap{client}
+	return &dockerWrap{docker: client, maxRetries: maxRetries}
 }
 
 type dockerWrap struct {
-	docker *docker.Client
+	docker     *docker.Client
+	maxRetries uint64
 }
 
 var (
@@ -210,13 +216,13 @@ func RegisterViews(tagKeys []string, latencyDist []float64) {
 }
 
 func (d *dockerWrap) retry(ctx context.Context, logger logrus.FieldLogger, f func() error) error {
-	var i int
+	var i uint64
 	var err error
 	defer func() { stats.Record(ctx, dockerRetriesMeasure.M(int64(i))) }()
 
 	var b common.Backoff
 	// 10 retries w/o change to backoff is ~13s if ops take ~0 time
-	for ; i < 10; i++ {
+	for ; i < d.maxRetries; i++ {
 		select {
 		case <-ctx.Done():
 			stats.Record(ctx, dockerTimeoutMeasure.M(0))