Moved runner into this repo, update dep files and now builds.

api/mqs/bolt.go

@@ -14,7 +14,7 @@ import (
 	"github.com/Sirupsen/logrus"
 	"github.com/boltdb/bolt"
 	"github.com/kumokit/functions/api/models"
-	"github.com/kumokit/runner/common"
+	"github.com/kumokit/functions/api/runner/common"
 )
 
 type BoltDbMQ struct {

api/mqs/ironmq.go

@@ -10,9 +10,9 @@ import (
 	"sync"
 
 	"github.com/Sirupsen/logrus"
+	mq_config "github.com/iron-io/iron_go3/config"
+	ironmq "github.com/iron-io/iron_go3/mq"
 	"github.com/kumokit/functions/api/models"
-	mq_config "github.com/kumokit/iron_go3/config"
-	ironmq "github.com/kumokit/iron_go3/mq"
 )
 
 type assoc struct {

api/mqs/memory.go

@@ -10,7 +10,7 @@ import (
 	"github.com/Sirupsen/logrus"
 	"github.com/google/btree"
 	"github.com/kumokit/functions/api/models"
-	"github.com/kumokit/runner/common"
+	"github.com/kumokit/functions/api/runner/common"
 )
 
 type MemoryMQ struct {

api/mqs/redis.go

@@ -12,7 +12,7 @@ import (
 	"github.com/Sirupsen/logrus"
 	"github.com/garyburd/redigo/redis"
 	"github.com/kumokit/functions/api/models"
-	"github.com/kumokit/runner/common"
+	"github.com/kumokit/functions/api/runner/common"
 )
 
 type RedisMQ struct {

api/runner/async_runner.go

@@ -15,7 +15,7 @@ import (
 	"github.com/Sirupsen/logrus"
 	"github.com/kumokit/functions/api/models"
 	"github.com/kumokit/functions/api/runner/task"
-	"github.com/kumokit/runner/common"
+	"github.com/kumokit/functions/api/runner/common"
 )
 
 func getTask(ctx context.Context, url string) (*models.Task, error) {

api/runner/common/backoff.go

@@ -0,0 +1,85 @@
+// Copyright 2016 Iron.io
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package common
+
+import (
+	"math"
+	"math/rand"
+	"sync"
+	"time"
+)
+
+type BoxTime struct{}
+
+func (BoxTime) Now() time.Time                         { return time.Now() }
+func (BoxTime) Sleep(d time.Duration)                  { time.Sleep(d) }
+func (BoxTime) After(d time.Duration) <-chan time.Time { return time.After(d) }
+
+type Backoff int
+
+func (b *Backoff) Sleep() { b.RandomSleep(nil, nil) }
+
+func (b *Backoff) RandomSleep(rng *rand.Rand, clock Clock) {
+	const (
+		maxexp   = 7
+		interval = 25 * time.Millisecond
+	)
+
+	if rng == nil {
+		rng = defaultRNG
+	}
+	if clock == nil {
+		clock = defaultClock
+	}
+
+	// 25-50ms, 50-100ms, 100-200ms, 200-400ms, 400-800ms, 800-1600ms, 1600-3200ms, 3200-6400ms
+	d := time.Duration(math.Pow(2, float64(*b))) * interval
+	d += (d * time.Duration(rng.Float64()))
+
+	clock.Sleep(d)
+
+	if *b < maxexp {
+		(*b)++
+	}
+}
+
+var (
+	defaultRNG   = NewRNG(time.Now().UnixNano())
+	defaultClock = BoxTime{}
+)
+
+func NewRNG(seed int64) *rand.Rand {
+	return rand.New(&lockedSource{src: rand.NewSource(seed)})
+}
+
+// taken from go1.5.1 math/rand/rand.go +233-250
+// bla bla if it puts a hole in the earth don't sue them
+type lockedSource struct {
+	lk  sync.Mutex
+	src rand.Source
+}
+
+func (r *lockedSource) Int63() (n int64) {
+	r.lk.Lock()
+	n = r.src.Int63()
+	r.lk.Unlock()
+	return
+}
+
+func (r *lockedSource) Seed(seed int64) {
+	r.lk.Lock()
+	r.src.Seed(seed)
+	r.lk.Unlock()
+}

api/runner/common/clock.go

@@ -0,0 +1,23 @@
+// Copyright 2016 Iron.io
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package common
+
+import "time"
+
+type Clock interface {
+	Now() time.Time
+	Sleep(time.Duration)
+	After(time.Duration) <-chan time.Time
+}

api/runner/common/ctx.go

@@ -0,0 +1,44 @@
+// Copyright 2016 Iron.io
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package common
+
+import (
+	"context"
+
+	"github.com/Sirupsen/logrus"
+)
+
+// WithLogger stores the logger.
+func WithLogger(ctx context.Context, l logrus.FieldLogger) context.Context {
+	return context.WithValue(ctx, "logger", l)
+}
+
+// Logger returns the structured logger.
+func Logger(ctx context.Context) logrus.FieldLogger {
+	l, ok := ctx.Value("logger").(logrus.FieldLogger)
+	if !ok {
+		return logrus.StandardLogger()
+	}
+	return l
+}
+
+// Attempt at simplifying this whole logger in the context thing
+// Could even make this take a generic map, then the logger that gets returned could be used just like the stdlib too, since it's compatible
+func LoggerWithFields(ctx context.Context, fields logrus.Fields) (context.Context, logrus.FieldLogger) {
+	l := Logger(ctx)
+	l = l.WithFields(fields)
+	ctx = WithLogger(ctx, l)
+	return ctx, l
+}

api/runner/common/environment.go

@@ -0,0 +1,38 @@
+// Copyright 2016 Iron.io
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package common
+
+import (
+	"github.com/kumokit/functions/api/runner/common/stats"
+)
+
+// An Environment is a long lived object that carries around 'configuration'
+// for the program. Other long-lived objects may embed an environment directly
+// into their definition. Environments wrap common functionality like logging
+// and metrics. For short-lived request-response like tasks use `Context`,
+// which wraps an Environment.
+
+type Environment struct {
+	stats.Statter
+}
+
+// Initializers are functions that may set up the environment as they like. By default the environment is 'inactive' in the sense that metrics aren't reported.
+func NewEnvironment(initializers ...func(e *Environment)) *Environment {
+	env := &Environment{&stats.NilStatter{}}
+	for _, init := range initializers {
+		init(env)
+	}
+	return env
+}

api/runner/common/errors.go

@@ -0,0 +1,70 @@
+// Copyright 2016 Iron.io
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package common
+
+import (
+	"io"
+	"net"
+	"syscall"
+)
+
+// Errors that can be directly exposed to task creators/users.
+type UserVisibleError interface {
+	UserVisible() bool
+}
+
+func IsUserVisibleError(err error) bool {
+	ue, ok := err.(UserVisibleError)
+	return ok && ue.UserVisible()
+}
+
+type userVisibleError struct {
+	error
+}
+
+func (u *userVisibleError) UserVisible() bool { return true }
+
+func UserError(err error) error {
+	return &userVisibleError{err}
+}
+
+type Temporary interface {
+	Temporary() bool
+}
+
+func IsTemporary(err error) bool {
+	v, ok := err.(Temporary)
+	return (ok && v.Temporary()) || isNet(err)
+}
+
+func isNet(err error) bool {
+	if _, ok := err.(net.Error); ok {
+		return true
+	}
+
+	switch err := err.(type) {
+	case *net.OpError:
+		return true
+	case syscall.Errno:
+		if err == syscall.ECONNREFUSED { // linux only? maybe ok for prod
+			return true // connection refused
+		}
+	default:
+		if err == io.ErrUnexpectedEOF || err == io.EOF {
+			return true
+		}
+	}
+	return false
+}

api/runner/common/logging.go

@@ -0,0 +1,83 @@
+// Copyright 2016 Iron.io
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package common
+
+import (
+	"net/url"
+	"os"
+
+	"github.com/Sirupsen/logrus"
+)
+
+func SetLogLevel(ll string) {
+	if ll == "" {
+		ll = "info"
+	}
+	logrus.WithFields(logrus.Fields{"level": ll}).Info("Setting log level to")
+	logLevel, err := logrus.ParseLevel(ll)
+	if err != nil {
+		logrus.WithFields(logrus.Fields{"level": ll}).Warn("Could not parse log level, setting to INFO")
+		logLevel = logrus.InfoLevel
+	}
+	logrus.SetLevel(logLevel)
+}
+
+func SetLogDest(to, prefix string) {
+	logrus.SetOutput(os.Stderr) // in case logrus changes their mind...
+	if to == "stderr" {
+		return
+	}
+
+	// possible schemes: { udp, tcp, file }
+	// file url must contain only a path, syslog must contain only a host[:port]
+	// expect: [scheme://][host][:port][/path]
+	// default scheme to udp:// if none given
+
+	url, err := url.Parse(to)
+	if url.Host == "" && url.Path == "" {
+		logrus.WithFields(logrus.Fields{"to": to}).Warn("No scheme on logging url, adding udp://")
+		// this happens when no scheme like udp:// is present
+		to = "udp://" + to
+		url, err = url.Parse(to)
+	}
+	if err != nil {
+		logrus.WithError(err).WithFields(logrus.Fields{"to": to}).Error("could not parse logging URI, defaulting to stderr")
+		return
+	}
+
+	// File URL must contain only `url.Path`. Syslog location must contain only `url.Host`
+	if (url.Host == "" && url.Path == "") || (url.Host != "" && url.Path != "") {
+		logrus.WithFields(logrus.Fields{"to": to, "uri": url}).Error("invalid logging location, defaulting to stderr")
+		return
+	}
+
+	switch url.Scheme {
+	case "udp", "tcp":
+		err = NewSyslogHook(url, prefix)
+		if err != nil {
+			logrus.WithFields(logrus.Fields{"uri": url, "to": to}).WithError(err).Error("unable to connect to syslog, defaulting to stderr")
+			return
+		}
+	case "file":
+		f, err := os.OpenFile(url.Path, os.O_RDWR|os.O_APPEND|os.O_CREATE, 0666)
+		if err != nil {
+			logrus.WithError(err).WithFields(logrus.Fields{"to": to, "path": url.Path}).Error("cannot open file, defaulting to stderr")
+			return
+		}
+		logrus.SetOutput(f)
+	default:
+		logrus.WithFields(logrus.Fields{"scheme": url.Scheme, "to": to}).Error("unknown logging location scheme, defaulting to stderr")
+	}
+}

api/runner/common/stats/aggregator.go

@@ -0,0 +1,188 @@
+// Copyright 2016 Iron.io
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package stats
+
+import (
+	"sync"
+	"time"
+)
+
+type reporter interface {
+	report([]*collectedStat)
+}
+
+type collectedStat struct {
+	Name     string
+	Counters map[string]int64
+	Values   map[string]float64
+	Gauges   map[string]int64
+	Timers   map[string]time.Duration
+
+	avgCounts map[string]uint64
+}
+
+func newCollectedStatUnescaped(name string) *collectedStat {
+	return &collectedStat{
+		Name:      name,
+		Counters:  map[string]int64{},
+		Values:    map[string]float64{},
+		Gauges:    map[string]int64{},
+		Timers:    map[string]time.Duration{},
+		avgCounts: map[string]uint64{},
+	}
+}
+
+// What do you call an alligator in a vest?
+
+// Aggregator collects a stats and merges them together if they've been added
+// previously. Useful for reporters that have low throughput ie stathat.
+type Aggregator struct {
+	// Holds all of our stats based on stat.Name
+	sl    sync.RWMutex
+	stats map[string]*statHolder
+
+	reporters []reporter
+}
+
+func newAggregator(reporters []reporter) *Aggregator {
+	return &Aggregator{
+		stats:     make(map[string]*statHolder),
+		reporters: reporters,
+	}
+}
+
+type statHolder struct {
+	cl sync.RWMutex // Lock on Counters
+	vl sync.RWMutex // Lock on Values
+	s  *collectedStat
+}
+
+func newStatHolder(st *collectedStat) *statHolder {
+	return &statHolder{s: st}
+}
+
+type kind int16
+
+const (
+	counterKind kind = iota
+	valueKind
+	gaugeKind
+	durationKind
+)
+
+func (a *Aggregator) add(component, key string, kind kind, value interface{}) {
+	a.sl.RLock()
+	stat, ok := a.stats[component]
+	a.sl.RUnlock()
+	if !ok {
+		a.sl.Lock()
+		stat, ok = a.stats[component]
+		if !ok {
+			stat = newStatHolder(newCollectedStatUnescaped(component))
+			a.stats[component] = stat
+		}
+		a.sl.Unlock()
+	}
+
+	if kind == counterKind || kind == gaugeKind {
+		var mapPtr map[string]int64
+		if kind == counterKind {
+			mapPtr = stat.s.Counters
+		} else {
+			mapPtr = stat.s.Gauges
+		}
+		value := value.(int64)
+		stat.cl.Lock()
+		mapPtr[key] += value
+		stat.cl.Unlock()
+	}
+
+	/* TODO: this ends up ignoring tags so yeah gg
+	/  lets just calculate a running average for now. Can do percentiles later
+	/  Recalculated Average
+	/
+	/     currentAverage * currentCount + newValue
+	/    ------------------------------------------
+	/                (currentCount +1)
+	/
+	*/
+	if kind == valueKind || kind == durationKind {
+		var typedValue int64
+		if kind == valueKind {
+			typedValue = value.(int64)
+		} else {
+			typedValue = int64(value.(time.Duration))
+		}
+
+		stat.vl.Lock()
+		switch kind {
+		case valueKind:
+			oldAverage := stat.s.Values[key]
+			count := stat.s.avgCounts[key]
+			newAverage := (oldAverage*float64(count) + float64(typedValue)) / (float64(count + 1))
+			stat.s.avgCounts[key] = count + 1
+			stat.s.Values[key] = newAverage
+		case durationKind:
+			oldAverage := float64(stat.s.Timers[key])
+			count := stat.s.avgCounts[key]
+			newAverage := (oldAverage*float64(count) + float64(typedValue)) / (float64(count + 1))
+			stat.s.avgCounts[key] = count + 1
+			stat.s.Timers[key] = time.Duration(newAverage)
+		}
+		stat.vl.Unlock()
+	}
+}
+
+func (a *Aggregator) dump() []*collectedStat {
+	a.sl.Lock()
+	bucket := a.stats
+	// Clear out the maps, effectively resetting our average
+	a.stats = make(map[string]*statHolder)
+	a.sl.Unlock()
+
+	stats := make([]*collectedStat, 0, len(bucket))
+	for _, v := range bucket {
+		stats = append(stats, v.s)
+	}
+	return stats
+}
+
+func (a *Aggregator) report(st []*collectedStat) {
+	stats := a.dump()
+	stats = append(stats, st...)
+	for _, r := range a.reporters {
+		r.report(stats)
+	}
+}
+
+func (r *Aggregator) Inc(component string, stat string, value int64, rate float32) {
+	r.add(component, stat, counterKind, value)
+}
+
+func (r *Aggregator) Gauge(component string, stat string, value int64, rate float32) {
+	r.add(component, stat, gaugeKind, value)
+}
+
+func (r *Aggregator) Measure(component string, stat string, value int64, rate float32) {
+	r.add(component, stat, valueKind, value)
+}
+
+func (r *Aggregator) Time(component string, stat string, value time.Duration, rate float32) {
+	r.add(component, stat, durationKind, value)
+}
+
+func (r *Aggregator) NewTimer(component string, stat string, rate float32) *Timer {
+	return newTimer(r, component, stat, rate)
+}

api/runner/common/stats/aggregator_test.go

@@ -0,0 +1,95 @@
+// Copyright 2016 Iron.io
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package stats
+
+import (
+	"fmt"
+	"math/rand"
+	"testing"
+)
+
+func TestAggregator(t *testing.T) {
+	ag := newAggregator([]reporter{})
+	var sum int64 = 0
+	var times int64 = 0
+	for i := 0; i < 100; i++ {
+		ag.add("mq push", "messages", counterKind, int64(1))
+		ag.add("mq push", "latency", valueKind, int64(i))
+		ag.add("mq pull", "latency", valueKind, int64(i))
+		sum += int64(i)
+		times += 1
+	}
+
+	for _, stat := range ag.dump() {
+		for k, v := range stat.Values {
+			if v != float64(sum)/float64(times) {
+				t.Error("key:", k, "Expected", sum/times, "got", v)
+			}
+		}
+
+		for k, v := range stat.Counters {
+			if v != times {
+				t.Error("key:", k, "Expected", times, "got", v)
+			}
+		}
+	}
+	if len(ag.stats) != 0 {
+		t.Error("expected stats map to be clear, got", len(ag.stats))
+	}
+}
+
+type testStat struct {
+	component string
+	key       string
+	kind      kind
+	value     int64
+}
+
+func BenchmarkAggregatorAdd(b *testing.B) {
+	ag := &Aggregator{
+		stats: make(map[string]*statHolder, 1000),
+	}
+
+	s := createStatList(1000)
+
+	sl := len(s)
+	b.ResetTimer()
+	b.RunParallel(func(pb *testing.PB) {
+		for pb.Next() {
+			e := s[rand.Intn(sl)]
+			ag.add(e.component, e.key, e.kind, e.value)
+		}
+	})
+}
+
+func createStatList(n int) []*testStat {
+	var stats []*testStat
+	for i := 0; i < n; i++ {
+		st := testStat{
+			component: "aggregator_test",
+			key:       fmt.Sprintf("latency.%d", i),
+			kind:      counterKind,
+			value:     1,
+		}
+
+		if rand.Float32() < 0.5 {
+			st.key = fmt.Sprintf("test.%d", i)
+			st.kind = valueKind
+			st.value = 15999
+		}
+		stats = append(stats, &st)
+	}
+	return stats
+}

api/runner/common/stats/log.go

@@ -0,0 +1,45 @@
+// Copyright 2016 Iron.io
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package stats
+
+import (
+	"time"
+
+	"github.com/Sirupsen/logrus"
+)
+
+type LogReporter struct {
+}
+
+func NewLogReporter() *LogReporter {
+	return (&LogReporter{})
+}
+
+func (lr *LogReporter) report(stats []*collectedStat) {
+	for _, s := range stats {
+		f := make(logrus.Fields)
+		for k, v := range s.Counters {
+			f[k] = v
+		}
+		for k, v := range s.Values {
+			f[k] = v
+		}
+		for k, v := range s.Timers {
+			f[k] = time.Duration(v)
+		}
+
+		logrus.WithFields(f).Info(s.Name)
+	}
+}

api/runner/common/stats/mem.go

@@ -0,0 +1,40 @@
+// Copyright 2016 Iron.io
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package stats
+
+import (
+	"runtime"
+	"time"
+)
+
+func StartReportingMemoryAndGC(reporter Statter, d time.Duration) {
+	ticker := time.Tick(d)
+	for {
+		select {
+		case <-ticker:
+			var ms runtime.MemStats
+			runtime.ReadMemStats(&ms)
+
+			prefix := "runtime"
+
+			reporter.Measure(prefix, "allocated", int64(ms.Alloc), 1.0)
+			reporter.Measure(prefix, "allocated.heap", int64(ms.HeapAlloc), 1.0)
+			reporter.Time(prefix, "gc.pause", time.Duration(ms.PauseNs[(ms.NumGC+255)%256]), 1.0)
+
+			// GC CPU percentage.
+			reporter.Measure(prefix, "gc.cpufraction", int64(ms.GCCPUFraction*100), 1.0)
+		}
+	}
+}

api/runner/common/stats/newrelic.go

@@ -0,0 +1,142 @@
+// Copyright 2016 Iron.io
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package stats
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"strings"
+
+	"github.com/Sirupsen/logrus"
+)
+
+type NewRelicAgentConfig struct {
+	Host    string `json:"host"`
+	Version string `json:"version"`
+	Pid     int    `json:"pid"`
+}
+
+// examples: https://docs.newrelic.com/docs/plugins/plugin-developer-resources/developer-reference/metric-data-plugin-api#examples
+type newRelicRequest struct {
+	Agent      *agent       `json:"agent"`
+	Components []*component `json:"components"`
+}
+
+type NewRelicReporterConfig struct {
+	Agent      *NewRelicAgentConfig
+	LicenseKey string `json:"license_key"`
+}
+
+type NewRelicReporter struct {
+	Agent      *agent
+	LicenseKey string
+}
+
+func NewNewRelicReporter(version string, licenseKey string) *NewRelicReporter {
+	r := &NewRelicReporter{}
+	r.Agent = newNewRelicAgent(version)
+	r.LicenseKey = licenseKey
+	return r
+}
+
+func (r *NewRelicReporter) report(stats []*collectedStat) {
+	client := &http.Client{}
+	req := &newRelicRequest{}
+	req.Agent = r.Agent
+	comp := newComponent()
+	comp.Name = "IronMQ"
+	comp.Duration = 60
+	comp.GUID = "io.iron.ironmq"
+	// TODO - NR has a fixed 3 level heirarchy? and we just use 2?
+	req.Components = []*component{comp}
+
+	// now add metrics
+	for _, s := range stats {
+		for k, v := range s.Counters {
+			comp.Metrics[fmt.Sprintf("Component/%s %s", s.Name, k)] = v
+		}
+		for k, v := range s.Values {
+			comp.Metrics[fmt.Sprintf("Component/%s %s", s.Name, k)] = int64(v)
+		}
+		for k, v := range s.Timers {
+			comp.Metrics[fmt.Sprintf("Component/%s %s", s.Name, k)] = int64(v)
+		}
+	}
+
+	metricsJson, err := json.Marshal(req)
+	if err != nil {
+		logrus.WithError(err).Error("error encoding json for NewRelicReporter")
+	}
+
+	jsonAsString := string(metricsJson)
+
+	httpRequest, err := http.NewRequest("POST",
+		"https://platform-api.newrelic.com/platform/v1/metrics",
+		strings.NewReader(jsonAsString))
+	if err != nil {
+		logrus.WithError(err).Error("error creating New Relic request")
+		return
+	}
+	httpRequest.Header.Set("X-License-Key", r.LicenseKey)
+	httpRequest.Header.Set("Content-Type", "application/json")
+	httpRequest.Header.Set("Accept", "application/json")
+	httpResponse, err := client.Do(httpRequest)
+	if err != nil {
+		logrus.WithError(err).Error("error sending http request in NewRelicReporter")
+		return
+	}
+	defer httpResponse.Body.Close()
+	body, err := ioutil.ReadAll(httpResponse.Body)
+	if err != nil {
+		logrus.WithError(err).Error("error reading response body")
+	} else {
+		logrus.Debugln("response", "code", httpResponse.Status, "body", string(body))
+	}
+}
+
+type agent struct {
+	Host    string `json:"host"`
+	Version string `json:"version"`
+	Pid     int    `json:"pid"`
+}
+
+func newNewRelicAgent(Version string) *agent {
+	var err error
+	agent := &agent{
+		Version: Version,
+	}
+	agent.Pid = os.Getpid()
+	if agent.Host, err = os.Hostname(); err != nil {
+		logrus.WithError(err).Error("Can not get hostname")
+		return nil
+	}
+	return agent
+}
+
+type component struct {
+	Name     string           `json:"name"`
+	GUID     string           `json:"guid"`
+	Duration int              `json:"duration"`
+	Metrics  map[string]int64 `json:"metrics"`
+}
+
+func newComponent() *component {
+	c := &component{}
+	c.Metrics = make(map[string]int64)
+	return c
+}

api/runner/common/stats/riemann.go

@@ -0,0 +1,117 @@
+// +build riemann
+
+// Copyright 2016 Iron.io
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package stats
+
+import (
+	"os"
+	"time"
+
+	"github.com/Sirupsen/logrus"
+	"github.com/amir/raidman"
+)
+
+type RiemannClient struct {
+	client     *raidman.Client
+	attributes map[string]string
+}
+
+const (
+	StateNormal = "normal"
+)
+
+func (rc *RiemannClient) Report([]*Stat) {}
+
+func (rc *RiemannClient) Add(s *Stat) {
+	var events []*raidman.Event
+
+	t := time.Now().UnixNano()
+
+	for k, v := range rc.attributes {
+		s.Tags[k] = v
+	}
+
+	for k, v := range s.Counters {
+		events = append(events, &raidman.Event{
+			Ttl:        5.0,
+			Time:       t,
+			State:      StateNormal,
+			Service:    s.Name + " " + k,
+			Metric:     v,
+			Attributes: s.Tags,
+		})
+	}
+
+	for k, v := range s.Values {
+		events = append(events, &raidman.Event{
+			Ttl:        5.0,
+			Time:       t,
+			State:      StateNormal,
+			Service:    s.Name + " " + k,
+			Metric:     v,
+			Attributes: s.Tags,
+		})
+	}
+
+	rc.report(events)
+}
+
+func (rc *RiemannClient) report(events []*raidman.Event) {
+	err := rc.client.SendMulti(events)
+	if err != nil {
+		logrus.WithError(err).Error("error sending to Riemann")
+	}
+}
+
+func (rc *RiemannClient) heartbeat() {
+	events := []*raidman.Event{
+		&raidman.Event{
+			Ttl:        5.0,
+			Time:       time.Now().UnixNano(),
+			State:      StateNormal,
+			Service:    "heartbeat",
+			Metric:     1.0,
+			Attributes: rc.attributes,
+		},
+	}
+	rc.report(events)
+}
+
+func newRiemann(config Config) *RiemannClient {
+	c, err := raidman.Dial("tcp", config.Riemann.RiemannHost)
+	if err != nil {
+		logrus.WithError(err).Error("error dialing Riemann")
+		os.Exit(1)
+	}
+
+	client := &RiemannClient{
+		client:     c,
+		attributes: map[string]string{},
+	}
+
+	for k, v := range config.Tags {
+		client.attributes[k] = v
+	}
+
+	// Send out a heartbeat every second
+	go func(rc *RiemannClient) {
+		for _ = range time.Tick(1 * time.Second) {
+			rc.heartbeat()
+		}
+	}(client)
+
+	return client
+}

api/runner/common/stats/stathat.go

@@ -0,0 +1,65 @@
+// Copyright 2016 Iron.io
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package stats
+
+import (
+	"net/http"
+	"net/url"
+	"strconv"
+
+	"github.com/Sirupsen/logrus"
+)
+
+func postStatHat(key, stat string, values url.Values) {
+	values.Set("stat", stat)
+	values.Set("ezkey", key)
+	resp, err := http.PostForm("http://api.stathat.com/ez", values)
+	if err != nil {
+		logrus.WithError(err).Error("couldn't post to StatHat")
+		return
+	}
+	if resp.StatusCode != 200 {
+		logrus.Errorln("bad status posting to StatHat", "status_code", resp.StatusCode)
+	}
+	resp.Body.Close()
+}
+
+type StatHatReporterConfig struct {
+	Email  string
+	Prefix string
+}
+
+func (shr *StatHatReporterConfig) report(stats []*collectedStat) {
+	for _, s := range stats {
+		for k, v := range s.Counters {
+			n := shr.Prefix + " " + s.Name + " " + k
+			values := url.Values{}
+			values.Set("count", strconv.FormatInt(v, 10))
+			postStatHat(shr.Email, n, values)
+		}
+		for k, v := range s.Values {
+			n := shr.Prefix + " " + s.Name + " " + k
+			values := url.Values{}
+			values.Set("value", strconv.FormatFloat(v, 'f', 3, 64))
+			postStatHat(shr.Email, n, values)
+		}
+		for k, v := range s.Timers {
+			n := shr.Prefix + " " + s.Name + " " + k
+			values := url.Values{}
+			values.Set("value", strconv.FormatInt(int64(v), 10))
+			postStatHat(shr.Email, n, values)
+		}
+	}
+}

api/runner/common/stats/stats.go

@@ -0,0 +1,187 @@
+// Copyright 2016 Iron.io
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package stats
+
+import (
+	"encoding/json"
+	"errors"
+	"net/http"
+	"strings"
+	"time"
+
+	"github.com/Sirupsen/logrus"
+)
+
+type HTTPSubHandler interface {
+	HTTPHandler(relativeUrl []string, w http.ResponseWriter, r *http.Request)
+}
+
+type Config struct {
+	Interval float64 `json:"interval" envconfig:"STATS_INTERVAL"` // seconds
+	History  int     // minutes
+
+	Log      string `json:"log" envconfig:"STATS_LOG"`
+	StatHat  *StatHatReporterConfig
+	NewRelic *NewRelicReporterConfig
+	Statsd   *StatsdConfig
+	GCStats  int `json:"gc_stats" envconfig:"GC_STATS"` // seconds
+}
+
+type Statter interface {
+	Inc(component string, stat string, value int64, rate float32)
+	Gauge(component string, stat string, value int64, rate float32)
+	Measure(component string, stat string, value int64, rate float32)
+	Time(component string, stat string, value time.Duration, rate float32)
+	NewTimer(component string, stat string, rate float32) *Timer
+}
+
+type MultiStatter struct {
+	statters []Statter
+}
+
+func (s *MultiStatter) Inc(component string, stat string, value int64, rate float32) {
+	for _, st := range s.statters {
+		st.Inc(component, stat, value, rate)
+	}
+}
+
+func (s *MultiStatter) Gauge(component string, stat string, value int64, rate float32) {
+	for _, st := range s.statters {
+		st.Gauge(component, stat, value, rate)
+	}
+}
+
+func (s *MultiStatter) Measure(component string, stat string, value int64, rate float32) {
+	for _, st := range s.statters {
+		st.Measure(component, stat, value, rate)
+	}
+}
+
+func (s *MultiStatter) Time(component string, stat string, value time.Duration, rate float32) {
+	for _, st := range s.statters {
+		st.Time(component, stat, value, rate)
+	}
+}
+
+func (s *MultiStatter) NewTimer(component string, stat string, rate float32) *Timer {
+	return newTimer(s, component, stat, rate)
+}
+
+var badDecode error = errors.New("bad stats decode")
+
+func New(config Config) Statter {
+	s := new(MultiStatter)
+
+	if config.Interval == 0.0 {
+		config.Interval = 10.0 // convenience
+	}
+
+	var reporters []reporter
+	if config.StatHat != nil && config.StatHat.Email != "" {
+		reporters = append(reporters, config.StatHat)
+	}
+
+	if config.NewRelic != nil && config.NewRelic.LicenseKey != "" {
+		// NR wants version?
+		// can get it out of the namespace? roll it here?
+		reporters = append(reporters, NewNewRelicReporter("1.0", config.NewRelic.LicenseKey))
+	}
+
+	if config.Log != "" {
+		reporters = append(reporters, NewLogReporter())
+	}
+
+	if len(reporters) > 0 {
+		ag := newAggregator(reporters)
+		s.statters = append(s.statters, ag)
+		go func() {
+			for range time.Tick(time.Duration(config.Interval * float64(time.Second))) {
+				ag.report(nil)
+			}
+		}()
+	}
+
+	if config.Statsd != nil && config.Statsd.StatsdUdpTarget != "" {
+		std, err := NewStatsd(config.Statsd)
+		if err == nil {
+			s.statters = append(s.statters, std)
+		} else {
+			logrus.WithError(err).Error("Couldn't create statsd reporter")
+		}
+	}
+
+	if len(reporters) == 0 && config.Statsd == nil && config.History == 0 {
+		return &NilStatter{}
+	}
+
+	if config.GCStats >= 0 {
+		if config.GCStats == 0 {
+			config.GCStats = 1
+		}
+		go StartReportingMemoryAndGC(s, time.Duration(config.GCStats)*time.Second)
+	}
+
+	return s
+}
+
+func HTTPReturnJson(w http.ResponseWriter, result interface{}) {
+	w.Header().Set("Content-Type", "application/json")
+	res, err := json.Marshal(result)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+	} else {
+		w.Write(res)
+	}
+}
+
+// Convert a string to a stat name by replacing '.' with '_', lowercasing the
+// string and trimming it. Doesn't do any validation, so do try this out
+// locally before sending stats.
+func AsStatField(input string) string {
+	return strings.Replace(strings.ToLower(strings.TrimSpace(input)), ".", "_", -1)
+}
+
+// statsd like API on top of the map manipulation API.
+type Timer struct {
+	statter   Statter
+	component string
+	stat      string
+	start     time.Time
+	rate      float32
+	measured  bool
+}
+
+func newTimer(st Statter, component, stat string, rate float32) *Timer {
+	return &Timer{st, component, stat, time.Now(), rate, false}
+}
+
+func (timer *Timer) Measure() {
+	if timer.measured {
+		return
+	}
+
+	timer.measured = true
+	timer.statter.Time(timer.component, timer.stat, time.Since(timer.start), timer.rate)
+}
+
+type NilStatter struct{}
+
+func (n *NilStatter) Inc(component string, stat string, value int64, rate float32)          {}
+func (n *NilStatter) Gauge(component string, stat string, value int64, rate float32)        {}
+func (n *NilStatter) Measure(component string, stat string, value int64, rate float32)      {}
+func (n *NilStatter) Time(component string, stat string, value time.Duration, rate float32) {}
+func (r *NilStatter) NewTimer(component string, stat string, rate float32) *Timer {
+	return newTimer(r, component, stat, rate)
+}

api/runner/common/stats/statsd.go

@@ -0,0 +1,126 @@
+// Copyright 2016 Iron.io
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package stats
+
+import (
+	"bytes"
+	"fmt"
+	"net"
+	"strings"
+	"time"
+
+	"github.com/cactus/go-statsd-client/statsd"
+)
+
+type StatsdConfig struct {
+	StatsdUdpTarget string `json:"target" mapstructure:"target" envconfig:"STATSD_TARGET"`
+	Interval        int64  `json:"interval" envconfig:"STATSD_INTERVAL"`
+	Prefix          string `json:"prefix" envconfig:"STATSD_PREFIX"`
+}
+
+type keyCreator interface {
+	// The return value of Key *MUST* never have a '.' at the end.
+	Key(stat string) string
+}
+
+type theStatsdReporter struct {
+	keyCreator
+	client statsd.Statter
+}
+
+type prefixKeyCreator struct {
+	parent   keyCreator
+	prefixes []string
+}
+
+func (pkc *prefixKeyCreator) Key(stat string) string {
+	prefix := strings.Join(pkc.prefixes, ".")
+
+	if pkc.parent != nil {
+		prefix = pkc.parent.Key(prefix)
+	}
+
+	if stat == "" {
+		return prefix
+	}
+
+	if prefix == "" {
+		return stat
+	}
+
+	return prefix + "." + stat
+}
+
+func whoami() string {
+	a, _ := net.InterfaceAddrs()
+	for i := range a {
+		// is a textual representation of an IPv4 address
+		z, _, err := net.ParseCIDR(a[i].String())
+		if a[i].Network() == "ip+net" && err == nil && z.To4() != nil {
+			if !bytes.Equal(z, net.ParseIP("127.0.0.1")) {
+				return strings.Replace(fmt.Sprintf("%v", z), ".", "_", -1)
+			}
+		}
+	}
+	return "127_0_0_1" // shrug
+}
+
+// The config.Prefix is sent before each message and can be used to set API
+// keys. The prefix is used as the key prefix.
+// If config is nil, creates a noop reporter.
+//
+//	st, e := NewStatsd(config, "ironmq")
+//	st.Inc("enqueue", 1) -> Actually records to key ironmq.enqueue.
+func NewStatsd(config *StatsdConfig) (*theStatsdReporter, error) {
+	var client statsd.Statter
+	var err error
+	if config != nil {
+		// 512 for now since we are sending to hostedgraphite over the internet.
+		config.Prefix += "." + whoami()
+		client, err = statsd.NewBufferedClient(config.StatsdUdpTarget, config.Prefix, time.Duration(config.Interval)*time.Second, 512)
+	} else {
+		client, err = statsd.NewNoopClient()
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	return &theStatsdReporter{keyCreator: &prefixKeyCreator{}, client: client}, nil
+}
+
+func (sr *theStatsdReporter) Inc(component, stat string, value int64, rate float32) {
+	sr.client.Inc(sr.keyCreator.Key(component+"."+stat), value, rate)
+}
+
+func (sr *theStatsdReporter) Measure(component, stat string, delta int64, rate float32) {
+	sr.client.Timing(sr.keyCreator.Key(component+"."+stat), delta, rate)
+}
+
+func (sr *theStatsdReporter) Time(component, stat string, delta time.Duration, rate float32) {
+	sr.client.TimingDuration(sr.keyCreator.Key(component+"."+stat), delta, rate)
+}
+
+func (sr *theStatsdReporter) Gauge(component, stat string, value int64, rate float32) {
+	sr.client.Gauge(sr.keyCreator.Key(component+"."+stat), value, rate)
+}
+
+func (sr *theStatsdReporter) NewTimer(component string, stat string, rate float32) *Timer {
+	return newTimer(sr, component, stat, rate)
+}
+
+// We need some kind of all-or-nothing sampler where multiple stats can be
+// given the same rate and they are either all logged on that run or none of
+// them are. The statsd library we use ends up doing its own rate calculation
+// which is going to impede doing something like this.

api/runner/common/unix_logging.go

@@ -0,0 +1,22 @@
+// +build !windows,!nacl,!plan9
+
+package common
+
+import (
+	"io/ioutil"
+	"net/url"
+
+	"github.com/Sirupsen/logrus"
+	"github.com/Sirupsen/logrus/hooks/syslog"
+)
+
+func NewSyslogHook(url *url.URL, prefix string) error {
+	syslog, err := logrus_syslog.NewSyslogHook(url.Scheme, url.Host, 0, prefix)
+	if err != nil {
+		return err
+	}
+	logrus.AddHook(syslog)
+	// TODO we could support multiple destinations...
+	logrus.SetOutput(ioutil.Discard)
+	return nil
+}

api/runner/common/win_logging.go

@@ -0,0 +1,12 @@
+// +build !linux,!darwin
+
+package common
+
+import (
+	"errors"
+	"net/url"
+)
+
+func NewSyslogHook(url *url.URL, prefix string) error {
+	return errors.New("Syslog not supported on this system.")
+}

api/runner/common/writers.go

@@ -0,0 +1,204 @@
+// Copyright 2016 Iron.io
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package common
+
+import (
+	"bytes"
+	"errors"
+	"io"
+)
+
+// lineWriter will break apart a stream of data into individual lines.
+// Downstream writer will be called for each complete new line. When Flush
+// is called, a newline will be appended if there isn't one at the end.
+// Not thread-safe
+type LineWriter struct {
+	b *bytes.Buffer
+	w io.Writer
+}
+
+func NewLineWriter(w io.Writer) *LineWriter {
+	return &LineWriter{
+		w: w,
+		b: bytes.NewBuffer(make([]byte, 0, 1024)),
+	}
+}
+
+func (li *LineWriter) Write(p []byte) (int, error) {
+	n, err := li.b.Write(p)
+	if err != nil {
+		return n, err
+	}
+	if n != len(p) {
+		return n, errors.New("short write")
+	}
+
+	for {
+		b := li.b.Bytes()
+		i := bytes.IndexByte(b, '\n')
+		if i < 0 {
+			break
+		}
+
+		l := b[:i+1]
+		ns, err := li.w.Write(l)
+		if err != nil {
+			return ns, err
+		}
+		li.b.Next(len(l))
+	}
+
+	return n, nil
+}
+
+func (li *LineWriter) Flush() (int, error) {
+	b := li.b.Bytes()
+	if len(b) == 0 {
+		return 0, nil
+	}
+
+	if b[len(b)-1] != '\n' {
+		b = append(b, '\n')
+	}
+	return li.w.Write(b)
+}
+
+// HeadLinesWriter stores upto the first N lines in a buffer that can be
+// retrieved via Head().
+type HeadLinesWriter struct {
+	buffer bytes.Buffer
+	max    int
+}
+
+func NewHeadLinesWriter(max int) *HeadLinesWriter {
+	return &HeadLinesWriter{
+		buffer: bytes.Buffer{},
+		max:    max,
+	}
+}
+
+// Writes start failing once the writer has reached capacity.
+// In such cases the return value is the actual count written (may be zero) and io.ErrShortWrite.
+func (h *HeadLinesWriter) Write(p []byte) (n int, err error) {
+	var afterNewLine int
+	for h.max > 0 && afterNewLine < len(p) {
+		idx := bytes.IndexByte(p[afterNewLine:], '\n')
+		if idx == -1 {
+			h.buffer.Write(p[afterNewLine:])
+			afterNewLine = len(p)
+		} else {
+			h.buffer.Write(p[afterNewLine : afterNewLine+idx+1])
+			afterNewLine = afterNewLine + idx + 1
+			h.max--
+		}
+	}
+
+	if afterNewLine == len(p) {
+		return afterNewLine, nil
+	}
+
+	return afterNewLine, io.ErrShortWrite
+}
+
+// The returned bytes alias the buffer, the same restrictions as
+// bytes.Buffer.Bytes() apply.
+func (h *HeadLinesWriter) Head() []byte {
+	return h.buffer.Bytes()
+}
+
+// TailLinesWriter stores upto the last N lines in a buffer that can be retrieved
+// via Tail(). The truncation is only performed when more bytes are received
+// after '\n', so the buffer contents for both these writes are identical.
+//
+// tail writer that captures last 3 lines.
+// 'a\nb\nc\nd\n' -> 'b\nc\nd\n'
+// 'a\nb\nc\nd' -> 'b\nc\nd'
+type TailLinesWriter struct {
+	buffer             bytes.Buffer
+	max                int
+	newlineEncountered bool
+	// Tail is not idempotent without this.
+	tailCalled bool
+}
+
+func NewTailLinesWriter(max int) *TailLinesWriter {
+	return &TailLinesWriter{
+		buffer: bytes.Buffer{},
+		max:    max,
+	}
+}
+
+// Write always succeeds! This is because all len(p) bytes are written to the
+// buffer before it is truncated.
+func (t *TailLinesWriter) Write(p []byte) (n int, err error) {
+	if t.tailCalled {
+		return 0, errors.New("Tail() has already been called.")
+	}
+
+	var afterNewLine int
+	for afterNewLine < len(p) {
+		// This is at the top of the loop so it does not operate on trailing
+		// newlines. That is handled by Tail() where we have full knowledge that it
+		// is indeed the true trailing newline (if any).
+		if t.newlineEncountered {
+			if t.max > 0 {
+				// we still have capacity
+				t.max--
+			} else {
+				// chomp a newline.
+				t.chompNewline()
+			}
+		}
+
+		idx := bytes.IndexByte(p[afterNewLine:], '\n')
+		if idx == -1 {
+			t.buffer.Write(p[afterNewLine:])
+			afterNewLine = len(p)
+			t.newlineEncountered = false
+		} else {
+			t.buffer.Write(p[afterNewLine : afterNewLine+idx+1])
+			afterNewLine = afterNewLine + idx + 1
+			t.newlineEncountered = true
+		}
+
+	}
+	return len(p), nil
+}
+
+func (t *TailLinesWriter) chompNewline() {
+	b := t.buffer.Bytes()
+	idx := bytes.IndexByte(b, '\n')
+	if idx >= 0 {
+		t.buffer.Next(idx + 1)
+	} else {
+		// pretend a trailing newline exists. In the call in Write() this will
+		// never be hit.
+		t.buffer.Truncate(0)
+	}
+}
+
+// The returned bytes alias the buffer, the same restrictions as
+// bytes.Buffer.Bytes() apply.
+//
+// Once Tail() is called, further Write()s error.
+func (t *TailLinesWriter) Tail() []byte {
+	if !t.tailCalled {
+		t.tailCalled = true
+		if t.max <= 0 {
+			t.chompNewline()
+		}
+	}
+	return t.buffer.Bytes()
+}

api/runner/common/writers_test.go

@@ -0,0 +1,149 @@
+// Copyright 2016 Iron.io
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package common
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"testing"
+)
+
+type testSliceWriter struct {
+	b [][]byte
+}
+
+func (tsw *testSliceWriter) Write(p []byte) (n int, err error) {
+	l := make([]byte, len(p))
+	copy(l, p)
+	tsw.b = append(tsw.b, l)
+	return len(p), nil
+}
+
+func TestLineWriter(t *testing.T) {
+	tsw := &testSliceWriter{}
+	lw := NewLineWriter(tsw)
+
+	lineCount := 7
+	lw.Write([]byte("0 line\n1 line\n2 line\n\n4 line"))
+	lw.Write([]byte("+more\n5 line\n"))
+	lw.Write([]byte("6 line"))
+
+	lw.Flush()
+
+	if len(tsw.b) != lineCount {
+		t.Errorf("Expected %v individual rows; got %v", lineCount, len(tsw.b))
+	}
+
+	for x := 0; x < len(tsw.b); x++ {
+		l := fmt.Sprintf("%v line\n", x)
+		if x == 3 {
+			if len(tsw.b[x]) != 1 {
+				t.Errorf("Expected slice with only newline; got %v", tsw.b[x])
+			}
+			continue
+		} else if x == 4 {
+			l = "4 line+more\n"
+		}
+		if !bytes.Equal(tsw.b[x], []byte(l)) {
+			t.Errorf("Expected slice %s equal to %s", []byte(l), tsw.b[x])
+		}
+	}
+}
+
+func TestHeadWriter(t *testing.T) {
+	data := []byte("the quick\n brown\n fox jumped\n over the\n lazy dog.")
+	w := NewHeadLinesWriter(3)
+	_, err := w.Write(data[:4])
+	if err != nil {
+		t.Errorf("Expected nil error on small write")
+	}
+
+	if !bytes.Equal(w.Head(), []byte("the ")) {
+		t.Errorf("Expected 4 bytes in head, got '%s'", w.Head())
+	}
+
+	n, err := w.Write(data[4:16])
+	if n != len(data[4:16]) || err != nil {
+		t.Errorf("HeadWriter Write() does not satisfy contract about failing writes.")
+	}
+
+	if !bytes.Equal(w.Head(), []byte("the quick\n brown")) {
+		t.Errorf("unexpected contents of head, got '%s'", w.Head())
+	}
+
+	n, err = w.Write(data[16:])
+	if n != (29-16) || err != io.ErrShortWrite {
+		t.Errorf("HeadWriter Write() does not satisfy contract about failing writes.")
+	}
+	if !bytes.Equal(w.Head(), data[:29]) {
+		t.Errorf("unexpected contents of head, got '%s'", w.Head())
+	}
+}
+
+func testTail(t *testing.T, n int, output []byte, writes ...[]byte) {
+	w := NewTailLinesWriter(n)
+	for _, slice := range writes {
+		written, err := w.Write(slice)
+		if written != len(slice) || err != nil {
+			t.Errorf("Tail Write() should always succeed, but failed, input=%s, input length = %d, written=%d, err=%s", slice, len(slice), written, err)
+		}
+	}
+	if !bytes.Equal(w.Tail(), output) {
+		t.Errorf("Output did not match for tail writer of length %d: Expected '%s', got '%s'", n, output, w.Tail())
+	}
+}
+
+func TestTailWriter(t *testing.T) {
+	inputs := [][]byte{[]byte("a\nb\n"), []byte("gh"), []byte("\n")}
+	testTail(t, 2, []byte("b\ngh\n"), inputs...)
+}
+
+func TestZeroAndOneTailWriter(t *testing.T) {
+	// zero line writer, with only single line added to it should return empty buffer.
+	testTail(t, 0, []byte(""), []byte("Hello World\n"))
+	testTail(t, 0, []byte(""), []byte("Hello World"))
+
+	b1 := []byte("Hello World")
+	testTail(t, 1, b1, b1)
+
+	b1 = []byte("Hello World\n")
+	testTail(t, 1, b1, b1)
+
+	b2 := []byte("Yeah!\n")
+	testTail(t, 1, b2, b1, b2)
+
+	b1 = []byte("Flat write")
+	b2 = []byte("Yeah!\n")
+	j := bytes.Join([][]byte{b1, b2}, []byte{})
+	testTail(t, 1, j, b1, b2)
+}
+
+func TestTailWriterTrailing(t *testing.T) {
+	input1 := []byte("a\nb\nc\nd\ne")
+	input2 := []byte("a\nb\nc\nd\ne\n")
+	w1 := NewTailLinesWriter(4)
+	w1.Write(input1)
+	w2 := NewTailLinesWriter(4)
+	w2.Write(input2)
+	if !bytes.Equal(w1.Tail(), []byte("b\nc\nd\ne")) {
+		t.Errorf("Tail not working correctly, got '%s'", w1.Tail())
+	}
+
+	t2 := w2.Tail()
+	if !bytes.Equal(w1.Tail(), t2[:len(t2)-1]) {
+		t.Errorf("Tailwriter does not transition correctly over trailing newline. '%s', '%s'", w1.Tail(), t2)
+	}
+}

api/runner/dind/Dockerfile

@@ -0,0 +1,26 @@
+# Copyright 2016 Iron.io
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+FROM docker:1.13.1-dind
+
+RUN apk update && apk upgrade && apk add --no-cache ca-certificates
+
+COPY entrypoint.sh /usr/local/bin/
+COPY dind.sh /usr/local/bin/
+
+ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]
+
+# USAGE: Add a CMD to your own Dockerfile to use this (NOT an ENTRYPOINT, so that this is called)
+# CMD ["./runner"]

api/runner/dind/README.md

@@ -0,0 +1,2 @@
+This is the base image for all Titan's docker-in-docker images. 
+

api/runner/dind/build.sh

@@ -0,0 +1,22 @@
+
+# Copyright 2016 Iron.io
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+set -ex
+
+docker build -t iron/dind:latest .
+
+cd go-dind
+docker build -t iron/go-dind:latest .

api/runner/dind/chaos/Dockerfile

@@ -0,0 +1,27 @@
+# Copyright 2016 Iron.io
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# iron/dind-chaos
+FROM docker:1.12-rc-dind
+
+RUN apk update && apk upgrade && apk add --no-cache ca-certificates
+
+COPY entrypoint.sh /usr/local/bin/
+COPY chaos.sh /usr/local/bin/
+
+ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]
+
+# USAGE: Add a CMD to your own Dockerfile to use this (NOT an ENTRYPOINT, so that this is called)
+# CMD ["./runner"]

api/runner/dind/chaos/README.md

@@ -0,0 +1 @@
+dind docker to periodically kill docker to test against

api/runner/dind/chaos/chaos.sh

@@ -0,0 +1,31 @@
+#!/bin/sh
+
+# Copyright 2016 Iron.io
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+
+sleep 600 # 10 minutes
+
+for i in 1..1000; do
+  pkill -9 dockerd
+  pkill -9 docker-containerd
+  # remove pid file since we killed docker hard
+  rm /var/run/docker.pid
+  sleep 30
+  docker daemon \
+      --host=unix:///var/run/docker.sock \
+      --host=tcp://0.0.0.0:2375 &
+  sleep 300 # 5 minutes
+done

api/runner/dind/chaos/entrypoint.sh

@@ -0,0 +1,32 @@
+#!/bin/sh
+
+# Copyright 2016 Iron.io
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+set -ex
+
+# modified from: https://github.com/docker-library/docker/blob/866c3fbd87e8eeed524fdf19ba2d63288ad49cd2/1.11/dind/dockerd-entrypoint.sh
+# this will run either overlay or aufs as the docker fs driver, if the OS has both, overlay is preferred.
+
+docker daemon \
+		--host=unix:///var/run/docker.sock \
+		--host=tcp://0.0.0.0:2375 &
+
+# wait for daemon to initialize
+sleep 10
+
+/usr/local/bin/chaos.sh &
+
+exec "$@"

api/runner/dind/dind.sh

@@ -0,0 +1,42 @@
+#!/bin/sh
+
+# Copyright 2016 Iron.io
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+# modified from: https://github.com/docker-library/docker/blob/866c3fbd87e8eeed524fdf19ba2d63288ad49cd2/1.11/dind/dockerd-entrypoint.sh
+# this will run either overlay or aufs as the docker fs driver, if the OS has both, overlay is preferred.
+# rewrite overlay to use overlay2 (docker 1.12, linux >=4.x required), see https://docs.docker.com/engine/userguide/storagedriver/selectadriver/#overlay-vs-overlay2
+
+fsdriver=$(grep -Eh -w -m1 "overlay|aufs" /proc/filesystems | cut -f2)
+
+if [ $fsdriver == "overlay" ]; then
+  fsdriver="overlay2"
+fi
+
+cmd="dockerd \
+		--host=unix:///var/run/docker.sock \
+		--host=tcp://0.0.0.0:2375 \
+		--storage-driver=$fsdriver"
+
+# nanny and restart on crashes
+until eval $cmd; do
+  echo "Docker crashed with exit code $?.  Respawning.." >&2
+  # if we just restart it won't work, so start it (it wedges up) and
+  # then kill the wedgie and restart it again and ta da... yea, seriously
+  pidfile=/var/run/docker/libcontainerd/docker-containerd.pid
+  kill -9 $(cat $pidfile)
+  rm $pidfile
+  sleep 1
+done

api/runner/dind/entrypoint.sh

@@ -0,0 +1,24 @@
+#!/bin/sh
+
+# Copyright 2016 Iron.io
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+
+/usr/local/bin/dind.sh &
+
+# wait for daemon to initialize
+sleep 3
+
+exec "$@"

api/runner/dind/release.sh

@@ -0,0 +1,26 @@
+# Copyright 2016 Iron.io
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+./build.sh
+
+docker run --rm -v "$PWD":/app treeder/bump patch
+version=`cat VERSION`
+echo "version $version"
+
+docker tag iron/dind:latest iron/dind:$version
+
+docker push iron/dind:latest
+docker push iron/dind:$version

api/runner/drivers/README.md

@@ -0,0 +1 @@
+This package is intended as a general purpose container abstraction library. With the same code, you can run on Docker, Rkt, etc. 

api/runner/drivers/docker/docker.go

@@ -0,0 +1,638 @@
+// Copyright 2016 Iron.io
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package docker
+
+import (
+	"context"
+	"crypto/tls"
+	"errors"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"net/url"
+	"os"
+	"path"
+	"strings"
+	"time"
+
+	"github.com/Sirupsen/logrus"
+	manifest "github.com/docker/distribution/manifest/schema1"
+	"github.com/fsouza/go-dockerclient"
+	"github.com/heroku/docker-registry-client/registry"
+	"github.com/kumokit/functions/api/runner/common"
+	"github.com/kumokit/functions/api/runner/common/stats"
+	"github.com/kumokit/functions/api/runner/drivers"
+)
+
+const hubURL = "https://registry.hub.docker.com"
+
+var registryClient = &http.Client{
+	Transport: &http.Transport{
+		Dial: (&net.Dialer{
+			Timeout:   10 * time.Second,
+			KeepAlive: 2 * time.Minute,
+		}).Dial,
+		TLSClientConfig: &tls.Config{
+			ClientSessionCache: tls.NewLRUClientSessionCache(8192),
+		},
+		TLSHandshakeTimeout:   10 * time.Second,
+		MaxIdleConnsPerHost:   32, // TODO tune; we will likely be making lots of requests to same place
+		Proxy:                 http.ProxyFromEnvironment,
+		IdleConnTimeout:       90 * time.Second,
+		MaxIdleConns:          512,
+		ExpectContinueTimeout: 1 * time.Second,
+	},
+}
+
+// A drivers.ContainerTask should implement the Auther interface if it would
+// like to use not-necessarily-public docker images for any or all task
+// invocations.
+type Auther interface {
+	// DockerAuth should return docker auth credentials that will authenticate
+	// against a docker registry for a given drivers.ContainerTask.Image(). An
+	// error may be returned which will cause the task not to be run, this can be
+	// useful for an implementer to do things like testing auth configurations
+	// before returning them; e.g. if the implementer would like to impose
+	// certain restrictions on images or if credentials must be acquired right
+	// before runtime and there's an error doing so. If these credentials don't
+	// work, the docker pull will fail and the task will be set to error status.
+	DockerAuth() (docker.AuthConfiguration, error)
+}
+
+type runResult struct {
+	error
+	StatusValue string
+}
+
+func (r *runResult) Error() string {
+	if r.error == nil {
+		return ""
+	}
+	return r.error.Error()
+}
+
+func (r *runResult) Status() string    { return r.StatusValue }
+func (r *runResult) UserVisible() bool { return common.IsUserVisibleError(r.error) }
+
+type DockerDriver struct {
+	conf     drivers.Config
+	docker   dockerClient // retries on *docker.Client, restricts ad hoc *docker.Client usage / retries
+	hostname string
+
+	*common.Environment
+}
+
+// implements drivers.Driver
+func NewDocker(env *common.Environment, conf drivers.Config) *DockerDriver {
+	hostname, err := os.Hostname()
+	if err != nil {
+		logrus.WithError(err).Fatal("couldn't resolve hostname")
+	}
+
+	return &DockerDriver{
+		conf:        conf,
+		docker:      newClient(env),
+		hostname:    hostname,
+		Environment: env,
+	}
+}
+
+// CheckRegistry will return a sizer, which can be used to check the size of an
+// image if the returned error is nil. If the error returned is nil, then
+// authentication against the given credentials was successful, if the
+// configuration does not specify a config.ServerAddress,
+// https://hub.docker.com will be tried.  CheckRegistry is a package level
+// method since rkt can also use docker images, we may be interested in using
+// rkt w/o a docker driver configured; also, we don't have to tote around a
+// driver in any tasker that may be interested in registry information (2/2
+// cases thus far).
+func CheckRegistry(image string, config docker.AuthConfiguration) (Sizer, error) {
+	registry, repo, tag := drivers.ParseImage(image)
+
+	reg, err := registryForConfig(config, registry)
+	if err != nil {
+		return nil, err
+	}
+
+	mani, err := reg.Manifest(repo, tag)
+	if err != nil {
+		logrus.WithFields(logrus.Fields{"username": config.Username, "server": config.ServerAddress, "image": image}).WithError(err).Error("Credentials not authorized, trying next.")
+		//if !isAuthError(err) {
+		//  // TODO we might retry this, since if this was the registry that was supposed to
+		//  // auth the task will erroneously be set to 'error'
+		//}
+
+		return nil, err
+	}
+
+	return &sizer{mani, reg, repo}, nil
+}
+
+// Sizer returns size information. This interface is liable to contain more
+// than a size at some point, change as needed.
+type Sizer interface {
+	Size() (int64, error)
+}
+
+type sizer struct {
+	mani *manifest.SignedManifest
+	reg  *registry.Registry
+	repo string
+}
+
+func (s *sizer) Size() (int64, error) {
+	var sum int64
+	for _, r := range s.mani.References() {
+		desc, err := s.reg.LayerMetadata(s.repo, r.Digest)
+		if err != nil {
+			return 0, err
+		}
+		sum += desc.Size
+	}
+	return sum, nil
+}
+
+func registryURL(addr string) (string, error) {
+	if addr == "" || strings.Contains(addr, "hub.docker.com") || strings.Contains(addr, "index.docker.io") {
+		return hubURL, nil
+	}
+
+	url, err := url.Parse(addr)
+	if err != nil {
+		// TODO we could error the task out from this with a user error but since
+		// we have a list of auths to check, just return the error so as to be
+		// skipped... horrible api as it is
+		logrus.WithFields(logrus.Fields{"auth_addr": addr}).WithError(err).Error("error parsing server address url, skipping")
+		return "", err
+	}
+
+	if url.Scheme == "" {
+		url.Scheme = "https"
+	}
+	url.Path = strings.TrimSuffix(url.Path, "/")
+	url.Path = strings.TrimPrefix(url.Path, "/v2")
+	url.Path = strings.TrimPrefix(url.Path, "/v1") // just try this, if it fails it fails, not supporting v1
+	return url.String(), nil
+}
+
+func isAuthError(err error) bool {
+	// AARGH!
+	if urlError, ok := err.(*url.Error); ok {
+		if httpError, ok := urlError.Err.(*registry.HttpStatusError); ok {
+			if httpError.Response.StatusCode == 401 {
+				return true
+			}
+		}
+	}
+
+	return false
+}
+
+func registryForConfig(config docker.AuthConfiguration, reg string) (*registry.Registry, error) {
+	if reg == "" {
+		reg = config.ServerAddress
+	}
+
+	var err error
+	config.ServerAddress, err = registryURL(reg)
+	if err != nil {
+		return nil, err
+	}
+
+	// Use this instead of registry.New to avoid the Ping().
+	transport := registry.WrapTransport(registryClient.Transport, reg, config.Username, config.Password)
+	r := &registry.Registry{
+		URL: config.ServerAddress,
+		Client: &http.Client{
+			Transport: transport,
+		},
+		Logf: registry.Quiet,
+	}
+	return r, nil
+}
+
+func (drv *DockerDriver) Prepare(ctx context.Context, task drivers.ContainerTask) (drivers.Cookie, error) {
+	var cmd []string
+	if task.Command() != "" {
+		// NOTE: this is hyper-sensitive and may not be correct like this even, but it passes old tests
+		// task.Command() in swapi is always "sh /mnt/task/.runtask" so fields is safe
+		cmd = strings.Fields(task.Command())
+		logrus.WithFields(logrus.Fields{"task_id": task.Id(), "cmd": cmd, "len": len(cmd)}).Debug("docker command")
+	}
+
+	envvars := make([]string, 0, len(task.EnvVars()))
+	for name, val := range task.EnvVars() {
+		envvars = append(envvars, name+"="+val)
+	}
+
+	containerName := newContainerID(task)
+	container := docker.CreateContainerOptions{
+		Name: containerName,
+		Config: &docker.Config{
+			Env:         envvars,
+			Cmd:         cmd,
+			Memory:      int64(drv.conf.Memory),
+			CPUShares:   drv.conf.CPUShares,
+			Hostname:    drv.hostname,
+			Image:       task.Image(),
+			Volumes:     map[string]struct{}{},
+			Labels:      task.Labels(),
+			OpenStdin:   true,
+			AttachStdin: true,
+			StdinOnce:   true,
+		},
+		HostConfig: &docker.HostConfig{},
+		Context:    ctx,
+	}
+
+	volumes := task.Volumes()
+	for _, mapping := range volumes {
+		hostDir := mapping[0]
+		containerDir := mapping[1]
+		container.Config.Volumes[containerDir] = struct{}{}
+		mapn := fmt.Sprintf("%s:%s", hostDir, containerDir)
+		container.HostConfig.Binds = append(container.HostConfig.Binds, mapn)
+		logrus.WithFields(logrus.Fields{"volumes": mapn, "task_id": task.Id()}).Debug("setting volumes")
+	}
+
+	if wd := task.WorkDir(); wd != "" {
+		logrus.WithFields(logrus.Fields{"wd": wd, "task_id": task.Id()}).Debug("setting work dir")
+		container.Config.WorkingDir = wd
+	}
+
+	err := drv.ensureImage(ctx, task)
+	if err != nil {
+		return nil, err
+	}
+
+	createTimer := drv.NewTimer("docker", "create_container", 1.0)
+	_, err = drv.docker.CreateContainer(container)
+	createTimer.Measure()
+	if err != nil {
+		// since we retry under the hood, if the container gets created and retry fails, we can just ignore error
+		if err != docker.ErrContainerAlreadyExists {
+			logrus.WithFields(logrus.Fields{"task_id": task.Id(), "command": container.Config.Cmd, "memory": container.Config.Memory,
+				"cpu_shares": container.Config.CPUShares, "hostname": container.Config.Hostname, "name": container.Name,
+				"image": container.Config.Image, "volumes": container.Config.Volumes, "binds": container.HostConfig.Binds, "container": containerName,
+			}).WithError(err).Error("Could not create container")
+
+			if ce := containerConfigError(err); ce != nil {
+				return nil, common.UserError(fmt.Errorf("Failed to create container from task configuration '%s'", ce))
+			}
+			return nil, err
+		}
+	}
+
+	// discard removal error
+	return &cookie{id: containerName, task: task, drv: drv}, nil
+}
+
+type cookie struct {
+	id   string
+	task drivers.ContainerTask
+	drv  *DockerDriver
+}
+
+func (c *cookie) Close() error { return c.drv.removeContainer(c.id) }
+
+func (c *cookie) Run(ctx context.Context) (drivers.RunResult, error) {
+	return c.drv.run(ctx, c.id, c.task)
+}
+
+func (drv *DockerDriver) removeContainer(container string) error {
+	removeTimer := drv.NewTimer("docker", "remove_container", 1.0)
+	defer removeTimer.Measure()
+	err := drv.docker.RemoveContainer(docker.RemoveContainerOptions{
+		ID: container, Force: true, RemoveVolumes: true})
+
+	if err != nil {
+		logrus.WithError(err).WithFields(logrus.Fields{"container": container}).Error("error removing container")
+	}
+	return nil
+}
+
+func (drv *DockerDriver) ensureImage(ctx context.Context, task drivers.ContainerTask) error {
+	reg, _, _ := drivers.ParseImage(task.Image())
+
+	// ask for docker creds before looking for image, as the tasker may need to
+	// validate creds even if the image is downloaded.
+
+	var config docker.AuthConfiguration // default, tries docker hub w/o user/pass
+	if task, ok := task.(Auther); ok {
+		var err error
+		config, err = task.DockerAuth()
+		if err != nil {
+			return err
+		}
+	}
+
+	if reg != "" {
+		config.ServerAddress = reg
+	}
+
+	// see if we already have it, if not, pull it
+	_, err := drv.docker.InspectImage(task.Image())
+	if err == docker.ErrNoSuchImage {
+		err = drv.pullImage(ctx, task, config)
+	}
+
+	return err
+}
+
+func (drv *DockerDriver) pullImage(ctx context.Context, task drivers.ContainerTask, config docker.AuthConfiguration) error {
+	log := common.Logger(ctx)
+
+	reg, repo, tag := drivers.ParseImage(task.Image())
+	globalRepo := path.Join(reg, repo)
+
+	pullTimer := drv.NewTimer("docker", "pull_image", 1.0)
+	defer pullTimer.Measure()
+
+	drv.Inc("docker", "pull_image_count."+stats.AsStatField(task.Image()), 1, 1)
+
+	if reg != "" {
+		config.ServerAddress = reg
+	}
+
+	var err error
+	config.ServerAddress, err = registryURL(config.ServerAddress)
+	if err != nil {
+		return err
+	}
+
+	log.WithFields(logrus.Fields{"registry": config.ServerAddress, "username": config.Username, "image": task.Image()}).Info("Pulling image")
+
+	err = drv.docker.PullImage(docker.PullImageOptions{Repository: globalRepo, Tag: tag, Context: ctx}, config)
+	if err != nil {
+		drv.Inc("task", "error.pull."+stats.AsStatField(task.Image()), 1, 1)
+		log.WithFields(logrus.Fields{"registry": config.ServerAddress, "username": config.Username, "image": task.Image()}).WithError(err).Error("Failed to pull image")
+
+		// TODO need to inspect for hub or network errors and pick.
+		return common.UserError(fmt.Errorf("Failed to pull image '%s': %s", task.Image(), err))
+
+		// TODO what about a case where credentials were good, then credentials
+		// were invalidated -- do we need to keep the credential cache docker
+		// driver side and after pull for this case alone?
+	}
+
+	return nil
+}
+
+// Run executes the docker container. If task runs, drivers.RunResult will be returned. If something fails outside the task (ie: Docker), it will return error.
+// The docker driver will attempt to cast the task to a Auther. If that succeeds, private image support is available. See the Auther interface for how to implement this.
+func (drv *DockerDriver) run(ctx context.Context, container string, task drivers.ContainerTask) (drivers.RunResult, error) {
+	log := common.Logger(ctx)
+	timeout := task.Timeout()
+
+	var cancel context.CancelFunc
+	if timeout <= 0 {
+		ctx, cancel = context.WithCancel(ctx)
+	} else {
+		ctx, cancel = context.WithTimeout(ctx, timeout)
+	}
+	defer cancel() // do this so that after Run exits, nanny and collect stop
+	var complete bool
+	defer func() { complete = true }() // run before cancel is called
+	ctx = context.WithValue(ctx, completeKey, &complete)
+
+	go drv.nanny(ctx, container)
+	go drv.collectStats(ctx, container, task)
+
+	mwOut, mwErr := task.Logger()
+
+	timer := drv.NewTimer("docker", "attach_container", 1)
+	waiter, err := drv.docker.AttachToContainerNonBlocking(docker.AttachToContainerOptions{
+		Container: container, OutputStream: mwOut, ErrorStream: mwErr,
+		Stream: true, Logs: true, Stdout: true, Stderr: true,
+		Stdin: true, InputStream: task.Input()})
+	timer.Measure()
+	if err != nil {
+		return nil, err
+	}
+
+	err = drv.startTask(ctx, container)
+	if err != nil {
+		return nil, err
+	}
+
+	taskTimer := drv.NewTimer("docker", "container_runtime", 1)
+
+	// can discard error, inspect will tell us about the task and wait will retry under the hood
+	drv.docker.WaitContainerWithContext(container, ctx)
+	taskTimer.Measure()
+
+	waiter.Close()
+	err = waiter.Wait()
+	if err != nil {
+		// TODO need to make sure this error isn't just a context error or something we can ignore
+		log.WithError(err).Error("attach to container returned error, task may be missing logs")
+	}
+
+	status, err := drv.status(ctx, container)
+	return &runResult{
+		StatusValue: status,
+		error:       err,
+	}, nil
+}
+
+const completeKey = "complete"
+
+// watch for cancel or timeout and kill process.
+func (drv *DockerDriver) nanny(ctx context.Context, container string) {
+	select {
+	case <-ctx.Done():
+		if *(ctx.Value(completeKey).(*bool)) {
+			return
+		}
+		drv.cancel(container)
+	}
+}
+
+func (drv *DockerDriver) cancel(container string) {
+	stopTimer := drv.NewTimer("docker", "stop_container", 1.0)
+	err := drv.docker.StopContainer(container, 30)
+	stopTimer.Measure()
+	if err != nil {
+		logrus.WithError(err).WithFields(logrus.Fields{"container": container, "errType": fmt.Sprintf("%T", err)}).Error("something managed to escape our retries web, could not kill container")
+	}
+}
+
+func (drv *DockerDriver) collectStats(ctx context.Context, container string, task drivers.ContainerTask) {
+	done := make(chan bool)
+	defer close(done)
+	dstats := make(chan *docker.Stats, 1)
+	go func() {
+		// NOTE: docker automatically streams every 1s. we can skip or avg samples if we'd like but
+		// the memory overhead is < 1MB for 3600 stat points so this seems fine, seems better to stream
+		// (internal docker api streams) than open/close stream for 1 sample over and over.
+		// must be called in goroutine, docker.Stats() blocks
+		err := drv.docker.Stats(docker.StatsOptions{
+			ID:     container,
+			Stats:  dstats,
+			Stream: true,
+			Done:   done, // A flag that enables stopping the stats operation
+		})
+
+		if err != nil && err != io.ErrClosedPipe {
+			logrus.WithError(err).WithFields(logrus.Fields{"container": container, "task_id": task.Id()}).Error("error streaming docker stats for task")
+		}
+	}()
+
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case ds, ok := <-dstats:
+			if !ok {
+				return
+			}
+			task.WriteStat(cherryPick(ds))
+		}
+	}
+}
+
+func cherryPick(ds *docker.Stats) drivers.Stat {
+	// TODO cpu % is as a % of the whole system... cpu is weird since we're sharing it
+	// across a bunch of containers and it scales based on how many we're sharing with,
+	// do we want users to see as a % of system?
+	systemDelta := float64(ds.CPUStats.SystemCPUUsage - ds.PreCPUStats.SystemCPUUsage)
+	cores := float64(len(ds.CPUStats.CPUUsage.PercpuUsage))
+	var cpuUser, cpuKernel, cpuTotal float64
+	if systemDelta > 0 {
+		// TODO we could leave these in docker format and let hud/viz tools do this instead of us... like net is, could do same for mem, too. thoughts?
+		cpuUser = (float64(ds.CPUStats.CPUUsage.UsageInUsermode-ds.PreCPUStats.CPUUsage.UsageInUsermode) / systemDelta) * cores * 100.0
+		cpuKernel = (float64(ds.CPUStats.CPUUsage.UsageInKernelmode-ds.PreCPUStats.CPUUsage.UsageInKernelmode) / systemDelta) * cores * 100.0
+		cpuTotal = (float64(ds.CPUStats.CPUUsage.TotalUsage-ds.PreCPUStats.CPUUsage.TotalUsage) / systemDelta) * cores * 100.0
+	}
+
+	var rx, tx float64
+	for _, v := range ds.Networks {
+		rx += float64(v.RxBytes)
+		tx += float64(v.TxBytes)
+	}
+
+	var blkRead, blkWrite uint64
+	for _, bioEntry := range ds.BlkioStats.IOServiceBytesRecursive {
+		switch strings.ToLower(bioEntry.Op) {
+		case "read":
+			blkRead = blkRead + bioEntry.Value
+		case "write":
+			blkWrite = blkWrite + bioEntry.Value
+		}
+	}
+
+	return drivers.Stat{
+		Timestamp: ds.Read,
+		Metrics: map[string]uint64{
+			// source: https://godoc.org/github.com/fsouza/go-dockerclient#Stats
+			// ex (for future expansion): {"read":"2016-08-03T18:08:05Z","pids_stats":{},"network":{},"networks":{"eth0":{"rx_bytes":508,"tx_packets":6,"rx_packets":6,"tx_bytes":508}},"memory_stats":{"stats":{"cache":16384,"pgpgout":281,"rss":8826880,"pgpgin":2440,"total_rss":8826880,"hierarchical_memory_limit":536870912,"total_pgfault":3809,"active_anon":8843264,"total_active_anon":8843264,"total_pgpgout":281,"total_cache":16384,"pgfault":3809,"total_pgpgin":2440},"max_usage":8953856,"usage":8953856,"limit":536870912},"blkio_stats":{"io_service_bytes_recursive":[{"major":202,"op":"Read"},{"major":202,"op":"Write"},{"major":202,"op":"Sync"},{"major":202,"op":"Async"},{"major":202,"op":"Total"}],"io_serviced_recursive":[{"major":202,"op":"Read"},{"major":202,"op":"Write"},{"major":202,"op":"Sync"},{"major":202,"op":"Async"},{"major":202,"op":"Total"}]},"cpu_stats":{"cpu_usage":{"percpu_usage":[47641874],"usage_in_usermode":30000000,"total_usage":47641874},"system_cpu_usage":8880800500000000,"throttling_data":{}},"precpu_stats":{"cpu_usage":{"percpu_usage":[44946186],"usage_in_usermode":30000000,"total_usage":44946186},"system_cpu_usage":8880799510000000,"throttling_data":{}}}
+			// mostly stolen values from docker stats cli api...
+
+			// net
+			"net_rx": uint64(rx),
+			"net_tx": uint64(tx),
+			// mem
+			"mem_limit": ds.MemoryStats.Limit,
+			"mem_usage": ds.MemoryStats.Usage,
+			// i/o
+			"disk_read":  blkRead,
+			"disk_write": blkWrite,
+			// cpu
+			"cpu_user":   uint64(cpuUser),
+			"cpu_total":  uint64(cpuTotal),
+			"cpu_kernel": uint64(cpuKernel),
+		},
+	}
+}
+
+// Introduces some randomness to prevent container name clashes where task ID remains the same.
+func newContainerID(task drivers.ContainerTask) string {
+	return fmt.Sprintf("task-%d-%s", time.Now().UnixNano(), task.Id())
+}
+
+func (drv *DockerDriver) startTask(ctx context.Context, container string) error {
+	log := common.Logger(ctx)
+	startTimer := drv.NewTimer("docker", "start_container", 1.0)
+	log.WithFields(logrus.Fields{"container": container}).Debug("Starting container execution")
+	err := drv.docker.StartContainerWithContext(container, nil, ctx)
+	startTimer.Measure()
+	if err != nil {
+		dockerErr, ok := err.(*docker.Error)
+		_, containerAlreadyRunning := err.(*docker.ContainerAlreadyRunning)
+		if containerAlreadyRunning || (ok && dockerErr.Status == 304) {
+			// 304=container already started -- so we can ignore error
+		} else {
+			return err
+		}
+	}
+	return nil
+}
+
+func (drv *DockerDriver) status(ctx context.Context, container string) (status string, err error) {
+	log := common.Logger(ctx)
+
+	cinfo, err := drv.docker.InspectContainer(container)
+	if err != nil {
+		// this is pretty sad, but better to say we had an error than to not.
+		// task has run to completion and logs will be uploaded, user can decide
+		log.WithFields(logrus.Fields{"container": container}).WithError(err).Error("Inspecting container")
+		return drivers.StatusError, err
+	}
+
+	exitCode := cinfo.State.ExitCode
+	log.WithFields(logrus.Fields{
+		"exit_code":          exitCode,
+		"container_running":  cinfo.State.Running,
+		"container_status":   cinfo.State.Status,
+		"container_finished": cinfo.State.FinishedAt,
+		"container_error":    cinfo.State.Error,
+	}).Info("container status")
+
+	select { // do this after inspect so we can see exit code
+	case <-ctx.Done(): // check if task was canceled or timed out
+		switch ctx.Err() {
+		case context.DeadlineExceeded:
+			return drivers.StatusTimeout, nil
+		case context.Canceled:
+			return drivers.StatusCancelled, nil
+		}
+	default:
+	}
+
+	if cinfo.State.Running {
+		log.Warn("getting status of task that is still running, need to fix this")
+		return drivers.StatusError, errors.New("task in running state but not timed out. weird")
+	}
+
+	switch exitCode {
+	default:
+		return drivers.StatusError, common.UserError(fmt.Errorf("exit code %d", exitCode))
+	case 0:
+		return drivers.StatusSuccess, nil
+	case 137: // OOM
+		drv.Inc("docker", "oom", 1, 1)
+		if !cinfo.State.OOMKilled {
+			// It is possible that the host itself is running out of memory and
+			// the host kernel killed one of the container processes.
+			// See: https://github.com/docker/docker/issues/15621
+			// TODO reed: isn't an OOM an OOM? this is wasting space imo
+			log.WithFields(logrus.Fields{"container": container}).Info("Setting task as OOM killed, but docker disagreed.")
+			drv.Inc("docker", "possible_oom_false_alarm", 1, 1.0)
+		}
+
+		return drivers.StatusKilled, drivers.ErrOutOfMemory
+	}
+}

api/runner/drivers/docker/docker_client.go

@@ -0,0 +1,315 @@
+// +build go1.7
+
+// Copyright 2016 Iron.io
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package docker
+
+import (
+	"context"
+	"crypto/tls"
+	"encoding/json"
+	"fmt"
+	"net"
+	"net/http"
+	"strings"
+	"time"
+
+	"github.com/Sirupsen/logrus"
+	"github.com/fsouza/go-dockerclient"
+	"github.com/kumokit/functions/api/runner/common"
+)
+
+const (
+	retryTimeout = 10 * time.Minute
+)
+
+// wrap docker client calls so we can retry 500s, kind of sucks but fsouza doesn't
+// bake in retries we can use internally, could contribute it at some point, would
+// be much more convenient if we didn't have to do this, but it's better than ad hoc retries.
+// also adds timeouts to many operations, varying by operation
+// TODO could generate this, maybe not worth it, may not change often
+type dockerClient interface {
+	// Each of these are github.com/fsouza/go-dockerclient methods
+
+	AttachToContainerNonBlocking(opts docker.AttachToContainerOptions) (docker.CloseWaiter, error)
+	WaitContainerWithContext(id string, ctx context.Context) (int, error)
+	StartContainerWithContext(id string, hostConfig *docker.HostConfig, ctx context.Context) error
+	CreateContainer(opts docker.CreateContainerOptions) (*docker.Container, error)
+	RemoveContainer(opts docker.RemoveContainerOptions) error
+	PullImage(opts docker.PullImageOptions, auth docker.AuthConfiguration) error
+	InspectImage(name string) (*docker.Image, error)
+	InspectContainer(id string) (*docker.Container, error)
+	StopContainer(id string, timeout uint) error
+	Stats(opts docker.StatsOptions) error
+}
+
+// TODO: switch to github.com/docker/engine-api
+func newClient(env *common.Environment) dockerClient {
+	// TODO this was much easier, don't need special settings at the moment
+	// docker, err := docker.NewClient(conf.Docker)
+	client, err := docker.NewClientFromEnv()
+	if err != nil {
+		logrus.WithError(err).Fatal("couldn't create docker client")
+	}
+
+	t := &http.Transport{
+		Dial: (&net.Dialer{
+			Timeout:   10 * time.Second,
+			KeepAlive: 1 * time.Minute,
+		}).Dial,
+		TLSClientConfig: &tls.Config{
+			ClientSessionCache: tls.NewLRUClientSessionCache(8192),
+		},
+		TLSHandshakeTimeout:   10 * time.Second,
+		MaxIdleConnsPerHost:   512,
+		Proxy:                 http.ProxyFromEnvironment,
+		MaxIdleConns:          512,
+		IdleConnTimeout:       90 * time.Second,
+		ExpectContinueTimeout: 1 * time.Second,
+	}
+
+	client.HTTPClient = &http.Client{Transport: t}
+
+	if err := client.Ping(); err != nil {
+		logrus.WithError(err).Fatal("couldn't connect to docker daemon")
+	}
+
+	client.SetTimeout(120 * time.Second)
+
+	// get 2 clients, one with a small timeout, one with no timeout to use contexts
+
+	clientNoTimeout, err := docker.NewClientFromEnv()
+	if err != nil {
+		logrus.WithError(err).Fatal("couldn't create other docker client")
+	}
+
+	clientNoTimeout.HTTPClient = &http.Client{Transport: t}
+
+	if err := clientNoTimeout.Ping(); err != nil {
+		logrus.WithError(err).Fatal("couldn't connect to other docker daemon")
+	}
+
+	return &dockerWrap{client, clientNoTimeout, env}
+}
+
+type dockerWrap struct {
+	docker          *docker.Client
+	dockerNoTimeout *docker.Client
+	*common.Environment
+}
+
+func (d *dockerWrap) retry(ctx context.Context, f func() error) error {
+	var b common.Backoff
+	for {
+		select {
+		case <-ctx.Done():
+			d.Inc("task", "fail.docker", 1, 1)
+			logrus.WithError(ctx.Err()).Warnf("retrying on docker errors timed out, restart docker or rotate this instance?")
+			return ctx.Err()
+		default:
+		}
+
+		err := filter(f())
+		if common.IsTemporary(err) || isDocker50x(err) {
+			logrus.WithError(err).Warn("docker temporary error, retrying")
+			b.Sleep()
+			d.Inc("task", "error.docker", 1, 1)
+			continue
+		}
+		if err != nil {
+			d.Inc("task", "error.docker", 1, 1)
+		}
+		return err
+	}
+}
+
+func isDocker50x(err error) bool {
+	derr, ok := err.(*docker.Error)
+	return ok && derr.Status >= 500
+}
+
+func containerConfigError(err error) error {
+	derr, ok := err.(*docker.Error)
+	if ok && derr.Status == 400 {
+		// derr.Message is a JSON response from docker, which has a "message" field we want to extract if possible.
+		var v struct {
+			Msg string `json:"message"`
+		}
+
+		err := json.Unmarshal([]byte(derr.Message), &v)
+		if err != nil {
+			// If message was not valid JSON, the raw body is still better than nothing.
+			return fmt.Errorf("%s", derr.Message)
+		}
+		return fmt.Errorf("%s", v.Msg)
+	}
+
+	return nil
+}
+
+type temporary struct {
+	error
+}
+
+func (t *temporary) Temporary() bool { return true }
+
+func temp(err error) error {
+	return &temporary{err}
+}
+
+// some 500s are totally cool
+func filter(err error) error {
+	// "API error (500): {\"message\":\"service endpoint with name task-57d722ecdecb9e7be16aff17 already exists\"}\n" -> ok since container exists
+	switch {
+	default:
+		return err
+	case err == nil:
+		return err
+	case strings.Contains(err.Error(), "service endpoint with name"):
+	}
+	logrus.WithError(err).Warn("filtering error")
+	return nil
+}
+
+func filterNoSuchContainer(err error) error {
+	if err == nil {
+		return nil
+	}
+	_, containerNotFound := err.(*docker.NoSuchContainer)
+	dockerErr, ok := err.(*docker.Error)
+	if containerNotFound || (ok && dockerErr.Status == 404) {
+		logrus.WithError(err).Error("filtering error")
+		return nil
+	}
+	return err
+}
+
+func filterNotRunning(err error) error {
+	if err == nil {
+		return nil
+	}
+
+	_, containerNotRunning := err.(*docker.ContainerNotRunning)
+	dockerErr, ok := err.(*docker.Error)
+	if containerNotRunning || (ok && dockerErr.Status == 304) {
+		logrus.WithError(err).Error("filtering error")
+		return nil
+	}
+
+	return err
+}
+
+func (d *dockerWrap) AttachToContainerNonBlocking(opts docker.AttachToContainerOptions) (w docker.CloseWaiter, err error) {
+	ctx, cancel := context.WithTimeout(context.Background(), retryTimeout)
+	defer cancel()
+	err = d.retry(ctx, func() error {
+		w, err = d.docker.AttachToContainerNonBlocking(opts)
+		if err != nil {
+			// always retry if attach errors, task is running, we want logs!
+			err = temp(err)
+		}
+		return err
+	})
+	return w, err
+}
+
+func (d *dockerWrap) WaitContainerWithContext(id string, ctx context.Context) (code int, err error) {
+	err = d.retry(ctx, func() error {
+		code, err = d.dockerNoTimeout.WaitContainerWithContext(id, ctx)
+		return err
+	})
+	return code, filterNoSuchContainer(err)
+}
+
+func (d *dockerWrap) StartContainerWithContext(id string, hostConfig *docker.HostConfig, ctx context.Context) (err error) {
+	err = d.retry(ctx, func() error {
+		err = d.dockerNoTimeout.StartContainerWithContext(id, hostConfig, ctx)
+		if _, ok := err.(*docker.NoSuchContainer); ok {
+			// for some reason create will sometimes return successfully then say no such container here. wtf. so just retry like normal
+			return temp(err)
+		}
+		return err
+	})
+	return err
+}
+
+func (d *dockerWrap) CreateContainer(opts docker.CreateContainerOptions) (c *docker.Container, err error) {
+	err = d.retry(opts.Context, func() error {
+		c, err = d.dockerNoTimeout.CreateContainer(opts)
+		return err
+	})
+	return c, err
+}
+
+func (d *dockerWrap) PullImage(opts docker.PullImageOptions, auth docker.AuthConfiguration) (err error) {
+	err = d.retry(opts.Context, func() error {
+		err = d.dockerNoTimeout.PullImage(opts, auth)
+		return err
+	})
+	return err
+}
+
+func (d *dockerWrap) RemoveContainer(opts docker.RemoveContainerOptions) (err error) {
+	ctx, cancel := context.WithTimeout(context.Background(), retryTimeout)
+	defer cancel()
+	err = d.retry(ctx, func() error {
+		err = d.docker.RemoveContainer(opts)
+		return err
+	})
+	return filterNoSuchContainer(err)
+}
+
+func (d *dockerWrap) InspectImage(name string) (i *docker.Image, err error) {
+	ctx, cancel := context.WithTimeout(context.Background(), retryTimeout)
+	defer cancel()
+	err = d.retry(ctx, func() error {
+		i, err = d.docker.InspectImage(name)
+		return err
+	})
+	return i, err
+}
+
+func (d *dockerWrap) InspectContainer(id string) (c *docker.Container, err error) {
+	ctx, cancel := context.WithTimeout(context.Background(), retryTimeout)
+	defer cancel()
+	err = d.retry(ctx, func() error {
+		c, err = d.docker.InspectContainer(id)
+		return err
+	})
+	return c, err
+}
+
+func (d *dockerWrap) StopContainer(id string, timeout uint) (err error) {
+	ctx, cancel := context.WithTimeout(context.Background(), retryTimeout)
+	defer cancel()
+	err = d.retry(ctx, func() error {
+		err = d.docker.StopContainer(id, timeout)
+		return err
+	})
+	return filterNotRunning(filterNoSuchContainer(err))
+}
+
+func (d *dockerWrap) Stats(opts docker.StatsOptions) (err error) {
+	// we can't retry this one this way since the callee closes the
+	// stats chan, need a fancier retry mechanism where we can swap out
+	// channels, but stats isn't crucial so... be lazy for now
+	return d.docker.Stats(opts)
+
+	//err = d.retry(func() error {
+	//err = d.docker.Stats(opts)
+	//return err
+	//})
+	//return err
+}

api/runner/drivers/docker/docker_test.go

@@ -0,0 +1,121 @@
+// Copyright 2016 Iron.io
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package docker
+
+import (
+	"bytes"
+	"context"
+	"io"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/kumokit/functions/api/runner/common"
+	"github.com/kumokit/functions/api/runner/drivers"
+	"github.com/vrischmann/envconfig"
+)
+
+type taskDockerTest struct {
+	id     string
+	input  io.Reader
+	output io.Writer
+}
+
+func (f *taskDockerTest) Command() string { return "" }
+func (f *taskDockerTest) EnvVars() map[string]string {
+	return map[string]string{}
+}
+func (f *taskDockerTest) Labels() map[string]string          { return nil }
+func (f *taskDockerTest) Id() string                         { return f.id }
+func (f *taskDockerTest) Group() string                      { return "" }
+func (f *taskDockerTest) Image() string                      { return "iron/hello" }
+func (f *taskDockerTest) Timeout() time.Duration             { return 30 * time.Second }
+func (f *taskDockerTest) Logger() (stdout, stderr io.Writer) { return f.output, nil }
+func (f *taskDockerTest) WriteStat(drivers.Stat)             { /* TODO */ }
+func (f *taskDockerTest) Volumes() [][2]string               { return [][2]string{} }
+func (f *taskDockerTest) WorkDir() string                    { return "" }
+func (f *taskDockerTest) Close()                             {}
+func (f *taskDockerTest) Input() io.Reader                   { return f.input }
+
+func TestRunnerDocker(t *testing.T) {
+	env := common.NewEnvironment(func(e *common.Environment) {})
+	dkr := NewDocker(env, drivers.Config{})
+	ctx := context.Background()
+
+	task := &taskDockerTest{"test-docker", nil, nil}
+
+	cookie, err := dkr.Prepare(ctx, task)
+	if err != nil {
+		t.Fatal("Couldn't prepare task test")
+	}
+	defer cookie.Close()
+
+	result, err := cookie.Run(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if result.Status() != "success" {
+		t.Fatal("Test should successfully run the image")
+	}
+}
+
+func TestRunnerDockerStdin(t *testing.T) {
+	env := common.NewEnvironment(func(e *common.Environment) {})
+	dkr := NewDocker(env, drivers.Config{})
+	ctx := context.Background()
+
+	input := `{"name": "test"}`
+	var output bytes.Buffer
+
+	task := &taskDockerTest{"test-docker-stdin", bytes.NewBufferString(input), &output}
+
+	cookie, err := dkr.Prepare(ctx, task)
+	if err != nil {
+		t.Fatal("Couldn't prepare task test")
+	}
+	defer cookie.Close()
+
+	result, err := cookie.Run(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if result.Status() != "success" {
+		t.Error("Test should successfully run the image")
+	}
+
+	expect := "Hello test!"
+	got := output.String()
+	if !strings.Contains(got, expect) {
+		t.Errorf("Test expected output to contain '%s', got '%s'", expect, got)
+	}
+}
+
+func TestConfigLoadMemory(t *testing.T) {
+	if err := os.Setenv("MEMORY_PER_JOB", "128M"); err != nil {
+		t.Fatalf("Could not set MEMORY_PER_JOB: %v", err)
+	}
+
+	var conf drivers.Config
+	if err := envconfig.Init(&conf); err != nil {
+		t.Fatalf("Could not read config: %v", err)
+	}
+
+	if conf.Memory != 128*1024*1024 {
+		t.Fatalf("Memory read from config should match 128M, got %d", conf.Memory)
+	}
+}

api/runner/drivers/driver.go

@@ -0,0 +1,288 @@
+// Copyright 2016 Iron.io
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Interface for all container drivers
+
+package drivers
+
+import (
+	"context"
+	"errors"
+	"io"
+	"strings"
+	"time"
+
+	"code.cloudfoundry.org/bytefmt"
+)
+
+// A DriverCookie identifies a unique request to run a task.
+//
+// Clients should always call Close() on a DriverCookie after they are done
+// with it.
+type Cookie interface {
+	io.Closer
+
+	// Run should execute task on the implementation.
+	// RunResult captures the result of task execution. This means if task
+	// execution fails due to a problem in the task, Run() MUST return a valid
+	// RunResult and nil as the error. The RunResult's Error() and Status()
+	// should be used to indicate failure.
+	// If the implementation itself suffers problems (lost of network, out of
+	// disk etc.), a nil RunResult and an error message is preferred.
+	//
+	// Run() MUST monitor the context. task cancellation is indicated by
+	// cancelling the context.
+	Run(ctx context.Context) (RunResult, error)
+}
+
+type Driver interface {
+	// Prepare can be used in order to do any preparation that a specific driver
+	// may need to do before running the task, and can be useful to put
+	// preparation that the task can recover from into (i.e. if pulling an image
+	// fails because a registry is down, the task doesn't need to be failed).  It
+	// returns a cookie that can be used to execute the task.
+	// Callers should Close the cookie regardless of whether they run it.
+	//
+	// The returned cookie should respect the task's timeout when it is run.
+	Prepare(ctx context.Context, task ContainerTask) (Cookie, error)
+}
+
+// RunResult indicates only the final state of the task.
+type RunResult interface {
+	// Error is an actionable/checkable error from the container.
+	error
+
+	// Status should return the current status of the task.
+	// Only valid options are {"error", "success", "timeout", "killed", "cancelled"}.
+	Status() string
+}
+
+// The ContainerTask interface guides task execution across a wide variety of
+// container oriented runtimes.
+// This interface is unstable.
+//
+// FIXME: This interface is large, and it is currently a little Docker specific.
+type ContainerTask interface {
+	// Command returns the command to run within the container.
+	Command() string
+	// EnvVars returns environment variable key-value pairs.
+	EnvVars() map[string]string
+	// Input feeds the container with data
+	Input() io.Reader
+	// Labels returns container label key-value pairs.
+	Labels() map[string]string
+	Id() string
+	// Image returns the runtime specific image to run.
+	Image() string
+	// Timeout specifies the maximum time a task is allowed to run. Return 0 to let it run forever.
+	Timeout() time.Duration
+	// Driver will write output log from task execution to these writers. Must be
+	// non-nil. Use io.Discard if log is irrelevant.
+	Logger() (stdout, stderr io.Writer)
+	// WriteStat writes a single Stat, implementation need not be thread safe.
+	WriteStat(Stat)
+	// Volumes returns an array of 2-element tuples indicating storage volume mounts.
+	// The first element is the path on the host, and the second element is the
+	// path in the container.
+	Volumes() [][2]string
+	// WorkDir returns the working directory to use for the task. Empty string
+	// leaves it unset.
+	WorkDir() string
+
+	// Close is used to perform cleanup after task execution.
+	// Close should be safe to call multiple times.
+	Close()
+}
+
+// Stat is a bucket of stats from a driver at a point in time for a certain task.
+type Stat struct {
+	Timestamp time.Time
+	Metrics   map[string]uint64
+}
+
+// Set of acceptable errors coming from container engines to TaskRunner
+var (
+	// ErrOutOfMemory for OOM in container engine
+	ErrOutOfMemory = userError(errors.New("out of memory error"))
+)
+
+// TODO agent.UserError should be elsewhere
+func userError(err error) error { return &ue{err} }
+
+type ue struct {
+	error
+}
+
+func (u *ue) UserVisible() bool { return true }
+
+// TODO: ensure some type is applied to these statuses.
+const (
+	// task statuses
+	StatusRunning   = "running"
+	StatusSuccess   = "success"
+	StatusError     = "error"
+	StatusTimeout   = "timeout"
+	StatusKilled    = "killed"
+	StatusCancelled = "cancelled"
+)
+
+// Allows us to implement custom unmarshaling of JSON and envconfig.
+type Memory uint64
+
+func (m *Memory) Unmarshal(s string) error {
+	temp, err := bytefmt.ToBytes(s)
+	if err != nil {
+		return err
+	}
+
+	*m = Memory(temp)
+	return nil
+}
+
+func (m *Memory) UnmarshalJSON(p []byte) error {
+	temp, err := bytefmt.ToBytes(string(p))
+	if err != nil {
+		return err
+	}
+
+	*m = Memory(temp)
+	return nil
+}
+
+type Config struct {
+	Docker    string `json:"docker" envconfig:"default=unix:///var/run/docker.sock,DOCKER"`
+	Memory    Memory `json:"memory" envconfig:"default=256M,MEMORY_PER_JOB"`
+	CPUShares int64  `json:"cpu_shares" envconfig:"default=2,CPU_SHARES"`
+}
+
+// for tests
+func DefaultConfig() Config {
+	return Config{
+		Docker:    "unix:///var/run/docker.sock",
+		Memory:    256 * 1024 * 1024,
+		CPUShares: 0,
+	}
+}
+
+func average(samples []Stat) (Stat, bool) {
+	l := len(samples)
+	if l == 0 {
+		return Stat{}, false
+	} else if l == 1 {
+		return samples[0], true
+	}
+
+	s := Stat{
+		Metrics: samples[0].Metrics, // Recycle Metrics map from first sample
+	}
+	t := samples[0].Timestamp.UnixNano() / int64(l)
+	for _, sample := range samples[1:] {
+		t += sample.Timestamp.UnixNano() / int64(l)
+		for k, v := range sample.Metrics {
+			s.Metrics[k] += v
+		}
+	}
+
+	s.Timestamp = time.Unix(0, t)
+	for k, v := range s.Metrics {
+		s.Metrics[k] = v / uint64(l)
+	}
+	return s, true
+}
+
+// Decimate will down sample to a max number of points in a given sample by
+// averaging samples together. i.e. max=240, if we have 240 samples, return
+// them all, if we have 480 samples, every 2 samples average them (and time
+// distance), and return 240 samples. This is relatively naive and if len(in) >
+// max, <= max points will be returned, not necessarily max: length(out) =
+// ceil(length(in)/max) -- feel free to fix this, setting a relatively high max
+// will allow good enough granularity at higher lengths, i.e. for max of 1 hour
+// tasks, sampling every 1s, decimate will return 15s samples if max=240.
+// Large gaps in time between samples (a factor > (last-start)/max) will result
+// in a shorter list being returned to account for lost samples.
+// Decimate will modify the input list for efficiency, it is not copy safe.
+// Input must be sorted by timestamp or this will fail gloriously.
+func Decimate(maxSamples int, stats []Stat) []Stat {
+	if len(stats) <= maxSamples {
+		return stats
+	} else if maxSamples <= 0 { // protect from nefarious input
+		return nil
+	}
+
+	start := stats[0].Timestamp
+	window := stats[len(stats)-1].Timestamp.Sub(start) / time.Duration(maxSamples)
+
+	nextEntry, current := 0, start // nextEntry is the index tracking next Stats record location
+	for x := 0; x < len(stats); {
+		isLastEntry := nextEntry == maxSamples-1 // Last bin is larger than others to handle imprecision
+
+		var samples []Stat
+		for offset := 0; x+offset < len(stats); offset++ { // Iterate through samples until out of window
+			if !isLastEntry && stats[x+offset].Timestamp.After(current.Add(window)) {
+				break
+			}
+			samples = stats[x : x+offset+1]
+		}
+
+		x += len(samples)                      // Skip # of samples for next window
+		if entry, ok := average(samples); ok { // Only record Stat if 1+ samples exist
+			stats[nextEntry] = entry
+			nextEntry++
+		}
+
+		current = current.Add(window)
+	}
+	return stats[:nextEntry] // Return slice of []Stats that was modified with averages
+}
+
+// https://github.com/fsouza/go-dockerclient/blob/master/misc.go#L166
+func parseRepositoryTag(repoTag string) (repository string, tag string) {
+	parts := strings.SplitN(repoTag, "@", 2)
+	repoTag = parts[0]
+	n := strings.LastIndex(repoTag, ":")
+	if n < 0 {
+		return repoTag, ""
+	}
+	if tag := repoTag[n+1:]; !strings.Contains(tag, "/") {
+		return repoTag[:n], tag
+	}
+	return repoTag, ""
+}
+
+func ParseImage(image string) (registry, repo, tag string) {
+	repo, tag = parseRepositoryTag(image)
+	// Officially sanctioned at https://github.com/docker/docker/blob/master/registry/session.go#L319 to deal with "Official Repositories".
+	// Without this, token auth fails.
+	// Registries must exist at root (https://github.com/docker/docker/issues/7067#issuecomment-54302847)
+	// This cannot support the `library/` shortcut for private registries.
+	parts := strings.Split(repo, "/")
+	switch len(parts) {
+	case 1:
+		repo = "library/" + repo
+	case 2:
+		if strings.Contains(repo, ".") {
+			registry = parts[0]
+			repo = parts[1]
+		}
+	case 3:
+		registry = parts[0]
+		repo = parts[1] + "/" + parts[2]
+	}
+
+	if tag == "" {
+		tag = "latest"
+	}
+
+	return registry, repo, tag
+}

api/runner/drivers/driver_test.go

@@ -0,0 +1,128 @@
+// Copyright 2016 Iron.io
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package drivers
+
+import (
+	"testing"
+	"time"
+)
+
+func TestAverage(t *testing.T) {
+	start := time.Date(2016, 8, 11, 0, 0, 0, 0, time.UTC)
+	stats := make([]Stat, 10)
+	for i := 0; i < len(stats); i++ {
+		stats[i] = Stat{
+			Timestamp: start.Add(time.Duration(i) * time.Minute),
+			Metrics:   map[string]uint64{"x": uint64(i)},
+		}
+	}
+
+	res, ok := average(stats)
+	if !ok {
+		t.Error("Expected good record")
+	}
+
+	expectedV := uint64(4)
+	if v, ok := res.Metrics["x"]; !ok || v != expectedV {
+		t.Error("Actual average didn't match expected", "actual", v, "expected", expectedV)
+	}
+
+	expectedT := time.Unix(1470873870, 0)
+	if res.Timestamp != expectedT {
+		t.Error("Actual average didn't match expected", "actual", res.Timestamp, "expected", expectedT)
+	}
+}
+
+func TestDecimate(t *testing.T) {
+	start := time.Now()
+	stats := make([]Stat, 480)
+	for i := range stats {
+		stats[i] = Stat{
+			Timestamp: start.Add(time.Duration(i) * time.Second),
+			Metrics:   map[string]uint64{"x": uint64(i)},
+		}
+		//		t.Log(stats[i])
+	}
+
+	stats = Decimate(240, stats)
+	if len(stats) != 240 {
+		t.Error("decimate function bad", len(stats))
+	}
+
+	//for i := range stats {
+	//t.Log(stats[i])
+	//}
+
+	stats = make([]Stat, 700)
+	for i := range stats {
+		stats[i] = Stat{
+			Timestamp: start.Add(time.Duration(i) * time.Second),
+			Metrics:   map[string]uint64{"x": uint64(i)},
+		}
+	}
+	stats = Decimate(240, stats)
+	if len(stats) != 240 {
+		t.Error("decimate function bad", len(stats))
+	}
+
+	stats = make([]Stat, 300)
+	for i := range stats {
+		stats[i] = Stat{
+			Timestamp: start.Add(time.Duration(i) * time.Second),
+			Metrics:   map[string]uint64{"x": uint64(i)},
+		}
+	}
+	stats = Decimate(240, stats)
+	if len(stats) != 240 {
+		t.Error("decimate function bad", len(stats))
+	}
+
+	stats = make([]Stat, 300)
+	for i := range stats {
+		if i == 150 {
+			// leave 1 large gap
+			start = start.Add(20 * time.Minute)
+		}
+		stats[i] = Stat{
+			Timestamp: start.Add(time.Duration(i) * time.Second),
+			Metrics:   map[string]uint64{"x": uint64(i)},
+		}
+	}
+	stats = Decimate(240, stats)
+	if len(stats) != 49 {
+		t.Error("decimate function bad", len(stats))
+	}
+}
+
+func TestParseImage(t *testing.T) {
+	cases := map[string][]string{
+		"iron/hello":                                        {"", "iron/hello", "latest"},
+		"iron/hello:v1":                                     {"", "iron/hello", "v1"},
+		"my.registry/hello":                                 {"my.registry", "hello", "latest"},
+		"my.registry/hello:v1":                              {"my.registry", "hello", "v1"},
+		"mongo":                                             {"", "library/mongo", "latest"},
+		"mongo:v1":                                          {"", "library/mongo", "v1"},
+		"quay.com/iron/hello":                               {"quay.com", "iron/hello", "latest"},
+		"quay.com:8080/iron/hello:v2":                       {"quay.com:8080", "iron/hello", "v2"},
+		"localhost.localdomain:5000/samalba/hipache:latest": {"localhost.localdomain:5000", "samalba/hipache", "latest"},
+	}
+
+	for in, out := range cases {
+		reg, repo, tag := ParseImage(in)
+		if reg != out[0] || repo != out[1] || tag != out[2] {
+			t.Errorf("Test input %q wasn't parsed as expected. Expected %q, got %q", in, out, []string{reg, repo, tag})
+		}
+	}
+}

api/runner/drivers/mock/mocker.go

@@ -0,0 +1,60 @@
+// Copyright 2016 Iron.io
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package mock
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/kumokit/functions/api/runner/drivers"
+)
+
+func New() drivers.Driver {
+	return &Mocker{}
+}
+
+type Mocker struct {
+	count int
+}
+
+func (m *Mocker) Prepare(context.Context, drivers.ContainerTask) (drivers.Cookie, error) {
+	return &cookie{m}, nil
+}
+
+type cookie struct {
+	m *Mocker
+}
+
+func (c *cookie) Close() error { return nil }
+
+func (c *cookie) Run(ctx context.Context) (drivers.RunResult, error) {
+	c.m.count++
+	if c.m.count%100 == 0 {
+		return nil, fmt.Errorf("Mocker error! Bad.")
+	}
+	return &runResult{
+		error:       nil,
+		StatusValue: "success",
+	}, nil
+}
+
+type runResult struct {
+	error
+	StatusValue string
+}
+
+func (runResult *runResult) Status() string {
+	return runResult.StatusValue
+}

api/runner/func_logger.go

@@ -6,7 +6,7 @@ import (
 
 	"context"
 	"github.com/Sirupsen/logrus"
-	"github.com/kumokit/runner/common"
+	"github.com/kumokit/functions/api/runner/common"
 )
 
 type FuncLogger interface {

api/runner/metric_logger.go

@@ -5,7 +5,7 @@ import (
 	"time"
 
 	"github.com/Sirupsen/logrus"
-	"github.com/kumokit/runner/common"
+	"github.com/kumokit/functions/api/runner/common"
 )
 
 type MetricLogger interface {

api/runner/runner.go

@@ -14,12 +14,12 @@ import (
 	"time"
 
 	"github.com/Sirupsen/logrus"
+	"github.com/kumokit/functions/api/runner/common"
+	"github.com/kumokit/functions/api/runner/drivers"
+	driverscommon "github.com/kumokit/functions/api/runner/drivers"
+	"github.com/kumokit/functions/api/runner/drivers/docker"
+	"github.com/kumokit/functions/api/runner/drivers/mock"
 	"github.com/kumokit/functions/api/runner/task"
-	"github.com/kumokit/runner/common"
-	"github.com/kumokit/runner/drivers"
-	driverscommon "github.com/kumokit/runner/drivers"
-	"github.com/kumokit/runner/drivers/docker"
-	"github.com/kumokit/runner/drivers/mock"
 )
 
 type Runner struct {

api/runner/task.go

@@ -12,7 +12,7 @@ import (
 	"github.com/docker/docker/cli/config/configfile"
 	docker "github.com/fsouza/go-dockerclient"
 	"github.com/kumokit/functions/api/runner/task"
-	"github.com/kumokit/runner/drivers"
+	"github.com/kumokit/functions/api/runner/drivers"
 )
 
 var registries dockerRegistries

api/runner/task/task.go

@@ -5,7 +5,7 @@ import (
 	"io"
 	"time"
 
-	"github.com/kumokit/runner/drivers"
+	"github.com/kumokit/functions/api/runner/drivers"
 )
 
 type Config struct {

api/runner/worker.go

@@ -11,7 +11,7 @@ import (
 	"github.com/Sirupsen/logrus"
 	"github.com/kumokit/functions/api/runner/protocol"
 	"github.com/kumokit/functions/api/runner/task"
-	"github.com/kumokit/runner/drivers"
+	"github.com/kumokit/functions/api/runner/drivers"
 )
 
 // hot functions - theory of operation

api/server/apps_create.go

@@ -6,7 +6,7 @@ import (
 
 	"github.com/gin-gonic/gin"
 	"github.com/kumokit/functions/api/models"
-	"github.com/kumokit/runner/common"
+	"github.com/kumokit/functions/api/runner/common"
 )
 
 func (s *Server) handleAppCreate(c *gin.Context) {

api/server/apps_delete.go

@@ -7,7 +7,7 @@ import (
 	"github.com/gin-gonic/gin"
 	"github.com/kumokit/functions/api"
 	"github.com/kumokit/functions/api/models"
-	"github.com/kumokit/runner/common"
+	"github.com/kumokit/functions/api/runner/common"
 )
 
 func (s *Server) handleAppDelete(c *gin.Context) {

api/server/apps_update.go

@@ -7,7 +7,7 @@ import (
 	"github.com/gin-gonic/gin"
 	"github.com/kumokit/functions/api"
 	"github.com/kumokit/functions/api/models"
-	"github.com/kumokit/runner/common"
+	"github.com/kumokit/functions/api/runner/common"
 )
 
 func (s *Server) handleAppUpdate(c *gin.Context) {

api/server/error_response.go

@@ -5,7 +5,7 @@ import (
 	"errors"
 	"github.com/gin-gonic/gin"
 	"github.com/kumokit/functions/api/models"
-	"github.com/kumokit/runner/common"
+	"github.com/kumokit/functions/api/runner/common"
 	"net/http"
 )
 

api/server/routes_create.go

@@ -7,7 +7,7 @@ import (
 	"github.com/gin-gonic/gin"
 	"github.com/kumokit/functions/api"
 	"github.com/kumokit/functions/api/models"
-	"github.com/kumokit/runner/common"
+	"github.com/kumokit/functions/api/runner/common"
 )
 
 func (s *Server) handleRouteCreate(c *gin.Context) {

api/server/routes_update.go

@@ -8,7 +8,7 @@ import (
 	"github.com/gin-gonic/gin"
 	"github.com/kumokit/functions/api"
 	"github.com/kumokit/functions/api/models"
-	"github.com/kumokit/runner/common"
+	"github.com/kumokit/functions/api/runner/common"
 )
 
 func (s *Server) handleRouteUpdate(c *gin.Context) {

api/server/runner.go

@@ -18,7 +18,7 @@ import (
 	"github.com/kumokit/functions/api/models"
 	"github.com/kumokit/functions/api/runner"
 	"github.com/kumokit/functions/api/runner/task"
-	"github.com/kumokit/runner/common"
+	"github.com/kumokit/functions/api/runner/common"
 	uuid "github.com/satori/go.uuid"
 )
 

api/server/server.go

@@ -21,7 +21,7 @@ import (
 	"github.com/kumokit/functions/api/runner"
 	"github.com/kumokit/functions/api/runner/task"
 	"github.com/kumokit/functions/api/server/internal/routecache"
-	"github.com/kumokit/runner/common"
+	"github.com/kumokit/functions/api/runner/common"
 	"github.com/spf13/viper"
 )