Moved runner into this repo, update dep files and now builds.

api/runner/common/backoff.go

@@ -0,0 +1,85 @@
+// Copyright 2016 Iron.io
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package common
+
+import (
+	"math"
+	"math/rand"
+	"sync"
+	"time"
+)
+
+type BoxTime struct{}
+
+func (BoxTime) Now() time.Time                         { return time.Now() }
+func (BoxTime) Sleep(d time.Duration)                  { time.Sleep(d) }
+func (BoxTime) After(d time.Duration) <-chan time.Time { return time.After(d) }
+
+type Backoff int
+
+func (b *Backoff) Sleep() { b.RandomSleep(nil, nil) }
+
+func (b *Backoff) RandomSleep(rng *rand.Rand, clock Clock) {
+	const (
+		maxexp   = 7
+		interval = 25 * time.Millisecond
+	)
+
+	if rng == nil {
+		rng = defaultRNG
+	}
+	if clock == nil {
+		clock = defaultClock
+	}
+
+	// 25-50ms, 50-100ms, 100-200ms, 200-400ms, 400-800ms, 800-1600ms, 1600-3200ms, 3200-6400ms
+	d := time.Duration(math.Pow(2, float64(*b))) * interval
+	d += (d * time.Duration(rng.Float64()))
+
+	clock.Sleep(d)
+
+	if *b < maxexp {
+		(*b)++
+	}
+}
+
+var (
+	defaultRNG   = NewRNG(time.Now().UnixNano())
+	defaultClock = BoxTime{}
+)
+
+func NewRNG(seed int64) *rand.Rand {
+	return rand.New(&lockedSource{src: rand.NewSource(seed)})
+}
+
+// taken from go1.5.1 math/rand/rand.go +233-250
+// bla bla if it puts a hole in the earth don't sue them
+type lockedSource struct {
+	lk  sync.Mutex
+	src rand.Source
+}
+
+func (r *lockedSource) Int63() (n int64) {
+	r.lk.Lock()
+	n = r.src.Int63()
+	r.lk.Unlock()
+	return
+}
+
+func (r *lockedSource) Seed(seed int64) {
+	r.lk.Lock()
+	r.src.Seed(seed)
+	r.lk.Unlock()
+}

api/runner/common/clock.go

@@ -0,0 +1,23 @@
+// Copyright 2016 Iron.io
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package common
+
+import "time"
+
+type Clock interface {
+	Now() time.Time
+	Sleep(time.Duration)
+	After(time.Duration) <-chan time.Time
+}

api/runner/common/ctx.go

@@ -0,0 +1,44 @@
+// Copyright 2016 Iron.io
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package common
+
+import (
+	"context"
+
+	"github.com/Sirupsen/logrus"
+)
+
+// WithLogger stores the logger.
+func WithLogger(ctx context.Context, l logrus.FieldLogger) context.Context {
+	return context.WithValue(ctx, "logger", l)
+}
+
+// Logger returns the structured logger.
+func Logger(ctx context.Context) logrus.FieldLogger {
+	l, ok := ctx.Value("logger").(logrus.FieldLogger)
+	if !ok {
+		return logrus.StandardLogger()
+	}
+	return l
+}
+
+// Attempt at simplifying this whole logger in the context thing
+// Could even make this take a generic map, then the logger that gets returned could be used just like the stdlib too, since it's compatible
+func LoggerWithFields(ctx context.Context, fields logrus.Fields) (context.Context, logrus.FieldLogger) {
+	l := Logger(ctx)
+	l = l.WithFields(fields)
+	ctx = WithLogger(ctx, l)
+	return ctx, l
+}

api/runner/common/environment.go

@@ -0,0 +1,38 @@
+// Copyright 2016 Iron.io
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package common
+
+import (
+	"github.com/kumokit/functions/api/runner/common/stats"
+)
+
+// An Environment is a long lived object that carries around 'configuration'
+// for the program. Other long-lived objects may embed an environment directly
+// into their definition. Environments wrap common functionality like logging
+// and metrics. For short-lived request-response like tasks use `Context`,
+// which wraps an Environment.
+
+type Environment struct {
+	stats.Statter
+}
+
+// Initializers are functions that may set up the environment as they like. By default the environment is 'inactive' in the sense that metrics aren't reported.
+func NewEnvironment(initializers ...func(e *Environment)) *Environment {
+	env := &Environment{&stats.NilStatter{}}
+	for _, init := range initializers {
+		init(env)
+	}
+	return env
+}

api/runner/common/errors.go

@@ -0,0 +1,70 @@
+// Copyright 2016 Iron.io
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package common
+
+import (
+	"io"
+	"net"
+	"syscall"
+)
+
+// Errors that can be directly exposed to task creators/users.
+type UserVisibleError interface {
+	UserVisible() bool
+}
+
+func IsUserVisibleError(err error) bool {
+	ue, ok := err.(UserVisibleError)
+	return ok && ue.UserVisible()
+}
+
+type userVisibleError struct {
+	error
+}
+
+func (u *userVisibleError) UserVisible() bool { return true }
+
+func UserError(err error) error {
+	return &userVisibleError{err}
+}
+
+type Temporary interface {
+	Temporary() bool
+}
+
+func IsTemporary(err error) bool {
+	v, ok := err.(Temporary)
+	return (ok && v.Temporary()) || isNet(err)
+}
+
+func isNet(err error) bool {
+	if _, ok := err.(net.Error); ok {
+		return true
+	}
+
+	switch err := err.(type) {
+	case *net.OpError:
+		return true
+	case syscall.Errno:
+		if err == syscall.ECONNREFUSED { // linux only? maybe ok for prod
+			return true // connection refused
+		}
+	default:
+		if err == io.ErrUnexpectedEOF || err == io.EOF {
+			return true
+		}
+	}
+	return false
+}

api/runner/common/logging.go

@@ -0,0 +1,83 @@
+// Copyright 2016 Iron.io
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package common
+
+import (
+	"net/url"
+	"os"
+
+	"github.com/Sirupsen/logrus"
+)
+
+func SetLogLevel(ll string) {
+	if ll == "" {
+		ll = "info"
+	}
+	logrus.WithFields(logrus.Fields{"level": ll}).Info("Setting log level to")
+	logLevel, err := logrus.ParseLevel(ll)
+	if err != nil {
+		logrus.WithFields(logrus.Fields{"level": ll}).Warn("Could not parse log level, setting to INFO")
+		logLevel = logrus.InfoLevel
+	}
+	logrus.SetLevel(logLevel)
+}
+
+func SetLogDest(to, prefix string) {
+	logrus.SetOutput(os.Stderr) // in case logrus changes their mind...
+	if to == "stderr" {
+		return
+	}
+
+	// possible schemes: { udp, tcp, file }
+	// file url must contain only a path, syslog must contain only a host[:port]
+	// expect: [scheme://][host][:port][/path]
+	// default scheme to udp:// if none given
+
+	url, err := url.Parse(to)
+	if url.Host == "" && url.Path == "" {
+		logrus.WithFields(logrus.Fields{"to": to}).Warn("No scheme on logging url, adding udp://")
+		// this happens when no scheme like udp:// is present
+		to = "udp://" + to
+		url, err = url.Parse(to)
+	}
+	if err != nil {
+		logrus.WithError(err).WithFields(logrus.Fields{"to": to}).Error("could not parse logging URI, defaulting to stderr")
+		return
+	}
+
+	// File URL must contain only `url.Path`. Syslog location must contain only `url.Host`
+	if (url.Host == "" && url.Path == "") || (url.Host != "" && url.Path != "") {
+		logrus.WithFields(logrus.Fields{"to": to, "uri": url}).Error("invalid logging location, defaulting to stderr")
+		return
+	}
+
+	switch url.Scheme {
+	case "udp", "tcp":
+		err = NewSyslogHook(url, prefix)
+		if err != nil {
+			logrus.WithFields(logrus.Fields{"uri": url, "to": to}).WithError(err).Error("unable to connect to syslog, defaulting to stderr")
+			return
+		}
+	case "file":
+		f, err := os.OpenFile(url.Path, os.O_RDWR|os.O_APPEND|os.O_CREATE, 0666)
+		if err != nil {
+			logrus.WithError(err).WithFields(logrus.Fields{"to": to, "path": url.Path}).Error("cannot open file, defaulting to stderr")
+			return
+		}
+		logrus.SetOutput(f)
+	default:
+		logrus.WithFields(logrus.Fields{"scheme": url.Scheme, "to": to}).Error("unknown logging location scheme, defaulting to stderr")
+	}
+}

api/runner/common/stats/aggregator.go

@@ -0,0 +1,188 @@
+// Copyright 2016 Iron.io
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package stats
+
+import (
+	"sync"
+	"time"
+)
+
+type reporter interface {
+	report([]*collectedStat)
+}
+
+type collectedStat struct {
+	Name     string
+	Counters map[string]int64
+	Values   map[string]float64
+	Gauges   map[string]int64
+	Timers   map[string]time.Duration
+
+	avgCounts map[string]uint64
+}
+
+func newCollectedStatUnescaped(name string) *collectedStat {
+	return &collectedStat{
+		Name:      name,
+		Counters:  map[string]int64{},
+		Values:    map[string]float64{},
+		Gauges:    map[string]int64{},
+		Timers:    map[string]time.Duration{},
+		avgCounts: map[string]uint64{},
+	}
+}
+
+// What do you call an alligator in a vest?
+
+// Aggregator collects a stats and merges them together if they've been added
+// previously. Useful for reporters that have low throughput ie stathat.
+type Aggregator struct {
+	// Holds all of our stats based on stat.Name
+	sl    sync.RWMutex
+	stats map[string]*statHolder
+
+	reporters []reporter
+}
+
+func newAggregator(reporters []reporter) *Aggregator {
+	return &Aggregator{
+		stats:     make(map[string]*statHolder),
+		reporters: reporters,
+	}
+}
+
+type statHolder struct {
+	cl sync.RWMutex // Lock on Counters
+	vl sync.RWMutex // Lock on Values
+	s  *collectedStat
+}
+
+func newStatHolder(st *collectedStat) *statHolder {
+	return &statHolder{s: st}
+}
+
+type kind int16
+
+const (
+	counterKind kind = iota
+	valueKind
+	gaugeKind
+	durationKind
+)
+
+func (a *Aggregator) add(component, key string, kind kind, value interface{}) {
+	a.sl.RLock()
+	stat, ok := a.stats[component]
+	a.sl.RUnlock()
+	if !ok {
+		a.sl.Lock()
+		stat, ok = a.stats[component]
+		if !ok {
+			stat = newStatHolder(newCollectedStatUnescaped(component))
+			a.stats[component] = stat
+		}
+		a.sl.Unlock()
+	}
+
+	if kind == counterKind || kind == gaugeKind {
+		var mapPtr map[string]int64
+		if kind == counterKind {
+			mapPtr = stat.s.Counters
+		} else {
+			mapPtr = stat.s.Gauges
+		}
+		value := value.(int64)
+		stat.cl.Lock()
+		mapPtr[key] += value
+		stat.cl.Unlock()
+	}
+
+	/* TODO: this ends up ignoring tags so yeah gg
+	/  lets just calculate a running average for now. Can do percentiles later
+	/  Recalculated Average
+	/
+	/     currentAverage * currentCount + newValue
+	/    ------------------------------------------
+	/                (currentCount +1)
+	/
+	*/
+	if kind == valueKind || kind == durationKind {
+		var typedValue int64
+		if kind == valueKind {
+			typedValue = value.(int64)
+		} else {
+			typedValue = int64(value.(time.Duration))
+		}
+
+		stat.vl.Lock()
+		switch kind {
+		case valueKind:
+			oldAverage := stat.s.Values[key]
+			count := stat.s.avgCounts[key]
+			newAverage := (oldAverage*float64(count) + float64(typedValue)) / (float64(count + 1))
+			stat.s.avgCounts[key] = count + 1
+			stat.s.Values[key] = newAverage
+		case durationKind:
+			oldAverage := float64(stat.s.Timers[key])
+			count := stat.s.avgCounts[key]
+			newAverage := (oldAverage*float64(count) + float64(typedValue)) / (float64(count + 1))
+			stat.s.avgCounts[key] = count + 1
+			stat.s.Timers[key] = time.Duration(newAverage)
+		}
+		stat.vl.Unlock()
+	}
+}
+
+func (a *Aggregator) dump() []*collectedStat {
+	a.sl.Lock()
+	bucket := a.stats
+	// Clear out the maps, effectively resetting our average
+	a.stats = make(map[string]*statHolder)
+	a.sl.Unlock()
+
+	stats := make([]*collectedStat, 0, len(bucket))
+	for _, v := range bucket {
+		stats = append(stats, v.s)
+	}
+	return stats
+}
+
+func (a *Aggregator) report(st []*collectedStat) {
+	stats := a.dump()
+	stats = append(stats, st...)
+	for _, r := range a.reporters {
+		r.report(stats)
+	}
+}
+
+func (r *Aggregator) Inc(component string, stat string, value int64, rate float32) {
+	r.add(component, stat, counterKind, value)
+}
+
+func (r *Aggregator) Gauge(component string, stat string, value int64, rate float32) {
+	r.add(component, stat, gaugeKind, value)
+}
+
+func (r *Aggregator) Measure(component string, stat string, value int64, rate float32) {
+	r.add(component, stat, valueKind, value)
+}
+
+func (r *Aggregator) Time(component string, stat string, value time.Duration, rate float32) {
+	r.add(component, stat, durationKind, value)
+}
+
+func (r *Aggregator) NewTimer(component string, stat string, rate float32) *Timer {
+	return newTimer(r, component, stat, rate)
+}

api/runner/common/stats/aggregator_test.go

@@ -0,0 +1,95 @@
+// Copyright 2016 Iron.io
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package stats
+
+import (
+	"fmt"
+	"math/rand"
+	"testing"
+)
+
+func TestAggregator(t *testing.T) {
+	ag := newAggregator([]reporter{})
+	var sum int64 = 0
+	var times int64 = 0
+	for i := 0; i < 100; i++ {
+		ag.add("mq push", "messages", counterKind, int64(1))
+		ag.add("mq push", "latency", valueKind, int64(i))
+		ag.add("mq pull", "latency", valueKind, int64(i))
+		sum += int64(i)
+		times += 1
+	}
+
+	for _, stat := range ag.dump() {
+		for k, v := range stat.Values {
+			if v != float64(sum)/float64(times) {
+				t.Error("key:", k, "Expected", sum/times, "got", v)
+			}
+		}
+
+		for k, v := range stat.Counters {
+			if v != times {
+				t.Error("key:", k, "Expected", times, "got", v)
+			}
+		}
+	}
+	if len(ag.stats) != 0 {
+		t.Error("expected stats map to be clear, got", len(ag.stats))
+	}
+}
+
+type testStat struct {
+	component string
+	key       string
+	kind      kind
+	value     int64
+}
+
+func BenchmarkAggregatorAdd(b *testing.B) {
+	ag := &Aggregator{
+		stats: make(map[string]*statHolder, 1000),
+	}
+
+	s := createStatList(1000)
+
+	sl := len(s)
+	b.ResetTimer()
+	b.RunParallel(func(pb *testing.PB) {
+		for pb.Next() {
+			e := s[rand.Intn(sl)]
+			ag.add(e.component, e.key, e.kind, e.value)
+		}
+	})
+}
+
+func createStatList(n int) []*testStat {
+	var stats []*testStat
+	for i := 0; i < n; i++ {
+		st := testStat{
+			component: "aggregator_test",
+			key:       fmt.Sprintf("latency.%d", i),
+			kind:      counterKind,
+			value:     1,
+		}
+
+		if rand.Float32() < 0.5 {
+			st.key = fmt.Sprintf("test.%d", i)
+			st.kind = valueKind
+			st.value = 15999
+		}
+		stats = append(stats, &st)
+	}
+	return stats
+}

api/runner/common/stats/log.go

@@ -0,0 +1,45 @@
+// Copyright 2016 Iron.io
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package stats
+
+import (
+	"time"
+
+	"github.com/Sirupsen/logrus"
+)
+
+type LogReporter struct {
+}
+
+func NewLogReporter() *LogReporter {
+	return (&LogReporter{})
+}
+
+func (lr *LogReporter) report(stats []*collectedStat) {
+	for _, s := range stats {
+		f := make(logrus.Fields)
+		for k, v := range s.Counters {
+			f[k] = v
+		}
+		for k, v := range s.Values {
+			f[k] = v
+		}
+		for k, v := range s.Timers {
+			f[k] = time.Duration(v)
+		}
+
+		logrus.WithFields(f).Info(s.Name)
+	}
+}

api/runner/common/stats/mem.go

@@ -0,0 +1,40 @@
+// Copyright 2016 Iron.io
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package stats
+
+import (
+	"runtime"
+	"time"
+)
+
+func StartReportingMemoryAndGC(reporter Statter, d time.Duration) {
+	ticker := time.Tick(d)
+	for {
+		select {
+		case <-ticker:
+			var ms runtime.MemStats
+			runtime.ReadMemStats(&ms)
+
+			prefix := "runtime"
+
+			reporter.Measure(prefix, "allocated", int64(ms.Alloc), 1.0)
+			reporter.Measure(prefix, "allocated.heap", int64(ms.HeapAlloc), 1.0)
+			reporter.Time(prefix, "gc.pause", time.Duration(ms.PauseNs[(ms.NumGC+255)%256]), 1.0)
+
+			// GC CPU percentage.
+			reporter.Measure(prefix, "gc.cpufraction", int64(ms.GCCPUFraction*100), 1.0)
+		}
+	}
+}

api/runner/common/stats/newrelic.go

@@ -0,0 +1,142 @@
+// Copyright 2016 Iron.io
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package stats
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"strings"
+
+	"github.com/Sirupsen/logrus"
+)
+
+type NewRelicAgentConfig struct {
+	Host    string `json:"host"`
+	Version string `json:"version"`
+	Pid     int    `json:"pid"`
+}
+
+// examples: https://docs.newrelic.com/docs/plugins/plugin-developer-resources/developer-reference/metric-data-plugin-api#examples
+type newRelicRequest struct {
+	Agent      *agent       `json:"agent"`
+	Components []*component `json:"components"`
+}
+
+type NewRelicReporterConfig struct {
+	Agent      *NewRelicAgentConfig
+	LicenseKey string `json:"license_key"`
+}
+
+type NewRelicReporter struct {
+	Agent      *agent
+	LicenseKey string
+}
+
+func NewNewRelicReporter(version string, licenseKey string) *NewRelicReporter {
+	r := &NewRelicReporter{}
+	r.Agent = newNewRelicAgent(version)
+	r.LicenseKey = licenseKey
+	return r
+}
+
+func (r *NewRelicReporter) report(stats []*collectedStat) {
+	client := &http.Client{}
+	req := &newRelicRequest{}
+	req.Agent = r.Agent
+	comp := newComponent()
+	comp.Name = "IronMQ"
+	comp.Duration = 60
+	comp.GUID = "io.iron.ironmq"
+	// TODO - NR has a fixed 3 level heirarchy? and we just use 2?
+	req.Components = []*component{comp}
+
+	// now add metrics
+	for _, s := range stats {
+		for k, v := range s.Counters {
+			comp.Metrics[fmt.Sprintf("Component/%s %s", s.Name, k)] = v
+		}
+		for k, v := range s.Values {
+			comp.Metrics[fmt.Sprintf("Component/%s %s", s.Name, k)] = int64(v)
+		}
+		for k, v := range s.Timers {
+			comp.Metrics[fmt.Sprintf("Component/%s %s", s.Name, k)] = int64(v)
+		}
+	}
+
+	metricsJson, err := json.Marshal(req)
+	if err != nil {
+		logrus.WithError(err).Error("error encoding json for NewRelicReporter")
+	}
+
+	jsonAsString := string(metricsJson)
+
+	httpRequest, err := http.NewRequest("POST",
+		"https://platform-api.newrelic.com/platform/v1/metrics",
+		strings.NewReader(jsonAsString))
+	if err != nil {
+		logrus.WithError(err).Error("error creating New Relic request")
+		return
+	}
+	httpRequest.Header.Set("X-License-Key", r.LicenseKey)
+	httpRequest.Header.Set("Content-Type", "application/json")
+	httpRequest.Header.Set("Accept", "application/json")
+	httpResponse, err := client.Do(httpRequest)
+	if err != nil {
+		logrus.WithError(err).Error("error sending http request in NewRelicReporter")
+		return
+	}
+	defer httpResponse.Body.Close()
+	body, err := ioutil.ReadAll(httpResponse.Body)
+	if err != nil {
+		logrus.WithError(err).Error("error reading response body")
+	} else {
+		logrus.Debugln("response", "code", httpResponse.Status, "body", string(body))
+	}
+}
+
+type agent struct {
+	Host    string `json:"host"`
+	Version string `json:"version"`
+	Pid     int    `json:"pid"`
+}
+
+func newNewRelicAgent(Version string) *agent {
+	var err error
+	agent := &agent{
+		Version: Version,
+	}
+	agent.Pid = os.Getpid()
+	if agent.Host, err = os.Hostname(); err != nil {
+		logrus.WithError(err).Error("Can not get hostname")
+		return nil
+	}
+	return agent
+}
+
+type component struct {
+	Name     string           `json:"name"`
+	GUID     string           `json:"guid"`
+	Duration int              `json:"duration"`
+	Metrics  map[string]int64 `json:"metrics"`
+}
+
+func newComponent() *component {
+	c := &component{}
+	c.Metrics = make(map[string]int64)
+	return c
+}

api/runner/common/stats/riemann.go

@@ -0,0 +1,117 @@
+// +build riemann
+
+// Copyright 2016 Iron.io
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package stats
+
+import (
+	"os"
+	"time"
+
+	"github.com/Sirupsen/logrus"
+	"github.com/amir/raidman"
+)
+
+type RiemannClient struct {
+	client     *raidman.Client
+	attributes map[string]string
+}
+
+const (
+	StateNormal = "normal"
+)
+
+func (rc *RiemannClient) Report([]*Stat) {}
+
+func (rc *RiemannClient) Add(s *Stat) {
+	var events []*raidman.Event
+
+	t := time.Now().UnixNano()
+
+	for k, v := range rc.attributes {
+		s.Tags[k] = v
+	}
+
+	for k, v := range s.Counters {
+		events = append(events, &raidman.Event{
+			Ttl:        5.0,
+			Time:       t,
+			State:      StateNormal,
+			Service:    s.Name + " " + k,
+			Metric:     v,
+			Attributes: s.Tags,
+		})
+	}
+
+	for k, v := range s.Values {
+		events = append(events, &raidman.Event{
+			Ttl:        5.0,
+			Time:       t,
+			State:      StateNormal,
+			Service:    s.Name + " " + k,
+			Metric:     v,
+			Attributes: s.Tags,
+		})
+	}
+
+	rc.report(events)
+}
+
+func (rc *RiemannClient) report(events []*raidman.Event) {
+	err := rc.client.SendMulti(events)
+	if err != nil {
+		logrus.WithError(err).Error("error sending to Riemann")
+	}
+}
+
+func (rc *RiemannClient) heartbeat() {
+	events := []*raidman.Event{
+		&raidman.Event{
+			Ttl:        5.0,
+			Time:       time.Now().UnixNano(),
+			State:      StateNormal,
+			Service:    "heartbeat",
+			Metric:     1.0,
+			Attributes: rc.attributes,
+		},
+	}
+	rc.report(events)
+}
+
+func newRiemann(config Config) *RiemannClient {
+	c, err := raidman.Dial("tcp", config.Riemann.RiemannHost)
+	if err != nil {
+		logrus.WithError(err).Error("error dialing Riemann")
+		os.Exit(1)
+	}
+
+	client := &RiemannClient{
+		client:     c,
+		attributes: map[string]string{},
+	}
+
+	for k, v := range config.Tags {
+		client.attributes[k] = v
+	}
+
+	// Send out a heartbeat every second
+	go func(rc *RiemannClient) {
+		for _ = range time.Tick(1 * time.Second) {
+			rc.heartbeat()
+		}
+	}(client)
+
+	return client
+}

api/runner/common/stats/stathat.go

@@ -0,0 +1,65 @@
+// Copyright 2016 Iron.io
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package stats
+
+import (
+	"net/http"
+	"net/url"
+	"strconv"
+
+	"github.com/Sirupsen/logrus"
+)
+
+func postStatHat(key, stat string, values url.Values) {
+	values.Set("stat", stat)
+	values.Set("ezkey", key)
+	resp, err := http.PostForm("http://api.stathat.com/ez", values)
+	if err != nil {
+		logrus.WithError(err).Error("couldn't post to StatHat")
+		return
+	}
+	if resp.StatusCode != 200 {
+		logrus.Errorln("bad status posting to StatHat", "status_code", resp.StatusCode)
+	}
+	resp.Body.Close()
+}
+
+type StatHatReporterConfig struct {
+	Email  string
+	Prefix string
+}
+
+func (shr *StatHatReporterConfig) report(stats []*collectedStat) {
+	for _, s := range stats {
+		for k, v := range s.Counters {
+			n := shr.Prefix + " " + s.Name + " " + k
+			values := url.Values{}
+			values.Set("count", strconv.FormatInt(v, 10))
+			postStatHat(shr.Email, n, values)
+		}
+		for k, v := range s.Values {
+			n := shr.Prefix + " " + s.Name + " " + k
+			values := url.Values{}
+			values.Set("value", strconv.FormatFloat(v, 'f', 3, 64))
+			postStatHat(shr.Email, n, values)
+		}
+		for k, v := range s.Timers {
+			n := shr.Prefix + " " + s.Name + " " + k
+			values := url.Values{}
+			values.Set("value", strconv.FormatInt(int64(v), 10))
+			postStatHat(shr.Email, n, values)
+		}
+	}
+}

api/runner/common/stats/stats.go

@@ -0,0 +1,187 @@
+// Copyright 2016 Iron.io
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package stats
+
+import (
+	"encoding/json"
+	"errors"
+	"net/http"
+	"strings"
+	"time"
+
+	"github.com/Sirupsen/logrus"
+)
+
+type HTTPSubHandler interface {
+	HTTPHandler(relativeUrl []string, w http.ResponseWriter, r *http.Request)
+}
+
+type Config struct {
+	Interval float64 `json:"interval" envconfig:"STATS_INTERVAL"` // seconds
+	History  int     // minutes
+
+	Log      string `json:"log" envconfig:"STATS_LOG"`
+	StatHat  *StatHatReporterConfig
+	NewRelic *NewRelicReporterConfig
+	Statsd   *StatsdConfig
+	GCStats  int `json:"gc_stats" envconfig:"GC_STATS"` // seconds
+}
+
+type Statter interface {
+	Inc(component string, stat string, value int64, rate float32)
+	Gauge(component string, stat string, value int64, rate float32)
+	Measure(component string, stat string, value int64, rate float32)
+	Time(component string, stat string, value time.Duration, rate float32)
+	NewTimer(component string, stat string, rate float32) *Timer
+}
+
+type MultiStatter struct {
+	statters []Statter
+}
+
+func (s *MultiStatter) Inc(component string, stat string, value int64, rate float32) {
+	for _, st := range s.statters {
+		st.Inc(component, stat, value, rate)
+	}
+}
+
+func (s *MultiStatter) Gauge(component string, stat string, value int64, rate float32) {
+	for _, st := range s.statters {
+		st.Gauge(component, stat, value, rate)
+	}
+}
+
+func (s *MultiStatter) Measure(component string, stat string, value int64, rate float32) {
+	for _, st := range s.statters {
+		st.Measure(component, stat, value, rate)
+	}
+}
+
+func (s *MultiStatter) Time(component string, stat string, value time.Duration, rate float32) {
+	for _, st := range s.statters {
+		st.Time(component, stat, value, rate)
+	}
+}
+
+func (s *MultiStatter) NewTimer(component string, stat string, rate float32) *Timer {
+	return newTimer(s, component, stat, rate)
+}
+
+var badDecode error = errors.New("bad stats decode")
+
+func New(config Config) Statter {
+	s := new(MultiStatter)
+
+	if config.Interval == 0.0 {
+		config.Interval = 10.0 // convenience
+	}
+
+	var reporters []reporter
+	if config.StatHat != nil && config.StatHat.Email != "" {
+		reporters = append(reporters, config.StatHat)
+	}
+
+	if config.NewRelic != nil && config.NewRelic.LicenseKey != "" {
+		// NR wants version?
+		// can get it out of the namespace? roll it here?
+		reporters = append(reporters, NewNewRelicReporter("1.0", config.NewRelic.LicenseKey))
+	}
+
+	if config.Log != "" {
+		reporters = append(reporters, NewLogReporter())
+	}
+
+	if len(reporters) > 0 {
+		ag := newAggregator(reporters)
+		s.statters = append(s.statters, ag)
+		go func() {
+			for range time.Tick(time.Duration(config.Interval * float64(time.Second))) {
+				ag.report(nil)
+			}
+		}()
+	}
+
+	if config.Statsd != nil && config.Statsd.StatsdUdpTarget != "" {
+		std, err := NewStatsd(config.Statsd)
+		if err == nil {
+			s.statters = append(s.statters, std)
+		} else {
+			logrus.WithError(err).Error("Couldn't create statsd reporter")
+		}
+	}
+
+	if len(reporters) == 0 && config.Statsd == nil && config.History == 0 {
+		return &NilStatter{}
+	}
+
+	if config.GCStats >= 0 {
+		if config.GCStats == 0 {
+			config.GCStats = 1
+		}
+		go StartReportingMemoryAndGC(s, time.Duration(config.GCStats)*time.Second)
+	}
+
+	return s
+}
+
+func HTTPReturnJson(w http.ResponseWriter, result interface{}) {
+	w.Header().Set("Content-Type", "application/json")
+	res, err := json.Marshal(result)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+	} else {
+		w.Write(res)
+	}
+}
+
+// Convert a string to a stat name by replacing '.' with '_', lowercasing the
+// string and trimming it. Doesn't do any validation, so do try this out
+// locally before sending stats.
+func AsStatField(input string) string {
+	return strings.Replace(strings.ToLower(strings.TrimSpace(input)), ".", "_", -1)
+}
+
+// statsd like API on top of the map manipulation API.
+type Timer struct {
+	statter   Statter
+	component string
+	stat      string
+	start     time.Time
+	rate      float32
+	measured  bool
+}
+
+func newTimer(st Statter, component, stat string, rate float32) *Timer {
+	return &Timer{st, component, stat, time.Now(), rate, false}
+}
+
+func (timer *Timer) Measure() {
+	if timer.measured {
+		return
+	}
+
+	timer.measured = true
+	timer.statter.Time(timer.component, timer.stat, time.Since(timer.start), timer.rate)
+}
+
+type NilStatter struct{}
+
+func (n *NilStatter) Inc(component string, stat string, value int64, rate float32)          {}
+func (n *NilStatter) Gauge(component string, stat string, value int64, rate float32)        {}
+func (n *NilStatter) Measure(component string, stat string, value int64, rate float32)      {}
+func (n *NilStatter) Time(component string, stat string, value time.Duration, rate float32) {}
+func (r *NilStatter) NewTimer(component string, stat string, rate float32) *Timer {
+	return newTimer(r, component, stat, rate)
+}

api/runner/common/stats/statsd.go

@@ -0,0 +1,126 @@
+// Copyright 2016 Iron.io
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package stats
+
+import (
+	"bytes"
+	"fmt"
+	"net"
+	"strings"
+	"time"
+
+	"github.com/cactus/go-statsd-client/statsd"
+)
+
+type StatsdConfig struct {
+	StatsdUdpTarget string `json:"target" mapstructure:"target" envconfig:"STATSD_TARGET"`
+	Interval        int64  `json:"interval" envconfig:"STATSD_INTERVAL"`
+	Prefix          string `json:"prefix" envconfig:"STATSD_PREFIX"`
+}
+
+type keyCreator interface {
+	// The return value of Key *MUST* never have a '.' at the end.
+	Key(stat string) string
+}
+
+type theStatsdReporter struct {
+	keyCreator
+	client statsd.Statter
+}
+
+type prefixKeyCreator struct {
+	parent   keyCreator
+	prefixes []string
+}
+
+func (pkc *prefixKeyCreator) Key(stat string) string {
+	prefix := strings.Join(pkc.prefixes, ".")
+
+	if pkc.parent != nil {
+		prefix = pkc.parent.Key(prefix)
+	}
+
+	if stat == "" {
+		return prefix
+	}
+
+	if prefix == "" {
+		return stat
+	}
+
+	return prefix + "." + stat
+}
+
+func whoami() string {
+	a, _ := net.InterfaceAddrs()
+	for i := range a {
+		// is a textual representation of an IPv4 address
+		z, _, err := net.ParseCIDR(a[i].String())
+		if a[i].Network() == "ip+net" && err == nil && z.To4() != nil {
+			if !bytes.Equal(z, net.ParseIP("127.0.0.1")) {
+				return strings.Replace(fmt.Sprintf("%v", z), ".", "_", -1)
+			}
+		}
+	}
+	return "127_0_0_1" // shrug
+}
+
+// The config.Prefix is sent before each message and can be used to set API
+// keys. The prefix is used as the key prefix.
+// If config is nil, creates a noop reporter.
+//
+//	st, e := NewStatsd(config, "ironmq")
+//	st.Inc("enqueue", 1) -> Actually records to key ironmq.enqueue.
+func NewStatsd(config *StatsdConfig) (*theStatsdReporter, error) {
+	var client statsd.Statter
+	var err error
+	if config != nil {
+		// 512 for now since we are sending to hostedgraphite over the internet.
+		config.Prefix += "." + whoami()
+		client, err = statsd.NewBufferedClient(config.StatsdUdpTarget, config.Prefix, time.Duration(config.Interval)*time.Second, 512)
+	} else {
+		client, err = statsd.NewNoopClient()
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	return &theStatsdReporter{keyCreator: &prefixKeyCreator{}, client: client}, nil
+}
+
+func (sr *theStatsdReporter) Inc(component, stat string, value int64, rate float32) {
+	sr.client.Inc(sr.keyCreator.Key(component+"."+stat), value, rate)
+}
+
+func (sr *theStatsdReporter) Measure(component, stat string, delta int64, rate float32) {
+	sr.client.Timing(sr.keyCreator.Key(component+"."+stat), delta, rate)
+}
+
+func (sr *theStatsdReporter) Time(component, stat string, delta time.Duration, rate float32) {
+	sr.client.TimingDuration(sr.keyCreator.Key(component+"."+stat), delta, rate)
+}
+
+func (sr *theStatsdReporter) Gauge(component, stat string, value int64, rate float32) {
+	sr.client.Gauge(sr.keyCreator.Key(component+"."+stat), value, rate)
+}
+
+func (sr *theStatsdReporter) NewTimer(component string, stat string, rate float32) *Timer {
+	return newTimer(sr, component, stat, rate)
+}
+
+// We need some kind of all-or-nothing sampler where multiple stats can be
+// given the same rate and they are either all logged on that run or none of
+// them are. The statsd library we use ends up doing its own rate calculation
+// which is going to impede doing something like this.

api/runner/common/unix_logging.go

@@ -0,0 +1,22 @@
+// +build !windows,!nacl,!plan9
+
+package common
+
+import (
+	"io/ioutil"
+	"net/url"
+
+	"github.com/Sirupsen/logrus"
+	"github.com/Sirupsen/logrus/hooks/syslog"
+)
+
+func NewSyslogHook(url *url.URL, prefix string) error {
+	syslog, err := logrus_syslog.NewSyslogHook(url.Scheme, url.Host, 0, prefix)
+	if err != nil {
+		return err
+	}
+	logrus.AddHook(syslog)
+	// TODO we could support multiple destinations...
+	logrus.SetOutput(ioutil.Discard)
+	return nil
+}

api/runner/common/win_logging.go

@@ -0,0 +1,12 @@
+// +build !linux,!darwin
+
+package common
+
+import (
+	"errors"
+	"net/url"
+)
+
+func NewSyslogHook(url *url.URL, prefix string) error {
+	return errors.New("Syslog not supported on this system.")
+}

api/runner/common/writers.go

@@ -0,0 +1,204 @@
+// Copyright 2016 Iron.io
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package common
+
+import (
+	"bytes"
+	"errors"
+	"io"
+)
+
+// lineWriter will break apart a stream of data into individual lines.
+// Downstream writer will be called for each complete new line. When Flush
+// is called, a newline will be appended if there isn't one at the end.
+// Not thread-safe
+type LineWriter struct {
+	b *bytes.Buffer
+	w io.Writer
+}
+
+func NewLineWriter(w io.Writer) *LineWriter {
+	return &LineWriter{
+		w: w,
+		b: bytes.NewBuffer(make([]byte, 0, 1024)),
+	}
+}
+
+func (li *LineWriter) Write(p []byte) (int, error) {
+	n, err := li.b.Write(p)
+	if err != nil {
+		return n, err
+	}
+	if n != len(p) {
+		return n, errors.New("short write")
+	}
+
+	for {
+		b := li.b.Bytes()
+		i := bytes.IndexByte(b, '\n')
+		if i < 0 {
+			break
+		}
+
+		l := b[:i+1]
+		ns, err := li.w.Write(l)
+		if err != nil {
+			return ns, err
+		}
+		li.b.Next(len(l))
+	}
+
+	return n, nil
+}
+
+func (li *LineWriter) Flush() (int, error) {
+	b := li.b.Bytes()
+	if len(b) == 0 {
+		return 0, nil
+	}
+
+	if b[len(b)-1] != '\n' {
+		b = append(b, '\n')
+	}
+	return li.w.Write(b)
+}
+
+// HeadLinesWriter stores upto the first N lines in a buffer that can be
+// retrieved via Head().
+type HeadLinesWriter struct {
+	buffer bytes.Buffer
+	max    int
+}
+
+func NewHeadLinesWriter(max int) *HeadLinesWriter {
+	return &HeadLinesWriter{
+		buffer: bytes.Buffer{},
+		max:    max,
+	}
+}
+
+// Writes start failing once the writer has reached capacity.
+// In such cases the return value is the actual count written (may be zero) and io.ErrShortWrite.
+func (h *HeadLinesWriter) Write(p []byte) (n int, err error) {
+	var afterNewLine int
+	for h.max > 0 && afterNewLine < len(p) {
+		idx := bytes.IndexByte(p[afterNewLine:], '\n')
+		if idx == -1 {
+			h.buffer.Write(p[afterNewLine:])
+			afterNewLine = len(p)
+		} else {
+			h.buffer.Write(p[afterNewLine : afterNewLine+idx+1])
+			afterNewLine = afterNewLine + idx + 1
+			h.max--
+		}
+	}
+
+	if afterNewLine == len(p) {
+		return afterNewLine, nil
+	}
+
+	return afterNewLine, io.ErrShortWrite
+}
+
+// The returned bytes alias the buffer, the same restrictions as
+// bytes.Buffer.Bytes() apply.
+func (h *HeadLinesWriter) Head() []byte {
+	return h.buffer.Bytes()
+}
+
+// TailLinesWriter stores upto the last N lines in a buffer that can be retrieved
+// via Tail(). The truncation is only performed when more bytes are received
+// after '\n', so the buffer contents for both these writes are identical.
+//
+// tail writer that captures last 3 lines.
+// 'a\nb\nc\nd\n' -> 'b\nc\nd\n'
+// 'a\nb\nc\nd' -> 'b\nc\nd'
+type TailLinesWriter struct {
+	buffer             bytes.Buffer
+	max                int
+	newlineEncountered bool
+	// Tail is not idempotent without this.
+	tailCalled bool
+}
+
+func NewTailLinesWriter(max int) *TailLinesWriter {
+	return &TailLinesWriter{
+		buffer: bytes.Buffer{},
+		max:    max,
+	}
+}
+
+// Write always succeeds! This is because all len(p) bytes are written to the
+// buffer before it is truncated.
+func (t *TailLinesWriter) Write(p []byte) (n int, err error) {
+	if t.tailCalled {
+		return 0, errors.New("Tail() has already been called.")
+	}
+
+	var afterNewLine int
+	for afterNewLine < len(p) {
+		// This is at the top of the loop so it does not operate on trailing
+		// newlines. That is handled by Tail() where we have full knowledge that it
+		// is indeed the true trailing newline (if any).
+		if t.newlineEncountered {
+			if t.max > 0 {
+				// we still have capacity
+				t.max--
+			} else {
+				// chomp a newline.
+				t.chompNewline()
+			}
+		}
+
+		idx := bytes.IndexByte(p[afterNewLine:], '\n')
+		if idx == -1 {
+			t.buffer.Write(p[afterNewLine:])
+			afterNewLine = len(p)
+			t.newlineEncountered = false
+		} else {
+			t.buffer.Write(p[afterNewLine : afterNewLine+idx+1])
+			afterNewLine = afterNewLine + idx + 1
+			t.newlineEncountered = true
+		}
+
+	}
+	return len(p), nil
+}
+
+func (t *TailLinesWriter) chompNewline() {
+	b := t.buffer.Bytes()
+	idx := bytes.IndexByte(b, '\n')
+	if idx >= 0 {
+		t.buffer.Next(idx + 1)
+	} else {
+		// pretend a trailing newline exists. In the call in Write() this will
+		// never be hit.
+		t.buffer.Truncate(0)
+	}
+}
+
+// The returned bytes alias the buffer, the same restrictions as
+// bytes.Buffer.Bytes() apply.
+//
+// Once Tail() is called, further Write()s error.
+func (t *TailLinesWriter) Tail() []byte {
+	if !t.tailCalled {
+		t.tailCalled = true
+		if t.max <= 0 {
+			t.chompNewline()
+		}
+	}
+	return t.buffer.Bytes()
+}

api/runner/common/writers_test.go

@@ -0,0 +1,149 @@
+// Copyright 2016 Iron.io
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package common
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"testing"
+)
+
+type testSliceWriter struct {
+	b [][]byte
+}
+
+func (tsw *testSliceWriter) Write(p []byte) (n int, err error) {
+	l := make([]byte, len(p))
+	copy(l, p)
+	tsw.b = append(tsw.b, l)
+	return len(p), nil
+}
+
+func TestLineWriter(t *testing.T) {
+	tsw := &testSliceWriter{}
+	lw := NewLineWriter(tsw)
+
+	lineCount := 7
+	lw.Write([]byte("0 line\n1 line\n2 line\n\n4 line"))
+	lw.Write([]byte("+more\n5 line\n"))
+	lw.Write([]byte("6 line"))
+
+	lw.Flush()
+
+	if len(tsw.b) != lineCount {
+		t.Errorf("Expected %v individual rows; got %v", lineCount, len(tsw.b))
+	}
+
+	for x := 0; x < len(tsw.b); x++ {
+		l := fmt.Sprintf("%v line\n", x)
+		if x == 3 {
+			if len(tsw.b[x]) != 1 {
+				t.Errorf("Expected slice with only newline; got %v", tsw.b[x])
+			}
+			continue
+		} else if x == 4 {
+			l = "4 line+more\n"
+		}
+		if !bytes.Equal(tsw.b[x], []byte(l)) {
+			t.Errorf("Expected slice %s equal to %s", []byte(l), tsw.b[x])
+		}
+	}
+}
+
+func TestHeadWriter(t *testing.T) {
+	data := []byte("the quick\n brown\n fox jumped\n over the\n lazy dog.")
+	w := NewHeadLinesWriter(3)
+	_, err := w.Write(data[:4])
+	if err != nil {
+		t.Errorf("Expected nil error on small write")
+	}
+
+	if !bytes.Equal(w.Head(), []byte("the ")) {
+		t.Errorf("Expected 4 bytes in head, got '%s'", w.Head())
+	}
+
+	n, err := w.Write(data[4:16])
+	if n != len(data[4:16]) || err != nil {
+		t.Errorf("HeadWriter Write() does not satisfy contract about failing writes.")
+	}
+
+	if !bytes.Equal(w.Head(), []byte("the quick\n brown")) {
+		t.Errorf("unexpected contents of head, got '%s'", w.Head())
+	}
+
+	n, err = w.Write(data[16:])
+	if n != (29-16) || err != io.ErrShortWrite {
+		t.Errorf("HeadWriter Write() does not satisfy contract about failing writes.")
+	}
+	if !bytes.Equal(w.Head(), data[:29]) {
+		t.Errorf("unexpected contents of head, got '%s'", w.Head())
+	}
+}
+
+func testTail(t *testing.T, n int, output []byte, writes ...[]byte) {
+	w := NewTailLinesWriter(n)
+	for _, slice := range writes {
+		written, err := w.Write(slice)
+		if written != len(slice) || err != nil {
+			t.Errorf("Tail Write() should always succeed, but failed, input=%s, input length = %d, written=%d, err=%s", slice, len(slice), written, err)
+		}
+	}
+	if !bytes.Equal(w.Tail(), output) {
+		t.Errorf("Output did not match for tail writer of length %d: Expected '%s', got '%s'", n, output, w.Tail())
+	}
+}
+
+func TestTailWriter(t *testing.T) {
+	inputs := [][]byte{[]byte("a\nb\n"), []byte("gh"), []byte("\n")}
+	testTail(t, 2, []byte("b\ngh\n"), inputs...)
+}
+
+func TestZeroAndOneTailWriter(t *testing.T) {
+	// zero line writer, with only single line added to it should return empty buffer.
+	testTail(t, 0, []byte(""), []byte("Hello World\n"))
+	testTail(t, 0, []byte(""), []byte("Hello World"))
+
+	b1 := []byte("Hello World")
+	testTail(t, 1, b1, b1)
+
+	b1 = []byte("Hello World\n")
+	testTail(t, 1, b1, b1)
+
+	b2 := []byte("Yeah!\n")
+	testTail(t, 1, b2, b1, b2)
+
+	b1 = []byte("Flat write")
+	b2 = []byte("Yeah!\n")
+	j := bytes.Join([][]byte{b1, b2}, []byte{})
+	testTail(t, 1, j, b1, b2)
+}
+
+func TestTailWriterTrailing(t *testing.T) {
+	input1 := []byte("a\nb\nc\nd\ne")
+	input2 := []byte("a\nb\nc\nd\ne\n")
+	w1 := NewTailLinesWriter(4)
+	w1.Write(input1)
+	w2 := NewTailLinesWriter(4)
+	w2.Write(input2)
+	if !bytes.Equal(w1.Tail(), []byte("b\nc\nd\ne")) {
+		t.Errorf("Tail not working correctly, got '%s'", w1.Tail())
+	}
+
+	t2 := w2.Tail()
+	if !bytes.Equal(w1.Tail(), t2[:len(t2)-1]) {
+		t.Errorf("Tailwriter does not transition correctly over trailing newline. '%s', '%s'", w1.Tail(), t2)
+	}
+}