opentracing -> opencensus (#802)

* update vendor directory, add go.opencensus.io

* update imports

* oops

* s/opentracing/opencensus/ & remove prometheus / zipkin stuff & remove old stats

* the dep train rides again

* fix gin build

* deps from last guy

* start in on the agent metrics

* she builds

* remove tags for now, cardinality error is fussing. subscribe instead of register

* update to patched version of opencensus to proceed for now TODO switch to a release

* meh

fix imports

* println debug the bad boys

* lace it with the tags

* update deps again

* fix all inconsistent cardinality errors

* add our own logger

* fix init

* fix oom measure

* remove bugged removal code

* fix s3 measures

* fix prom handler nil

Gopkg.lock

@@ -34,18 +34,6 @@
   packages = ["."]
   revision = "de5bf2ad457846296e2031421a34e2568e304e35"
 
-[[projects]]
-  name = "github.com/Shopify/sarama"
-  packages = ["."]
-  revision = "3b1b38866a79f06deddf0487d5c27ba0697ccd65"
-  version = "v1.15.0"
-
-[[projects]]
-  name = "github.com/apache/thrift"
-  packages = ["lib/go/thrift"]
-  revision = "327ebb6c2b6df8bf075da02ef45a2a034e9b79ba"
-  version = "0.11.0"
-
 [[projects]]
   name = "github.com/asaskevich/govalidator"
   packages = ["."]
@@ -71,6 +59,7 @@
     "aws/request",
     "aws/session",
     "aws/signer/v4",
+    "internal/sdkrand",
     "internal/shareddefaults",
     "private/protocol",
     "private/protocol/query",
@@ -83,8 +72,8 @@
     "service/s3/s3manager",
     "service/sts"
   ]
-  revision = "decd990ddc5dcdf2f73309cbcab90d06b996ca28"
+  revision = "bff41fb23b7550368282029f6478819d6a99ae0f"
-  version = "v1.12.67"
+  version = "v1.12.79"
 
 [[projects]]
   branch = "master"
@@ -101,23 +90,14 @@
   branch = "master"
   name = "github.com/containerd/continuity"
   packages = ["pathdriver"]
-  revision = "b2b946a77f5973f420514090d6f6dd58b08303f0"
+  revision = "d8fb8589b0e8e85b8c8bbaa8840226d0dfeb7371"
 
 [[projects]]
   name = "github.com/coreos/go-semver"
-  packages = [
+  packages = ["semver"]
-    ".",
-    "semver"
-  ]
   revision = "8ab6407b697782a06568d4b7f1db25550ec2e4c6"
   version = "v0.2.0"
 
-[[projects]]
-  name = "github.com/davecgh/go-spew"
-  packages = ["spew"]
-  revision = "346938d642f2ec3594ed81d874461961cd0faa76"
-  version = "v1.1.0"
-
 [[projects]]
   name = "github.com/docker/distribution"
   packages = [
@@ -168,7 +148,7 @@
     "pkg/term",
     "pkg/term/windows"
   ]
-  revision = "99cfb5f31ad82238573de3475bf5bb0435ac1ebc"
+  revision = "1346a2c89a11f2d111ff20f46d557f1f9ccbbeb7"
 
 [[projects]]
   name = "github.com/docker/go-connections"
@@ -188,29 +168,14 @@
   packages = ["."]
   revision = "aabc10ec26b754e797f9028f4589c5b7bd90dc20"
 
-[[projects]]
-  name = "github.com/eapache/go-resiliency"
-  packages = ["breaker"]
-  revision = "6800482f2c813e689c88b7ed3282262385011890"
-  version = "v1.0.0"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/eapache/go-xerial-snappy"
-  packages = ["."]
-  revision = "bb955e01b9346ac19dc29eb16586c90ded99a98c"
-
-[[projects]]
-  name = "github.com/eapache/queue"
-  packages = ["."]
-  revision = "44cc805cf13205b55f69e14bcb69867d1ae92f98"
-  version = "v1.1.0"
-
 [[projects]]
   branch = "master"
   name = "github.com/fnproject/fdk-go"
-  packages = ["."]
+  packages = [
-  revision = "46839d559f713a68137f3e4fc4117e7440231c57"
+    ".",
+    "utils"
+  ]
+  revision = "49183b946a083774a18b0de0a3184abebc1dec8d"
 
 [[projects]]
   name = "github.com/fnproject/fn_go"
@@ -237,7 +202,7 @@
     "redis"
   ]
   revision = "d1ed5c67e5794de818ea85e6b522fda02623a484"
-  version = "v1.4.0"
+  version = "v1.5.0"
 
 [[projects]]
   name = "github.com/gin-contrib/cors"
@@ -267,17 +232,11 @@
   revision = "32e4c1e6bc4e7d0d8451aa6b75200d19e37a536a"
   version = "v1.32.0"
 
-[[projects]]
-  name = "github.com/go-logfmt/logfmt"
-  packages = ["."]
-  revision = "390ab7935ee28ec6b286364bba9b4dd6410cb3d5"
-  version = "v0.3.0"
-
 [[projects]]
   branch = "master"
   name = "github.com/go-openapi/analysis"
   packages = ["."]
-  revision = "0db1e4cf47d6eb0fc6b25135868c3483cc6b066e"
+  revision = "f59a71f0ece6f9dfb438be7f45148f006cbad88e"
 
 [[projects]]
   branch = "master"
@@ -316,31 +275,31 @@
     "middleware/untyped",
     "security"
   ]
-  revision = "be1313bd8283bc82b48033942f0f3730eaef2d90"
+  revision = "09fac855d8504674b22594470227bd94e5005025"
 
 [[projects]]
   branch = "master"
   name = "github.com/go-openapi/spec"
   packages = ["."]
-  revision = "fa03337d7da5735229ee8f5e9d5d0b996014b7f8"
+  revision = "1de3e0542de65ad8d75452a595886fdd0befb363"
 
 [[projects]]
   branch = "master"
   name = "github.com/go-openapi/strfmt"
   packages = ["."]
-  revision = "4dd3d302e100bae008baedc42d446ce83bdd10ad"
+  revision = "6d1a47fad79c81e8cd732889cb80e91123951860"
 
 [[projects]]
   branch = "master"
   name = "github.com/go-openapi/swag"
   packages = ["."]
-  revision = "84f4bee7c0a6db40e3166044c7983c1c32125429"
+  revision = "0d03ad0b6405ada874d59d97c416b5cf4234e154"
 
 [[projects]]
   branch = "master"
   name = "github.com/go-openapi/validate"
   packages = ["."]
-  revision = "9b5947525af6a4843d2b9576e5943b9931ad535d"
+  revision = "180bba53b98899f743a112e568bed9e2ef31aa20"
 
 [[projects]]
   name = "github.com/go-sql-driver/mysql"
@@ -350,26 +309,20 @@
 [[projects]]
   name = "github.com/gogo/protobuf"
   packages = ["proto"]
-  revision = "342cbe0a04158f6dcb03ca0079991a51a4248c02"
+  revision = "1adfc126b41513cc696b209667c8656ea7aac67c"
-  version = "v0.5"
+  version = "v1.0.0"
 
 [[projects]]
-  branch = "master"
   name = "github.com/golang/protobuf"
   packages = ["proto"]
-  revision = "c65a0412e71e8b9b3bfd22925720d23c0f054237"
+  revision = "925541529c1fa6821df4e44ce2723319eb2be768"
-
+  version = "v1.0.0"
-[[projects]]
-  branch = "master"
-  name = "github.com/golang/snappy"
-  packages = ["."]
-  revision = "553a641470496b2327abcac10b36396bd98e45c9"
 
 [[projects]]
   branch = "master"
   name = "github.com/google/btree"
   packages = ["."]
-  revision = "316fb6d3f031ae8f4d457c6c5186b9e3ded70435"
+  revision = "e89373fe6b4a7413d7acd6da1725b83ef713e6e4"
 
 [[projects]]
   name = "github.com/gorilla/context"
@@ -395,13 +348,7 @@
     ".",
     "reflectx"
   ]
-  revision = "de8647470aafe4854c976707c431dbe1eb2822c6"
+  revision = "05cef0741ade10ca668982355b3f3f0bcf0ff0a8"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/kr/logfmt"
-  packages = ["."]
-  revision = "b84e30acd515aadc4b783ad4ff83aff3299bdfe0"
 
 [[projects]]
   branch = "master"
@@ -410,7 +357,7 @@
     ".",
     "oid"
   ]
-  revision = "61fe37aa2ee24fabcdbe5c4ac1d4ac566f88f345"
+  revision = "88edab0803230a3898347e77b474f8c1820a1f20"
 
 [[projects]]
   branch = "master"
@@ -454,7 +401,7 @@
   branch = "master"
   name = "github.com/mitchellh/mapstructure"
   packages = ["."]
-  revision = "b4575eea38cca1123ec2dc90c26529b5c5acfcff"
+  revision = "00c29f56e2386353d58c599509e8dc3801b0d716"
 
 [[projects]]
   name = "github.com/opencontainers/go-digest"
@@ -482,32 +429,13 @@
 
 [[projects]]
   branch = "master"
-  name = "github.com/opentracing-contrib/go-observer"
+  name = "github.com/openzipkin/zipkin-go"
-  packages = ["."]
-  revision = "a52f2342449246d5bcc273e65cbdcfa5f7d6c63c"
-
-[[projects]]
-  name = "github.com/opentracing/opentracing-go"
   packages = [
-    ".",
+    "model",
-    "ext",
+    "reporter",
-    "log"
+    "reporter/http"
   ]
-  revision = "1949ddbfd147afd4d964a9f00b24eb291e0e7c38"
+  revision = "f197ec29e729f226d23370ea60f0e49b8f44ccf4"
-  version = "v1.0.2"
-
-[[projects]]
-  name = "github.com/openzipkin/zipkin-go-opentracing"
-  packages = [
-    ".",
-    "flag",
-    "thrift/gen-go/scribe",
-    "thrift/gen-go/zipkincore",
-    "types",
-    "wire"
-  ]
-  revision = "45e90b00710a4c34a1a7d8a78d90f9b010b0bd4d"
-  version = "v0.3.2"
 
 [[projects]]
   name = "github.com/patrickmn/go-cache"
@@ -521,18 +449,6 @@
   packages = ["llrb"]
   revision = "53be0d36a84c2a886ca057d34b6aa4468df9ccb4"
 
-[[projects]]
-  name = "github.com/pierrec/lz4"
-  packages = ["."]
-  revision = "2fcda4cb7018ce05a25959d2fe08c83e3329f169"
-  version = "v1.1"
-
-[[projects]]
-  name = "github.com/pierrec/xxHash"
-  packages = ["xxHash32"]
-  revision = "f051bb7f1d1aaf1b5a665d74fb6b0217712c69f7"
-  version = "v0.1.1"
-
 [[projects]]
   name = "github.com/pkg/errors"
   packages = ["."]
@@ -570,16 +486,10 @@
   packages = [
     ".",
     "internal/util",
-    "nfsd",
+    "nfs",
     "xfs"
   ]
-  revision = "85fadb6e89903ef7cca6f6a804474cd5ea85b6e1"
+  revision = "75f2d6163c7a100bed6e971044ea3de30ee3a678"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/rcrowley/go-metrics"
-  packages = ["."]
-  revision = "e181e095bae94582363434144c61a9653aff6e50"
 
 [[projects]]
   name = "github.com/rdallman/migrate"
@@ -607,11 +517,30 @@
   revision = "9831f2c3ac1068a78f50999a30db84270f647af6"
   version = "v1.1"
 
+[[projects]]
+  branch = "master"
+  name = "go.opencensus.io"
+  packages = [
+    "exporter/prometheus",
+    "exporter/zipkin",
+    "internal",
+    "internal/tagencoding",
+    "plugin/ochttp",
+    "plugin/ochttp/propagation/b3",
+    "stats",
+    "stats/internal",
+    "stats/view",
+    "tag",
+    "trace",
+    "trace/propagation"
+  ]
+  revision = "4566f1f203a083514e9c57fecc0836ecc4a0eb11"
+
 [[projects]]
   branch = "master"
   name = "golang.org/x/crypto"
   packages = ["ssh/terminal"]
-  revision = "3d37316aaa6bd9929127ac9a527abf408178ea7b"
+  revision = "8c653846df49742c4c85ec37e5d9f8d3ba657895"
 
 [[projects]]
   branch = "master"
@@ -621,7 +550,7 @@
     "context/ctxhttp",
     "idna"
   ]
-  revision = "5ccada7d0a7ba9aeb5d3aca8d3501b4c2a509fec"
+  revision = "cbe0f9307d0156177f9dd5dc85da1a31abc5f2fb"
 
 [[projects]]
   branch = "master"
@@ -630,10 +559,9 @@
     "unix",
     "windows"
   ]
-  revision = "af50095a40f9041b3b38960738837185c26e9419"
+  revision = "f6cff0780e542efa0c8e864dc8fa522808f6a598"
 
 [[projects]]
-  branch = "master"
   name = "golang.org/x/text"
   packages = [
     "collate",
@@ -652,7 +580,8 @@
     "unicode/rangetable",
     "width"
   ]
-  revision = "e19ae1496984b1c655b8044a65c0300a3c878dd3"
+  revision = "f21a4dfb5e38f5895301dc265a8def02365cc3d0"
+  version = "v0.3.0"
 
 [[projects]]
   name = "gopkg.in/go-playground/validator.v8"
@@ -670,14 +599,14 @@
   revision = "3f83fa5005286a7fe593b055f0d7771a7dce4655"
 
 [[projects]]
-  branch = "v2"
   name = "gopkg.in/yaml.v2"
   packages = ["."]
-  revision = "d670f9405373e636a5a2765eea47fac0c9bc91a4"
+  revision = "7f97868eec74b32b0982dd158a51a446d1da7eb5"
+  version = "v2.1.1"
 
 [solve-meta]
   analyzer-name = "dep"
   analyzer-version = 1
-  inputs-digest = "f5c10f8c63262754a0ceb541105cfac5448693af7997269bb1edce828eb3a551"
+  inputs-digest = "51e5bbe66f1c9af38787323fffb614441513d6ec0eca0e31781954f445e12e63"
   solver-name = "gps-cdcl"
   solver-version = 1

Gopkg.toml

@@ -30,8 +30,8 @@ ignored = ["github.com/fnproject/fn/cli"]
   version = "0.2.0"
 
 [[constraint]]
-  name = "github.com/gin-contrib/cors"
+  name = "github.com/gin-gonic/gin"
-  version = "~1.2.0"
+  version = "v1.2"
 
 [[constraint]]
   branch = "master"
@@ -41,18 +41,10 @@ ignored = ["github.com/fnproject/fn/cli"]
   branch = "master"
   name = "github.com/google/btree"
 
-[[constraint]]
-  name = "github.com/openzipkin/zipkin-go-opentracing"
-  version = "0.3.1"
-
 [[constraint]]
   name = "github.com/patrickmn/go-cache"
   version = "2.1.0"
 
-[[constraint]]
-  name = "github.com/prometheus/client_golang"
-  version = "0.8.0"
-
 [[constraint]]
   name = "github.com/sirupsen/logrus"
   revision = "89742aefa4b206dcf400792f3bd35b542998eb3b"
@@ -66,6 +58,7 @@ ignored = ["github.com/fnproject/fn/cli"]
   revision = "bc72eeb997c7334cb5f05f5aefd2d70bc34d71ef"
 
 [[constraint]]
+  # NOTE: locked for a reason - https://github.com/go-sql-driver/mysql/issues/657
   name = "github.com/go-sql-driver/mysql"
   revision = "21d7e97c9f760ca685a01ecea202e1c84276daa1"
 
@@ -80,3 +73,7 @@ ignored = ["github.com/fnproject/fn/cli"]
 [[constraint]]
   name = "github.com/coreos/go-semver"
   version = "0.2.0"
+
+[[constraint]]
+  name = "go.opencensus.io"
+  branch = "master"

api/agent/agent.go

@@ -3,7 +3,6 @@ package agent
 import (
 	"context"
 	"io"
-	"net/http"
 	"strings"
 	"sync"
 	"time"
@@ -16,9 +15,11 @@ import (
 	"github.com/fnproject/fn/api/models"
 	"github.com/fnproject/fn/fnext"
 	"github.com/go-openapi/strfmt"
-	"github.com/opentracing/opentracing-go"
-	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/sirupsen/logrus"
+	"go.opencensus.io/stats"
+	"go.opencensus.io/stats/view"
+	"go.opencensus.io/tag"
+	"go.opencensus.io/trace"
 )
 
 // TODO we should prob store async calls in db immediately since we're returning id (will 404 until post-execution)
@@ -85,8 +86,6 @@ type Agent interface {
 	// Close is not safe to be called from multiple threads.
 	io.Closer
 
-	// Return the http.Handler used to handle Prometheus metric requests
-	PromHandler() http.Handler
 	AddCallListener(fnext.CallListener)
 
 	// Enqueue is to use the agent's sweet sweet client bindings to remotely
@@ -109,9 +108,6 @@ type agent struct {
 	wg       sync.WaitGroup // TODO rename
 	shutonce sync.Once
 	shutdown chan struct{}
-
-	// Prometheus HTTP handler
-	promHandler http.Handler
 }
 
 func New(da DataAccess) Agent {
@@ -128,13 +124,12 @@ func New(da DataAccess) Agent {
 	})
 
 	a := &agent{
-		cfg:         *cfg,
+		cfg:       *cfg,
-		da:          da,
+		da:        da,
-		driver:      driver,
+		driver:    driver,
-		slotMgr:     NewSlotQueueMgr(),
+		slotMgr:   NewSlotQueueMgr(),
-		resources:   NewResourceTracker(),
+		resources: NewResourceTracker(),
-		shutdown:    make(chan struct{}),
+		shutdown:  make(chan struct{}),
-		promHandler: promhttp.Handler(),
 	}
 
 	// TODO assert that agent doesn't get started for API nodes up above ?
@@ -174,8 +169,8 @@ func (a *agent) Submit(callI Call) error {
 	call.req = call.req.WithContext(ctx)
 	defer cancel()
 
-	ctx, finish := statSpans(ctx, call)
+	ctx, span := trace.StartSpan(ctx, "agent_submit")
-	defer finish()
+	defer span.End()
 
 	err := a.submit(ctx, call)
 	return err
@@ -202,14 +197,15 @@ func (a *agent) endStateTrackers(ctx context.Context, call *call) {
 }
 
 func (a *agent) submit(ctx context.Context, call *call) error {
-	StatsEnqueue(ctx)
+	statsEnqueue(ctx)
 
+	// TODO can we replace state trackers with metrics?
 	a.startStateTrackers(ctx, call)
 	defer a.endStateTrackers(ctx, call)
 
 	slot, err := a.getSlot(ctx, call)
 	if err != nil {
-		a.handleStatsDequeue(ctx, call, err)
+		handleStatsDequeue(ctx, err)
 		return transformTimeout(err, true)
 	}
 
@@ -217,20 +213,19 @@ func (a *agent) submit(ctx context.Context, call *call) error {
 
 	err = call.Start(ctx)
 	if err != nil {
-		a.handleStatsDequeue(ctx, call, err)
+		handleStatsDequeue(ctx, err)
 		return transformTimeout(err, true)
 	}
 
-	// decrement queued count, increment running count
+	statsDequeueAndStart(ctx)
-	StatsDequeueAndStart(ctx)
 
 	// pass this error (nil or otherwise) to end directly, to store status, etc
 	err = slot.exec(ctx, call)
-	a.handleStatsEnd(ctx, call, err)
+	handleStatsEnd(ctx, err)
 
 	// TODO: we need to allocate more time to store the call + logs in case the call timed out,
 	// but this could put us over the timeout if the call did not reply yet (need better policy).
-	ctx = opentracing.ContextWithSpan(context.Background(), opentracing.SpanFromContext(ctx))
+	ctx = common.BackgroundContext(ctx)
 	err = call.End(ctx, err)
 	return transformTimeout(err, false)
 }
@@ -247,54 +242,33 @@ func transformTimeout(e error, isRetriable bool) error {
 
 // handleStatsDequeue handles stats for dequeuing for early exit (getSlot or Start)
 // cases. Only timeouts can be a simple dequeue while other cases are actual errors.
-func (a *agent) handleStatsDequeue(ctx context.Context, call *call, err error) {
+func handleStatsDequeue(ctx context.Context, err error) {
 	if err == context.DeadlineExceeded {
-		StatsDequeue(ctx)
+		statsDequeue(ctx)
-		StatsIncrementTooBusy(ctx)
+		statsTooBusy(ctx)
 	} else {
-		StatsDequeueAndFail(ctx)
+		statsDequeueAndFail(ctx)
-		StatsIncrementErrors(ctx)
+		statsErrors(ctx)
 	}
 }
 
 // handleStatsEnd handles stats for after a call is ran, depending on error.
-func (a *agent) handleStatsEnd(ctx context.Context, call *call, err error) {
+func handleStatsEnd(ctx context.Context, err error) {
 	if err == nil {
 		// decrement running count, increment completed count
-		StatsComplete(ctx)
+		statsComplete(ctx)
 	} else {
 		// decrement running count, increment failed count
-		StatsFailed(ctx)
+		statsFailed(ctx)
 		// increment the timeout or errors count, as appropriate
 		if err == context.DeadlineExceeded {
-			StatsIncrementTimedout(ctx)
+			statsTimedout(ctx)
 		} else {
-			StatsIncrementErrors(ctx)
+			statsErrors(ctx)
 		}
 	}
 }
 
-func statSpans(ctx context.Context, call *call) (_ context.Context, finish func()) {
-	// agent_submit_global has no parent span because we don't want it to inherit fn_appname or fn_path
-	spanGlobal := opentracing.StartSpan("agent_submit_global")
-
-	// agent_submit_global has no parent span because we don't want it to inherit fn_path
-	spanApp := opentracing.StartSpan("agent_submit_app")
-	spanApp.SetBaggageItem("fn_appname", call.AppName)
-
-	// agent_submit has a parent span in the usual way
-	// it doesn't matter if it inherits fn_appname or fn_path (and we set them here in any case)
-	span, ctx := opentracing.StartSpanFromContext(ctx, "agent_submit")
-	span.SetBaggageItem("fn_appname", call.AppName)
-	span.SetBaggageItem("fn_path", call.Path)
-
-	return ctx, func() {
-		spanGlobal.Finish()
-		spanApp.Finish()
-		span.Finish()
-	}
-}
-
 // getSlot returns a Slot (or error) for the request to run. Depending on hot/cold
 // request type, this may launch a new container or wait for other containers to become idle
 // or it may wait for resources to become available to launch a new container.
@@ -303,8 +277,8 @@ func (a *agent) getSlot(ctx context.Context, call *call) (Slot, error) {
 	ctx, cancel := context.WithDeadline(ctx, call.slotDeadline)
 	defer cancel()
 
-	span, ctx := opentracing.StartSpanFromContext(ctx, "agent_get_slot")
+	ctx, span := trace.StartSpan(ctx, "agent_get_slot")
-	defer span.Finish()
+	defer span.End()
 
 	if protocol.IsStreamable(protocol.Protocol(call.Format)) {
 		// For hot requests, we use a long lived slot queue, which we use to manage hot containers
@@ -340,9 +314,9 @@ func (a *agent) hotLauncher(ctx context.Context, call *call) {
 
 	// IMPORTANT: get a context that has a child span / logger but NO timeout
 	// TODO this is a 'FollowsFrom'
-	ctx = opentracing.ContextWithSpan(common.WithLogger(context.Background(), logger), opentracing.SpanFromContext(ctx))
+	ctx = common.BackgroundContext(ctx)
-	span, ctx := opentracing.StartSpanFromContext(ctx, "agent_hot_launcher")
+	ctx, span := trace.StartSpan(ctx, "agent_hot_launcher")
-	defer span.Finish()
+	defer span.End()
 
 	for {
 		ctx, cancel := context.WithTimeout(ctx, timeout)
@@ -395,8 +369,8 @@ func (a *agent) checkLaunch(ctx context.Context, call *call) {
 
 // waitHot pings and waits for a hot container from the slot queue
 func (a *agent) waitHot(ctx context.Context, call *call) (Slot, error) {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "agent_wait_hot")
+	ctx, span := trace.StartSpan(ctx, "agent_wait_hot")
-	defer span.Finish()
+	defer span.End()
 
 	ctx, cancel := context.WithCancel(ctx)
 	defer cancel() // shut down dequeuer if we grab a slot
@@ -442,8 +416,8 @@ func (a *agent) launchCold(ctx context.Context, call *call) (Slot, error) {
 	isAsync := call.Type == models.TypeAsync
 	ch := make(chan Slot)
 
-	span, ctx := opentracing.StartSpanFromContext(ctx, "agent_launch_cold")
+	ctx, span := trace.StartSpan(ctx, "agent_launch_cold")
-	defer span.Finish()
+	defer span.End()
 
 	call.containerState.UpdateState(ctx, ContainerStateWait, call.slots)
 
@@ -479,8 +453,8 @@ func (s *coldSlot) Error() error {
 }
 
 func (s *coldSlot) exec(ctx context.Context, call *call) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "agent_cold_exec")
+	ctx, span := trace.StartSpan(ctx, "agent_cold_exec")
-	defer span.Finish()
+	defer span.End()
 
 	call.requestState.UpdateState(ctx, RequestStateExec, call.slots)
 	call.containerState.UpdateState(ctx, ContainerStateBusy, call.slots)
@@ -507,7 +481,7 @@ func (s *coldSlot) Close(ctx context.Context) error {
 		// call this from here so that in exec we don't have to eat container
 		// removal latency
 		// NOTE ensure container removal, no ctx timeout
-		ctx = opentracing.ContextWithSpan(context.Background(), opentracing.SpanFromContext(ctx))
+		ctx = common.BackgroundContext(ctx)
 		s.cookie.Close(ctx)
 	}
 	if s.tok != nil {
@@ -535,8 +509,8 @@ func (s *hotSlot) Error() error {
 }
 
 func (s *hotSlot) exec(ctx context.Context, call *call) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "agent_hot_exec")
+	ctx, span := trace.StartSpan(ctx, "agent_hot_exec")
-	defer span.Finish()
+	defer span.End()
 
 	call.requestState.UpdateState(ctx, RequestStateExec, call.slots)
 
@@ -575,8 +549,8 @@ func (s *hotSlot) exec(ctx context.Context, call *call) error {
 }
 
 func (a *agent) prepCold(ctx context.Context, call *call, tok ResourceToken, ch chan Slot) {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "agent_prep_cold")
+	ctx, span := trace.StartSpan(ctx, "agent_prep_cold")
-	defer span.Finish()
+	defer span.End()
 
 	call.containerState.UpdateState(ctx, ContainerStateStart, call.slots)
 
@@ -622,9 +596,9 @@ func (a *agent) prepCold(ctx context.Context, call *call, tok ResourceToken, ch
 func (a *agent) runHot(ctx context.Context, call *call, tok ResourceToken, state ContainerState) {
 	// IMPORTANT: get a context that has a child span / logger but NO timeout
 	// TODO this is a 'FollowsFrom'
-	ctx = opentracing.ContextWithSpan(context.Background(), opentracing.SpanFromContext(ctx))
+	ctx = common.BackgroundContext(ctx)
-	span, ctx := opentracing.StartSpanFromContext(ctx, "agent_run_hot")
+	ctx, span := trace.StartSpan(ctx, "agent_run_hot")
-	defer span.Finish()
+	defer span.End()
 	defer tok.Close() // IMPORTANT: this MUST get called
 
 	state.UpdateState(ctx, ContainerStateStart, call.slots)
@@ -852,17 +826,10 @@ func (c *container) CPUs() uint64                   { return c.cpus }
 
 // WriteStat publishes each metric in the specified Stats structure as a histogram metric
 func (c *container) WriteStat(ctx context.Context, stat drivers.Stat) {
-
-	// Convert each metric value from uint64 to float64
-	// and, for backward compatibility reasons, prepend each metric name with "docker_stats_fn_"
-	// (if we don't care about compatibility then we can remove that)
-	var metrics = make(map[string]float64)
 	for key, value := range stat.Metrics {
-		metrics["docker_stats_fn_"+key] = float64(value)
+		stats.Record(ctx, stats.FindMeasure("docker_stats_"+key).(*stats.Int64Measure).M(int64(value)))
 	}
 
-	common.PublishHistograms(ctx, metrics)
-
 	c.statsMu.Lock()
 	if c.stats != nil {
 		*(c.stats) = append(*(c.stats), stat)
@@ -870,6 +837,45 @@ func (c *container) WriteStat(ctx context.Context, stat drivers.Stat) {
 	c.statsMu.Unlock()
 }
 
+func init() {
+	// TODO this is nasty figure out how to use opencensus to not have to declare these
+	keys := []string{"net_rx", "net_tx", "mem_limit", "mem_usage", "disk_read", "disk_write", "cpu_user", "cpu_total", "cpu_kernel"}
+
+	// TODO necessary?
+	appKey, err := tag.NewKey("fn_appname")
+	if err != nil {
+		logrus.Fatal(err)
+	}
+	pathKey, err := tag.NewKey("fn_path")
+	if err != nil {
+		logrus.Fatal(err)
+	}
+
+	for _, key := range keys {
+		units := "bytes"
+		if strings.Contains(key, "cpu") {
+			units = "cpu"
+		}
+		dockerStatsDist, err := stats.Int64("docker_stats_"+key, "docker container stats for "+key, units)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+		v, err := view.New(
+			"docker_stats_"+key,
+			"docker container stats for "+key,
+			[]tag.Key{appKey, pathKey},
+			dockerStatsDist,
+			view.DistributionAggregation{},
+		)
+		if err != nil {
+			logrus.Fatalf("cannot create view: %v", err)
+		}
+		if err := v.Subscribe(); err != nil {
+			logrus.Fatal(err)
+		}
+	}
+}
+
 //func (c *container) DockerAuth() (docker.AuthConfiguration, error) {
 // Implementing the docker.AuthConfiguration interface.
 // TODO per call could implement this stored somewhere (vs. configured on host)

api/agent/async.go

@@ -4,9 +4,11 @@ import (
 	"context"
 	"time"
 
+	"github.com/fnproject/fn/api/common"
 	"github.com/fnproject/fn/api/models"
-	opentracing "github.com/opentracing/opentracing-go"
 	"github.com/sirupsen/logrus"
+	"go.opencensus.io/tag"
+	"go.opencensus.io/trace"
 )
 
 func (a *agent) asyncDequeue() {
@@ -17,8 +19,8 @@ func (a *agent) asyncDequeue() {
 	defer cancel()
 
 	// parent span here so that we can see how many async calls are running
-	span, ctx := opentracing.StartSpanFromContext(ctx, "agent_async_dequeue")
+	ctx, span := trace.StartSpan(ctx, "agent_async_dequeue")
-	defer span.Finish()
+	defer span.End()
 
 	for {
 		select {
@@ -73,11 +75,29 @@ func (a *agent) asyncChew(ctx context.Context) <-chan *models.Call {
 func (a *agent) asyncRun(ctx context.Context, model *models.Call) {
 	// IMPORTANT: get a context that has a child span but NO timeout (Submit imposes timeout)
 	// TODO this is a 'FollowsFrom'
-	ctx = opentracing.ContextWithSpan(context.Background(), opentracing.SpanFromContext(ctx))
+	ctx = common.BackgroundContext(ctx)
+
+	// since async doesn't come in through the normal request path,
+	// we've gotta add tags here for stats to come out properly.
+	appKey, err := tag.NewKey("fn_appname")
+	if err != nil {
+		logrus.Fatal(err)
+	}
+	pathKey, err := tag.NewKey("fn_path")
+	if err != nil {
+		logrus.Fatal(err)
+	}
+	ctx, err = tag.New(ctx,
+		tag.Insert(appKey, model.AppName),
+		tag.Insert(pathKey, model.Path),
+	)
+	if err != nil {
+		logrus.Fatal(err)
+	}
 
 	// additional enclosing context here since this isn't spawned from an http request
-	span, ctx := opentracing.StartSpanFromContext(ctx, "agent_async_run")
+	ctx, span := trace.StartSpan(ctx, "agent_async_run")
-	defer span.Finish()
+	defer span.End()
 
 	call, err := a.GetCall(
 		FromModel(model),

api/agent/call.go

@@ -9,12 +9,13 @@ import (
 	"strings"
 	"time"
 
+	"go.opencensus.io/trace"
+
 	"github.com/fnproject/fn/api/agent/drivers"
 	"github.com/fnproject/fn/api/common"
 	"github.com/fnproject/fn/api/id"
 	"github.com/fnproject/fn/api/models"
 	"github.com/go-openapi/strfmt"
-	"github.com/opentracing/opentracing-go"
 	"github.com/sirupsen/logrus"
 )
 
@@ -249,8 +250,8 @@ type call struct {
 func (c *call) Model() *models.Call { return c.Call }
 
 func (c *call) Start(ctx context.Context) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "agent_call_start")
+	ctx, span := trace.StartSpan(ctx, "agent_call_start")
-	defer span.Finish()
+	defer span.End()
 
 	// Check context timeouts, errors
 	if ctx.Err() != nil {
@@ -290,8 +291,8 @@ func (c *call) Start(ctx context.Context) error {
 }
 
 func (c *call) End(ctx context.Context, errIn error) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "agent_call_end")
+	ctx, span := trace.StartSpan(ctx, "agent_call_end")
-	defer span.Finish()
+	defer span.End()
 
 	c.CompletedAt = strfmt.DateTime(time.Now())
 

api/agent/drivers/docker/docker.go

@@ -12,13 +12,15 @@ import (
 	"strings"
 	"time"
 
+	"go.opencensus.io/stats"
+	"go.opencensus.io/trace"
+
 	"github.com/coreos/go-semver/semver"
 	"github.com/fnproject/fn/api/agent/drivers"
 	"github.com/fnproject/fn/api/common"
 	"github.com/fnproject/fn/api/models"
 	"github.com/fsouza/go-dockerclient"
 	"github.com/go-openapi/strfmt"
-	"github.com/opentracing/opentracing-go"
 	"github.com/sirupsen/logrus"
 )
 
@@ -77,7 +79,6 @@ func NewDocker(conf drivers.Config) *DockerDriver {
 }
 
 func checkDockerVersion(driver *DockerDriver, expected string) error {
-
 	info, err := driver.docker.Info(context.Background())
 	if err != nil {
 		return err
@@ -269,9 +270,9 @@ func (drv *DockerDriver) ensureImage(ctx context.Context, task drivers.Container
 
 	if task, ok := task.(Auther); ok {
 		var err error
-		span, _ := opentracing.StartSpanFromContext(ctx, "docker_auth")
+		_, span := trace.StartSpan(ctx, "docker_auth")
 		config, err = task.DockerAuth()
-		span.Finish()
+		span.End()
 		if err != nil {
 			return err
 		}
@@ -396,8 +397,8 @@ func (w *waitResult) Wait(ctx context.Context) (drivers.RunResult, error) {
 
 // Repeatedly collect stats from the specified docker container until the stopSignal is closed or the context is cancelled
 func (drv *DockerDriver) collectStats(ctx context.Context, stopSignal <-chan struct{}, container string, task drivers.ContainerTask) {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "docker_collect_stats")
+	ctx, span := trace.StartSpan(ctx, "docker_collect_stats")
-	defer span.Finish()
+	defer span.End()
 
 	log := common.Logger(ctx)
 
@@ -576,7 +577,7 @@ func (w *waitResult) wait(ctx context.Context) (status string, err error) {
 	case 0:
 		return drivers.StatusSuccess, nil
 	case 137: // OOM
-		// TODO put in stats opentracing.SpanFromContext(ctx).LogFields(log.String("docker", "oom"))
+		stats.Record(ctx, dockerOOMMeasure.M(1))
 		common.Logger(ctx).Error("docker oom")
 		err := errors.New("container out of memory, you may want to raise route.memory for this route (default: 128MB)")
 		return drivers.StatusKilled, models.NewAPIError(http.StatusBadGateway, err)

api/agent/drivers/docker/docker_client.go

@@ -12,9 +12,11 @@ import (
 
 	"github.com/fnproject/fn/api/common"
 	"github.com/fsouza/go-dockerclient"
-	"github.com/opentracing/opentracing-go"
-	"github.com/opentracing/opentracing-go/log"
 	"github.com/sirupsen/logrus"
+	"go.opencensus.io/stats"
+	"go.opencensus.io/stats/view"
+	"go.opencensus.io/tag"
+	"go.opencensus.io/trace"
 )
 
 const (
@@ -98,18 +100,123 @@ type dockerWrap struct {
 	dockerNoTimeout *docker.Client
 }
 
+func init() {
+	// TODO doing this at each call site seems not the intention of the library since measurements
+	// need to be created and views registered. doing this up front seems painful but maybe there
+	// are benefits?
+
+	// TODO do we have to do this? the measurements will be tagged on the context, will they be propagated
+	// or we have to white list them in the view for them to show up? test...
+	var err error
+	appKey, err := tag.NewKey("fn_appname")
+	if err != nil {
+		logrus.Fatal(err)
+	}
+	pathKey, err := tag.NewKey("fn_path")
+	if err != nil {
+		logrus.Fatal(err)
+	}
+
+	{
+		dockerRetriesMeasure, err = stats.Int64("docker_api_retries", "docker api retries", "")
+		if err != nil {
+			logrus.Fatal(err)
+		}
+		v, err := view.New(
+			"docker_api_retries",
+			"number of times we've retried docker API upon failure",
+			[]tag.Key{appKey, pathKey},
+			dockerRetriesMeasure,
+			view.SumAggregation{},
+		)
+		if err != nil {
+			logrus.Fatalf("cannot create view: %v", err)
+		}
+		if err := v.Subscribe(); err != nil {
+			logrus.Fatal(err)
+		}
+	}
+
+	{
+		dockerTimeoutMeasure, err = stats.Int64("docker_api_timeout", "docker api timeouts", "")
+		if err != nil {
+			logrus.Fatal(err)
+		}
+		v, err := view.New(
+			"docker_api_timeout_count",
+			"number of times we've timed out calling docker API",
+			[]tag.Key{appKey, pathKey},
+			dockerTimeoutMeasure,
+			view.CountAggregation{},
+		)
+		if err != nil {
+			logrus.Fatalf("cannot create view: %v", err)
+		}
+		if err := v.Subscribe(); err != nil {
+			logrus.Fatal(err)
+		}
+	}
+
+	{
+		dockerErrorMeasure, err = stats.Int64("docker_api_error", "docker api errors", "")
+		if err != nil {
+			logrus.Fatal(err)
+		}
+		v, err := view.New(
+			"docker_api_error_count",
+			"number of unrecoverable errors from docker API",
+			[]tag.Key{appKey, pathKey},
+			dockerErrorMeasure,
+			view.CountAggregation{},
+		)
+		if err != nil {
+			logrus.Fatalf("cannot create view: %v", err)
+		}
+		if err := v.Subscribe(); err != nil {
+			logrus.Fatal(err)
+		}
+	}
+
+	{
+		dockerOOMMeasure, err = stats.Int64("docker_oom", "docker oom", "")
+		if err != nil {
+			logrus.Fatal(err)
+		}
+		v, err := view.New(
+			"docker_oom_count",
+			"number of docker container oom",
+			[]tag.Key{appKey, pathKey},
+			dockerOOMMeasure,
+			view.CountAggregation{},
+		)
+		if err != nil {
+			logrus.Fatalf("cannot create view: %v", err)
+		}
+		if err := v.Subscribe(); err != nil {
+			logrus.Fatal(err)
+		}
+	}
+}
+
+var (
+	// TODO it's either this or stats.FindMeasure("string").M() -- this is safer but painful
+	dockerRetriesMeasure *stats.Int64Measure
+	dockerTimeoutMeasure *stats.Int64Measure
+	dockerErrorMeasure   *stats.Int64Measure
+	dockerOOMMeasure     *stats.Int64Measure
+)
+
 func (d *dockerWrap) retry(ctx context.Context, logger logrus.FieldLogger, f func() error) error {
 	var i int
 	var err error
-	span := opentracing.SpanFromContext(ctx)
+	defer func() { stats.Record(ctx, dockerRetriesMeasure.M(int64(i))) }()
-	defer func() { span.LogFields(log.Int("docker_call_retries", i)) }()
 
 	var b common.Backoff
 	// 10 retries w/o change to backoff is ~13s if ops take ~0 time
 	for ; i < 10; i++ {
 		select {
 		case <-ctx.Done():
-			span.LogFields(log.String("task", "fail.docker"))
+			stats.Record(ctx, dockerTimeoutMeasure.M(1))
 			logger.WithError(ctx.Err()).Warnf("docker call timed out")
 			return ctx.Err()
 		default:
@@ -119,11 +226,10 @@ func (d *dockerWrap) retry(ctx context.Context, logger logrus.FieldLogger, f fun
 		if common.IsTemporary(err) || isDocker50x(err) {
 			logger.WithError(err).Warn("docker temporary error, retrying")
 			b.Sleep(ctx)
-			span.LogFields(log.String("task", "tmperror.docker"))
 			continue
 		}
 		if err != nil {
-			span.LogFields(log.String("task", "error.docker"))
+			stats.Record(ctx, dockerErrorMeasure.M(1))
 		}
 		return err
 	}
@@ -176,22 +282,17 @@ func filterNoSuchContainer(ctx context.Context, err error) error {
 }
 
 func (d *dockerWrap) Info(ctx context.Context) (info *docker.DockerInfo, err error) {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "docker_server_version")
+	// NOTE: we're not very responsible and prometheus wasn't loved as a child, this
-	defer span.Finish()
+	// threads through directly down to the docker call, skipping retires, so that we
-
+	// don't have to add tags / tracing / logger to the bare context handed to the one
-	logger := common.Logger(ctx).WithField("docker_cmd", "DockerInfo")
+	// place this is called in initialization that has no context to report consistent
-	ctx, cancel := context.WithTimeout(ctx, retryTimeout)
+	// stats like everything else in here. tl;dr this works, just don't use it for anything else.
-	defer cancel()
+	return d.docker.Info()
-	err = d.retry(ctx, logger, func() error {
-		info, err = d.docker.Info()
-		return err
-	})
-	return info, err
 }
 
 func (d *dockerWrap) AttachToContainerNonBlocking(ctx context.Context, opts docker.AttachToContainerOptions) (w docker.CloseWaiter, err error) {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "docker_attach_container")
+	ctx, span := trace.StartSpan(ctx, "docker_attach_container")
-	defer span.Finish()
+	defer span.End()
 
 	logger := common.Logger(ctx).WithField("docker_cmd", "AttachContainer")
 	ctx, cancel := context.WithTimeout(ctx, retryTimeout)
@@ -208,8 +309,8 @@ func (d *dockerWrap) AttachToContainerNonBlocking(ctx context.Context, opts dock
 }
 
 func (d *dockerWrap) WaitContainerWithContext(id string, ctx context.Context) (code int, err error) {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "docker_wait_container")
+	ctx, span := trace.StartSpan(ctx, "docker_wait_container")
-	defer span.Finish()
+	defer span.End()
 
 	logger := common.Logger(ctx).WithField("docker_cmd", "WaitContainer")
 	err = d.retry(ctx, logger, func() error {
@@ -220,8 +321,8 @@ func (d *dockerWrap) WaitContainerWithContext(id string, ctx context.Context) (c
 }
 
 func (d *dockerWrap) StartContainerWithContext(id string, hostConfig *docker.HostConfig, ctx context.Context) (err error) {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "docker_start_container")
+	ctx, span := trace.StartSpan(ctx, "docker_start_container")
-	defer span.Finish()
+	defer span.End()
 
 	logger := common.Logger(ctx).WithField("docker_cmd", "StartContainer")
 	err = d.retry(ctx, logger, func() error {
@@ -236,8 +337,8 @@ func (d *dockerWrap) StartContainerWithContext(id string, hostConfig *docker.Hos
 }
 
 func (d *dockerWrap) CreateContainer(opts docker.CreateContainerOptions) (c *docker.Container, err error) {
-	span, ctx := opentracing.StartSpanFromContext(opts.Context, "docker_create_container")
+	ctx, span := trace.StartSpan(opts.Context, "docker_create_container")
-	defer span.Finish()
+	defer span.End()
 
 	logger := common.Logger(ctx).WithField("docker_cmd", "CreateContainer")
 	err = d.retry(ctx, logger, func() error {
@@ -248,8 +349,8 @@ func (d *dockerWrap) CreateContainer(opts docker.CreateContainerOptions) (c *doc
 }
 
 func (d *dockerWrap) PullImage(opts docker.PullImageOptions, auth docker.AuthConfiguration) (err error) {
-	span, ctx := opentracing.StartSpanFromContext(opts.Context, "docker_pull_image")
+	ctx, span := trace.StartSpan(opts.Context, "docker_pull_image")
-	defer span.Finish()
+	defer span.End()
 
 	logger := common.Logger(ctx).WithField("docker_cmd", "PullImage")
 	err = d.retry(ctx, logger, func() error {
@@ -262,9 +363,9 @@ func (d *dockerWrap) PullImage(opts docker.PullImageOptions, auth docker.AuthCon
 func (d *dockerWrap) RemoveContainer(opts docker.RemoveContainerOptions) (err error) {
 	// extract the span, but do not keep the context, since the enclosing context
 	// may be timed out, and we still want to remove the container. TODO in caller? who cares?
-	span, _ := opentracing.StartSpanFromContext(opts.Context, "docker_remove_container")
+	ctx := common.BackgroundContext(opts.Context)
-	defer span.Finish()
+	ctx, span := trace.StartSpan(ctx, "docker_remove_container")
-	ctx := opentracing.ContextWithSpan(context.Background(), span)
+	defer span.End()
 
 	ctx, cancel := context.WithTimeout(ctx, retryTimeout)
 	defer cancel()
@@ -278,8 +379,8 @@ func (d *dockerWrap) RemoveContainer(opts docker.RemoveContainerOptions) (err er
 }
 
 func (d *dockerWrap) PauseContainer(id string, ctx context.Context) (err error) {
-	span, _ := opentracing.StartSpanFromContext(ctx, "docker_pause_container")
+	_, span := trace.StartSpan(ctx, "docker_pause_container")
-	defer span.Finish()
+	defer span.End()
 	ctx, cancel := context.WithTimeout(ctx, pauseTimeout)
 	defer cancel()
 
@@ -292,8 +393,8 @@ func (d *dockerWrap) PauseContainer(id string, ctx context.Context) (err error)
 }
 
 func (d *dockerWrap) UnpauseContainer(id string, ctx context.Context) (err error) {
-	span, _ := opentracing.StartSpanFromContext(ctx, "docker_unpause_container")
+	_, span := trace.StartSpan(ctx, "docker_unpause_container")
-	defer span.Finish()
+	defer span.End()
 	ctx, cancel := context.WithTimeout(ctx, pauseTimeout)
 	defer cancel()
 
@@ -306,8 +407,8 @@ func (d *dockerWrap) UnpauseContainer(id string, ctx context.Context) (err error
 }
 
 func (d *dockerWrap) InspectImage(ctx context.Context, name string) (i *docker.Image, err error) {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "docker_inspect_image")
+	ctx, span := trace.StartSpan(ctx, "docker_inspect_image")
-	defer span.Finish()
+	defer span.End()
 	ctx, cancel := context.WithTimeout(ctx, retryTimeout)
 	defer cancel()
 
@@ -320,8 +421,8 @@ func (d *dockerWrap) InspectImage(ctx context.Context, name string) (i *docker.I
 }
 
 func (d *dockerWrap) InspectContainerWithContext(container string, ctx context.Context) (c *docker.Container, err error) {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "docker_inspect_container")
+	ctx, span := trace.StartSpan(ctx, "docker_inspect_container")
-	defer span.Finish()
+	defer span.End()
 	ctx, cancel := context.WithTimeout(ctx, retryTimeout)
 	defer cancel()
 

api/agent/hybrid/client.go

@@ -17,7 +17,8 @@ import (
 	"github.com/fnproject/fn/api/agent"
 	"github.com/fnproject/fn/api/common"
 	"github.com/fnproject/fn/api/models"
-	opentracing "github.com/opentracing/opentracing-go"
+	"go.opencensus.io/plugin/ochttp/propagation/b3"
+	"go.opencensus.io/trace"
 )
 
 // client implements agent.DataAccess
@@ -66,16 +67,16 @@ func NewClient(u string) (agent.DataAccess, error) {
 }
 
 func (cl *client) Enqueue(ctx context.Context, c *models.Call) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "hybrid_client_enqueue")
+	ctx, span := trace.StartSpan(ctx, "hybrid_client_enqueue")
-	defer span.Finish()
+	defer span.End()
 
 	err := cl.do(ctx, c, nil, "PUT", "runner", "async")
 	return err
 }
 
 func (cl *client) Dequeue(ctx context.Context) (*models.Call, error) {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "hybrid_client_dequeue")
+	ctx, span := trace.StartSpan(ctx, "hybrid_client_dequeue")
-	defer span.Finish()
+	defer span.End()
 
 	var c struct {
 		C []*models.Call `json:"calls"`
@@ -88,16 +89,16 @@ func (cl *client) Dequeue(ctx context.Context) (*models.Call, error) {
 }
 
 func (cl *client) Start(ctx context.Context, c *models.Call) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "hybrid_client_start")
+	ctx, span := trace.StartSpan(ctx, "hybrid_client_start")
-	defer span.Finish()
+	defer span.End()
 
 	err := cl.do(ctx, c, nil, "POST", "runner", "start")
 	return err
 }
 
 func (cl *client) Finish(ctx context.Context, c *models.Call, r io.Reader, async bool) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "hybrid_client_end")
+	ctx, span := trace.StartSpan(ctx, "hybrid_client_end")
-	defer span.Finish()
+	defer span.End()
 
 	var b bytes.Buffer // TODO pool / we should multipart this?
 	_, err := io.Copy(&b, r)
@@ -118,8 +119,8 @@ func (cl *client) Finish(ctx context.Context, c *models.Call, r io.Reader, async
 }
 
 func (cl *client) GetApp(ctx context.Context, appName string) (*models.App, error) {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "hybrid_client_get_app")
+	ctx, span := trace.StartSpan(ctx, "hybrid_client_get_app")
-	defer span.Finish()
+	defer span.End()
 
 	var a struct {
 		A models.App `json:"app"`
@@ -129,8 +130,8 @@ func (cl *client) GetApp(ctx context.Context, appName string) (*models.App, erro
 }
 
 func (cl *client) GetRoute(ctx context.Context, appName, route string) (*models.Route, error) {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "hybrid_client_get_route")
+	ctx, span := trace.StartSpan(ctx, "hybrid_client_get_route")
-	defer span.Finish()
+	defer span.End()
 
 	// TODO trim prefix is pretty odd here eh?
 	var r struct {
@@ -181,8 +182,8 @@ func (cl *client) do(ctx context.Context, request, result interface{}, method st
 }
 
 func (cl *client) once(ctx context.Context, request, result interface{}, method string, url ...string) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "hybrid_client_http_do")
+	ctx, span := trace.StartSpan(ctx, "hybrid_client_http_do")
-	defer span.Finish()
+	defer span.End()
 
 	var b bytes.Buffer // TODO pool
 	if request != nil {
@@ -196,12 +197,9 @@ func (cl *client) once(ctx context.Context, request, result interface{}, method
 	if err != nil {
 		return err
 	}
-	req = req.WithContext(ctx)
 	// shove the span headers in so that the server will continue this span
-	opentracing.GlobalTracer().Inject(
+	var xxx b3.HTTPFormat
-		span.Context(),
+	xxx.SpanContextToRequest(span.SpanContext(), req)
-		opentracing.HTTPHeaders,
-		opentracing.HTTPHeadersCarrier(req.Header))
 
 	resp, err := cl.http.Do(req)
 	if err != nil {

api/agent/prometheus_metrics.go

@@ -1,9 +0,0 @@
-package agent
-
-import (
-	"net/http"
-)
-
-func (a *agent) PromHandler() http.Handler {
-	return a.promHandler
-}

api/agent/protocol/http.go

@@ -7,8 +7,9 @@ import (
 	"io"
 	"net/http"
 
+	"go.opencensus.io/trace"
+
 	"github.com/fnproject/fn/api/models"
-	opentracing "github.com/opentracing/opentracing-go"
 )
 
 // HTTPProtocol converts stdin/stdout streams into HTTP/1.1 compliant
@@ -23,8 +24,8 @@ type HTTPProtocol struct {
 func (p *HTTPProtocol) IsStreamable() bool { return true }
 
 func (h *HTTPProtocol) Dispatch(ctx context.Context, ci CallInfo, w io.Writer) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "dispatch_http")
+	ctx, span := trace.StartSpan(ctx, "dispatch_http")
-	defer span.Finish()
+	defer span.End()
 
 	req := ci.Request()
 
@@ -36,23 +37,23 @@ func (h *HTTPProtocol) Dispatch(ctx context.Context, ci CallInfo, w io.Writer) e
 	req.Header.Set("FN_REQUEST_URL", ci.RequestURL())
 	req.Header.Set("FN_CALL_ID", ci.CallID())
 
-	span, _ = opentracing.StartSpanFromContext(ctx, "dispatch_http_write_request")
+	_, span = trace.StartSpan(ctx, "dispatch_http_write_request")
 	// req.Write handles if the user does not specify content length
 	err := req.Write(h.in)
-	span.Finish()
+	span.End()
 	if err != nil {
 		return err
 	}
 
-	span, _ = opentracing.StartSpanFromContext(ctx, "dispatch_http_read_response")
+	_, span = trace.StartSpan(ctx, "dispatch_http_read_response")
 	resp, err := http.ReadResponse(bufio.NewReader(h.out), ci.Request())
-	span.Finish()
+	span.End()
 	if err != nil {
 		return models.NewAPIError(http.StatusBadGateway, fmt.Errorf("invalid http response from function err: %v", err))
 	}
 
-	span, _ = opentracing.StartSpanFromContext(ctx, "dispatch_http_write_response")
+	_, span = trace.StartSpan(ctx, "dispatch_http_write_response")
-	defer span.Finish()
+	defer span.End()
 
 	rw, ok := w.(http.ResponseWriter)
 	if !ok {

api/agent/protocol/json.go

@@ -9,8 +9,9 @@ import (
 	"net/http"
 	"sync"
 
+	"go.opencensus.io/trace"
+
 	"github.com/fnproject/fn/api/models"
-	opentracing "github.com/opentracing/opentracing-go"
 )
 
 var (
@@ -87,26 +88,26 @@ func (h *JSONProtocol) writeJSONToContainer(ci CallInfo) error {
 }
 
 func (h *JSONProtocol) Dispatch(ctx context.Context, ci CallInfo, w io.Writer) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "dispatch_json")
+	ctx, span := trace.StartSpan(ctx, "dispatch_json")
-	defer span.Finish()
+	defer span.End()
 
-	span, _ = opentracing.StartSpanFromContext(ctx, "dispatch_json_write_request")
+	_, span = trace.StartSpan(ctx, "dispatch_json_write_request")
 	err := h.writeJSONToContainer(ci)
-	span.Finish()
+	span.End()
 	if err != nil {
 		return err
 	}
 
-	span, _ = opentracing.StartSpanFromContext(ctx, "dispatch_json_read_response")
+	_, span = trace.StartSpan(ctx, "dispatch_json_read_response")
 	var jout jsonOut
 	err = json.NewDecoder(h.out).Decode(&jout)
-	span.Finish()
+	span.End()
 	if err != nil {
 		return models.NewAPIError(http.StatusBadGateway, fmt.Errorf("invalid json response from function err: %v", err))
 	}
 
-	span, _ = opentracing.StartSpanFromContext(ctx, "dispatch_json_write_response")
+	_, span = trace.StartSpan(ctx, "dispatch_json_write_response")
-	defer span.Finish()
+	defer span.End()
 
 	rw, ok := w.(http.ResponseWriter)
 	if !ok {

api/agent/resource.go

@@ -13,7 +13,8 @@ import (
 	"strings"
 	"sync"
 
-	opentracing "github.com/opentracing/opentracing-go"
+	"go.opencensus.io/trace"
+
 	"github.com/sirupsen/logrus"
 )
 
@@ -166,9 +167,9 @@ func (a *resourceTracker) GetResourceToken(ctx context.Context, memory uint64, c
 		c.L.Unlock()
 	}()
 
-	span, ctx := opentracing.StartSpanFromContext(ctx, "agent_get_resource_token")
+	ctx, span := trace.StartSpan(ctx, "agent_get_resource_token")
 	go func() {
-		defer span.Finish()
+		defer span.End()
 		defer cancel()
 		c.L.Lock()
 
@@ -254,9 +255,9 @@ func (a *resourceTracker) WaitAsyncResource(ctx context.Context) chan struct{} {
 		c.L.Unlock()
 	}()
 
-	span, ctx := opentracing.StartSpanFromContext(ctx, "agent_wait_async_resource")
+	ctx, span := trace.StartSpan(ctx, "agent_wait_async_resource")
 	go func() {
-		defer span.Finish()
+		defer span.End()
 		defer cancel()
 		c.L.Lock()
 		isWaiting = true

api/agent/state_trackers.go

@@ -5,7 +5,10 @@ import (
 	"sync"
 	"time"
 
-	"github.com/fnproject/fn/api/common"
+	"github.com/sirupsen/logrus"
+	"go.opencensus.io/stats"
+	"go.opencensus.io/stats/view"
+	"go.opencensus.io/tag"
 )
 
 type RequestStateType int
@@ -137,16 +140,76 @@ func (c *containerState) UpdateState(ctx context.Context, newState ContainerStat
 	// update old state stats
 	gaugeKey := containerGaugeKeys[oldState]
 	if gaugeKey != "" {
-		common.DecrementGauge(ctx, gaugeKey)
+		stats.Record(ctx, stats.FindMeasure(gaugeKey).(*stats.Int64Measure).M(-1))
 	}
+
 	timeKey := containerTimeKeys[oldState]
 	if timeKey != "" {
-		common.PublishElapsedTimeHistogram(ctx, timeKey, before, now)
+		stats.Record(ctx, stats.FindMeasure(timeKey).(*stats.Int64Measure).M(int64(now.Sub(before).Round(time.Millisecond))))
 	}
 
 	// update new state stats
 	gaugeKey = containerGaugeKeys[newState]
 	if gaugeKey != "" {
-		common.IncrementGauge(ctx, gaugeKey)
+		stats.Record(ctx, stats.FindMeasure(gaugeKey).(*stats.Int64Measure).M(1))
+	}
+}
+
+func init() {
+	// TODO(reed): do we have to do this? the measurements will be tagged on the context, will they be propagated
+	// or we have to white list them in the view for them to show up? test...
+	appKey, err := tag.NewKey("fn_appname")
+	if err != nil {
+		logrus.Fatal(err)
+	}
+	pathKey, err := tag.NewKey("fn_path")
+	if err != nil {
+		logrus.Fatal(err)
+	}
+
+	for _, key := range containerGaugeKeys {
+		if key == "" { // leave nil intentionally, let it panic
+			continue
+		}
+		measure, err := stats.Int64(key, "containers in state "+key, "")
+		if err != nil {
+			logrus.Fatal(err)
+		}
+		v, err := view.New(
+			key,
+			"containers in state "+key,
+			[]tag.Key{appKey, pathKey},
+			measure,
+			view.CountAggregation{},
+		)
+		if err != nil {
+			logrus.Fatalf("cannot create view: %v", err)
+		}
+		if err := v.Subscribe(); err != nil {
+			logrus.Fatal(err)
+		}
+	}
+
+	for _, key := range containerTimeKeys {
+		if key == "" {
+			continue
+		}
+		measure, err := stats.Int64(key, "time spent in container state "+key, "ms")
+		if err != nil {
+			logrus.Fatal(err)
+		}
+		v, err := view.New(
+			key,
+			"time spent in container state "+key,
+			[]tag.Key{appKey, pathKey},
+			measure,
+			view.DistributionAggregation{},
+		)
+		if err != nil {
+			logrus.Fatalf("cannot create view: %v", err)
+		}
+		if err := v.Subscribe(); err != nil {
+			logrus.Fatal(err)
+		}
 	}
 }

api/agent/stats.go

@@ -2,54 +2,63 @@ package agent
 
 import (
 	"context"
-	"github.com/fnproject/fn/api/common"
+
+	"github.com/sirupsen/logrus"
+	"go.opencensus.io/stats"
+	"go.opencensus.io/stats/view"
+	"go.opencensus.io/tag"
 )
 
-func StatsEnqueue(ctx context.Context) {
+// TODO add some suga:
-	common.IncrementGauge(ctx, queuedMetricName)
+// * hot containers active
-	common.IncrementCounter(ctx, callsMetricName)
+// * memory used / available
+
+func statsEnqueue(ctx context.Context) {
+	stats.Record(ctx, queuedMeasure.M(1))
+	stats.Record(ctx, callsMeasure.M(1))
 }
 
 // Call when a function has been queued but cannot be started because of an error
-func StatsDequeue(ctx context.Context) {
+func statsDequeue(ctx context.Context) {
-	common.DecrementGauge(ctx, queuedMetricName)
+	stats.Record(ctx, queuedMeasure.M(-1))
 }
 
-func StatsDequeueAndStart(ctx context.Context) {
+func statsDequeueAndStart(ctx context.Context) {
-	common.DecrementGauge(ctx, queuedMetricName)
+	stats.Record(ctx, queuedMeasure.M(-1))
-	common.IncrementGauge(ctx, runningMetricName)
+	stats.Record(ctx, runningMeasure.M(1))
 }
 
-func StatsComplete(ctx context.Context) {
+func statsComplete(ctx context.Context) {
-	common.DecrementGauge(ctx, runningMetricName)
+	stats.Record(ctx, runningMeasure.M(-1))
-	common.IncrementCounter(ctx, completedMetricName)
+	stats.Record(ctx, completedMeasure.M(1))
 }
 
-func StatsFailed(ctx context.Context) {
+func statsFailed(ctx context.Context) {
-	common.DecrementGauge(ctx, runningMetricName)
+	stats.Record(ctx, runningMeasure.M(-1))
-	common.IncrementCounter(ctx, failedMetricName)
+	stats.Record(ctx, failedMeasure.M(1))
 }
 
-func StatsDequeueAndFail(ctx context.Context) {
+func statsDequeueAndFail(ctx context.Context) {
-	common.DecrementGauge(ctx, queuedMetricName)
+	stats.Record(ctx, queuedMeasure.M(-1))
-	common.IncrementCounter(ctx, failedMetricName)
+	stats.Record(ctx, failedMeasure.M(1))
 }
 
-func StatsIncrementTimedout(ctx context.Context) {
+func statsTimedout(ctx context.Context) {
-	common.IncrementCounter(ctx, timedoutMetricName)
+	stats.Record(ctx, timedoutMeasure.M(1))
 }
 
-func StatsIncrementErrors(ctx context.Context) {
+func statsErrors(ctx context.Context) {
-	common.IncrementCounter(ctx, errorsMetricName)
+	stats.Record(ctx, errorsMeasure.M(1))
 }
 
-func StatsIncrementTooBusy(ctx context.Context) {
+func statsTooBusy(ctx context.Context) {
-	common.IncrementCounter(ctx, serverBusyMetricName)
+	stats.Record(ctx, serverBusyMeasure.M(1))
 }
 
 const (
+	// TODO we should probably prefix these with calls_ ?
 	queuedMetricName     = "queued"
-	callsMetricName      = "calls"
+	callsMetricName      = "calls" // TODO this is a dupe of sum {complete,failed} ?
 	runningMetricName    = "running"
 	completedMetricName  = "completed"
 	failedMetricName     = "failed"
@@ -57,3 +66,192 @@ const (
 	errorsMetricName     = "errors"
 	serverBusyMetricName = "server_busy"
 )
+
+var (
+	queuedMeasure     *stats.Int64Measure
+	callsMeasure      *stats.Int64Measure // TODO this is a dupe of sum {complete,failed} ?
+	runningMeasure    *stats.Int64Measure
+	completedMeasure  *stats.Int64Measure
+	failedMeasure     *stats.Int64Measure
+	timedoutMeasure   *stats.Int64Measure
+	errorsMeasure     *stats.Int64Measure
+	serverBusyMeasure *stats.Int64Measure
+)
+
+func init() {
+	// TODO(reed): doing this at each call site seems not the intention of the library since measurements
+	// need to be created and views registered. doing this up front seems painful but maybe there
+	// are benefits?
+
+	// TODO(reed): do we have to do this? the measurements will be tagged on the context, will they be propagated
+	// or we have to white list them in the view for them to show up? test...
+	var err error
+	appKey, err := tag.NewKey("fn_appname")
+	if err != nil {
+		logrus.Fatal(err)
+	}
+	pathKey, err := tag.NewKey("fn_path")
+	if err != nil {
+		logrus.Fatal(err)
+	}
+
+	{
+		queuedMeasure, err = stats.Int64(queuedMetricName, "calls currently queued against agent", "")
+		if err != nil {
+			logrus.Fatal(err)
+		}
+		v, err := view.New(
+			queuedMetricName,
+			"calls currently queued to agent",
+			[]tag.Key{appKey, pathKey},
+			queuedMeasure,
+			view.SumAggregation{},
+		)
+		if err != nil {
+			logrus.Fatalf("cannot create view: %v", err)
+		}
+		if err := v.Subscribe(); err != nil {
+			logrus.Fatal(err)
+		}
+	}
+
+	{
+		callsMeasure, err = stats.Int64(callsMetricName, "calls created in agent", "")
+		if err != nil {
+			logrus.Fatal(err)
+		}
+		v, err := view.New(
+			callsMetricName,
+			"calls created in agent",
+			[]tag.Key{appKey, pathKey},
+			callsMeasure,
+			view.SumAggregation{},
+		)
+		if err != nil {
+			logrus.Fatalf("cannot create view: %v", err)
+		}
+		if err := v.Subscribe(); err != nil {
+			logrus.Fatal(err)
+		}
+	}
+
+	{
+		runningMeasure, err = stats.Int64(runningMetricName, "calls currently running in agent", "")
+		if err != nil {
+			logrus.Fatal(err)
+		}
+		v, err := view.New(
+			runningMetricName,
+			"calls currently running in agent",
+			[]tag.Key{appKey, pathKey},
+			runningMeasure,
+			view.SumAggregation{},
+		)
+		if err != nil {
+			logrus.Fatalf("cannot create view: %v", err)
+		}
+		if err := v.Subscribe(); err != nil {
+			logrus.Fatal(err)
+		}
+	}
+
+	{
+		completedMeasure, err = stats.Int64(completedMetricName, "calls completed in agent", "")
+		if err != nil {
+			logrus.Fatal(err)
+		}
+		v, err := view.New(
+			completedMetricName,
+			"calls completed in agent",
+			[]tag.Key{appKey, pathKey},
+			completedMeasure,
+			view.SumAggregation{},
+		)
+		if err != nil {
+			logrus.Fatalf("cannot create view: %v", err)
+		}
+		if err := v.Subscribe(); err != nil {
+			logrus.Fatal(err)
+		}
+	}
+
+	{
+		failedMeasure, err = stats.Int64(failedMetricName, "calls failed in agent", "")
+		if err != nil {
+			logrus.Fatal(err)
+		}
+		v, err := view.New(
+			failedMetricName,
+			"calls failed in agent",
+			[]tag.Key{appKey, pathKey},
+			failedMeasure,
+			view.SumAggregation{},
+		)
+		if err != nil {
+			logrus.Fatalf("cannot create view: %v", err)
+		}
+		if err := v.Subscribe(); err != nil {
+			logrus.Fatal(err)
+		}
+	}
+
+	{
+		timedoutMeasure, err = stats.Int64(timedoutMetricName, "calls timed out in agent", "")
+		if err != nil {
+			logrus.Fatal(err)
+		}
+		v, err := view.New(
+			timedoutMetricName,
+			"calls timed out in agent",
+			[]tag.Key{appKey, pathKey},
+			timedoutMeasure,
+			view.SumAggregation{},
+		)
+		if err != nil {
+			logrus.Fatalf("cannot create view: %v", err)
+		}
+		if err := v.Subscribe(); err != nil {
+			logrus.Fatal(err)
+		}
+	}
+
+	{
+		errorsMeasure, err = stats.Int64(errorsMetricName, "calls errored in agent", "")
+		if err != nil {
+			logrus.Fatal(err)
+		}
+		v, err := view.New(
+			errorsMetricName,
+			"calls errored in agent",
+			[]tag.Key{appKey, pathKey},
+			errorsMeasure,
+			view.SumAggregation{},
+		)
+		if err != nil {
+			logrus.Fatalf("cannot create view: %v", err)
+		}
+		if err := v.Subscribe(); err != nil {
+			logrus.Fatal(err)
+		}
+	}
+
+	{
+		serverBusyMeasure, err = stats.Int64(serverBusyMetricName, "calls where server was too busy in agent", "")
+		if err != nil {
+			logrus.Fatal(err)
+		}
+		v, err := view.New(
+			serverBusyMetricName,
+			"calls where server was too busy in agent",
+			[]tag.Key{appKey, pathKey},
+			serverBusyMeasure,
+			view.SumAggregation{},
+		)
+		if err != nil {
+			logrus.Fatalf("cannot create view: %v", err)
+		}
+		if err := v.Subscribe(); err != nil {
+			logrus.Fatal(err)
+		}
+	}
+}

api/common/ctx.go

@@ -4,6 +4,8 @@ import (
 	"context"
 
 	"github.com/sirupsen/logrus"
+	"go.opencensus.io/tag"
+	"go.opencensus.io/trace"
 )
 
 type contextKey string
@@ -22,11 +24,35 @@ func Logger(ctx context.Context) logrus.FieldLogger {
 	return l
 }
 
-// Attempt at simplifying this whole logger in the context thing
+// LoggerWithFields returns a child context of the provided parent that
-// Could even make this take a generic map, then the logger that gets returned could be used just like the stdlib too, since it's compatible
+// contains a logger with additional fields from the parent's logger, it
+// returns the new child logger, as well.
 func LoggerWithFields(ctx context.Context, fields logrus.Fields) (context.Context, logrus.FieldLogger) {
 	l := Logger(ctx)
 	l = l.WithFields(fields)
 	ctx = WithLogger(ctx, l)
 	return ctx, l
 }
+
+// BackgroundContext returns a context that is specifically not a child of the
+// provided parent context wrt any cancellation or deadline of the parent,
+// returning a context that contains all values only. At present, this is a
+// best effort as there is not a great way to extract all values, known values:
+// * logger
+// * span
+// * tags
+// (TODO(reed): we could have our own context.Context implementer that stores
+// all values from WithValue in a bucket we could extract more easily?)
+func BackgroundContext(ctx context.Context) context.Context {
+	logger := Logger(ctx)
+	span := trace.FromContext(ctx)
+	tagMap := tag.FromContext(ctx)
+
+	// fresh context
+	ctx = context.Background()
+
+	ctx = tag.NewContext(ctx, tagMap)
+	ctx = trace.WithSpan(ctx, span)
+	ctx = WithLogger(ctx, logger)
+	return ctx
+}

api/common/metrics.go

@@ -1,140 +0,0 @@
-package common
-
-import (
-	"context"
-	"time"
-
-	"github.com/opentracing/opentracing-go"
-	"github.com/opentracing/opentracing-go/log"
-)
-
-// IncrementGauge increments the specified gauge metric
-// It does this by logging an appropriate field value to a tracing span.
-func IncrementGauge(ctx context.Context, metric string) {
-	// The field name we use is the specified metric name prepended with FieldnamePrefixGauge to designate that it is a Prometheus gauge metric
-	// The collector will replace that prefix with "fn_" and use the result as the Prometheus metric name.
-	fieldname := FieldnamePrefixGauge + metric
-
-	// Spans are not processed by the collector until the span ends, so to prevent any delay
-	// in processing the stats when the current span is long-lived we create a new span for every call
-	// suffix the span name with SpannameSuffixDummy to denote that it is used only to hold a metric and isn't itself of any interest
-	span, ctx := opentracing.StartSpanFromContext(ctx, fieldname+SpannameSuffixDummy)
-	defer span.Finish()
-
-	// gauge metrics are actually float64; here we log that it should be increased by +1
-	span.LogFields(log.Float64(fieldname, 1.))
-}
-
-// DecrementGauge decrements the specified gauge metric
-// It does this by logging an appropriate field value to a tracing span.
-func DecrementGauge(ctx context.Context, metric string) {
-	// The field name we use is the specified metric name prepended with FieldnamePrefixGauge to designate that it is a Prometheus gauge metric
-	// The collector will replace that prefix with "fn_" and use the result as the Prometheus metric name.
-	fieldname := FieldnamePrefixGauge + metric
-
-	// Spans are not processed by the collector until the span ends, so to prevent any delay
-	// in processing the stats when the current span is long-lived we create a new span for every call.
-	// suffix the span name with SpannameSuffixDummy to denote that it is used only to hold a metric and isn't itself of any interest
-	span, ctx := opentracing.StartSpanFromContext(ctx, fieldname+SpannameSuffixDummy)
-	defer span.Finish()
-
-	// gauge metrics are actually float64; here we log that it should be increased by -1
-	span.LogFields(log.Float64(fieldname, -1.))
-}
-
-// IncrementCounter increments the specified counter metric
-// It does this by logging an appropriate field value to a tracing span.
-func IncrementCounter(ctx context.Context, metric string) {
-	// The field name we use is the specified metric name prepended with FieldnamePrefixCounter to designate that it is a Prometheus counter metric
-	// The collector will replace that prefix with "fn_" and use the result as the Prometheus metric name.
-	fieldname := FieldnamePrefixCounter + metric
-
-	// Spans are not processed by the collector until the span ends, so to prevent any delay
-	// in processing the stats when the current span is long-lived we create a new span for every call.
-	// suffix the span name with SpannameSuffixDummy to denote that it is used only to hold a metric and isn't itself of any interest
-	span, ctx := opentracing.StartSpanFromContext(ctx, fieldname+SpannameSuffixDummy)
-	defer span.Finish()
-
-	// counter metrics are actually float64; here we log that it should be increased by +1
-	span.LogFields(log.Float64(fieldname, 1.))
-}
-
-// If required, create a scalar version of PublishHistograms that publishes a single histogram metric
-
-// PublishHistograms publishes the specifed histogram metrics
-// It does this by logging appropriate field values to a tracing span
-// Use this when the current tracing span is long-lived and you want the metric to be visible before it ends
-func PublishHistograms(ctx context.Context, metrics map[string]float64) {
-
-	// Spans are not processed by the collector until the span ends, so to prevent any delay
-	// in processing the stats when the current span is long-lived we create a new span for every call.
-	// suffix the span name with SpannameSuffixDummy to denote that it is used only to hold a metric and isn't itself of any interest
-	span, ctx := opentracing.StartSpanFromContext(ctx, "histogram_metrics"+SpannameSuffixDummy)
-	defer span.Finish()
-
-	for key, value := range metrics {
-		// The field name we use is the metric name prepended with FieldnamePrefixHistogram to designate that it is a Prometheus histogram metric
-		// The collector will replace that prefix with "fn_" and use the result as the Prometheus metric name.
-		fieldname := FieldnamePrefixHistogram + key
-		span.LogFields(log.Float64(fieldname, value))
-	}
-}
-
-// PublishHistogram publishes the specifed histogram metric
-// It does this by logging an appropriate field value to a tracing span
-// Use this when the current tracing span is long-lived and you want the metric to be visible before it ends
-func PublishHistogram(ctx context.Context, key string, value float64) {
-
-	// Spans are not processed by the collector until the span ends, so to prevent any delay
-	// in processing the stats when the current span is long-lived we create a new span for every call.
-	// suffix the span name with SpannameSuffixDummy to denote that it is used only to hold a metric and isn't itself of any interest
-	span, ctx := opentracing.StartSpanFromContext(ctx, "histogram_metrics"+SpannameSuffixDummy)
-	defer span.Finish()
-
-	// The field name we use is the metric name prepended with FieldnamePrefixHistogram to designate that it is a Prometheus histogram metric
-	// The collector will replace that prefix with "fn_" and use the result as the Prometheus metric name.
-	fieldname := FieldnamePrefixHistogram + key
-	span.LogFields(log.Float64(fieldname, value))
-}
-
-// PublishHistogramToSpan publishes the specifed histogram metric
-// It does this by logging an appropriate field value to the specified tracing span
-// Use this when you don't need to create a new tracing span
-func PublishHistogramToSpan(span opentracing.Span, key string, value float64) {
-
-	// The field name we use is the metric name prepended with FieldnamePrefixHistogram to designate that it is a Prometheus histogram metric
-	// The collector will replace that prefix with "fn_" and use the result as the Prometheus metric name.
-	fieldname := FieldnamePrefixHistogram + key
-	span.LogFields(log.Float64(fieldname, value))
-}
-
-// PublishElapsedTimeToSpan publishes the specifed histogram elapsed time since start
-// It does this by logging an appropriate field value to a tracing span
-// Use this when the current tracing span is long-lived and you want the metric to be visible before it ends
-func PublishElapsedTimeHistogram(ctx context.Context, key string, start, end time.Time) {
-	elapsed := float64(end.Sub(start).Seconds())
-	PublishHistogram(ctx, key, elapsed)
-}
-
-const (
-
-	// FnPrefix is a constant for "fn_", used as a prefix for span names, field names, Prometheus metric names and Prometheus label names
-	FnPrefix = "fn_"
-
-	// FieldnamePrefixHistogram is prefixed to the name of a logged field
-	// to denote that it corresponds to a histogram metric
-	FieldnamePrefixHistogram = FnPrefix + "histogram_"
-
-	// FieldnamePrefixCounter is prefixed to the name of a logged field
-	// to denote that it corresponds to a counter metric
-	FieldnamePrefixCounter = FnPrefix + "counter_"
-
-	// FieldnamePrefixGauge is prefixed to the name of a logged field
-	// to denote that it corresponds to a gauge metric
-	FieldnamePrefixGauge = FnPrefix + "gauge_"
-
-	// SpannameSuffixDummy is suffixed to the name of a tracing span
-	// to denote that it has been created solely for the purpose of carrying metric values
-	// and is not itself of any interest and should not be converted to a Prometheus duration metric
-	SpannameSuffixDummy = "_dummy"
-)

api/datastore/internal/datastoreutil/metrics.go

@@ -4,9 +4,10 @@ import (
 	"context"
 	"io"
 
+	"go.opencensus.io/trace"
+
 	"github.com/fnproject/fn/api/models"
 	"github.com/jmoiron/sqlx"
-	"github.com/opentracing/opentracing-go"
 )
 
 func MetricDS(ds models.Datastore) models.Datastore {
@@ -18,98 +19,98 @@ type metricds struct {
 }
 
 func (m *metricds) GetApp(ctx context.Context, appName string) (*models.App, error) {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "ds_get_app")
+	ctx, span := trace.StartSpan(ctx, "ds_get_app")
-	defer span.Finish()
+	defer span.End()
 	return m.ds.GetApp(ctx, appName)
 }
 
 func (m *metricds) GetApps(ctx context.Context, filter *models.AppFilter) ([]*models.App, error) {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "ds_get_apps")
+	ctx, span := trace.StartSpan(ctx, "ds_get_apps")
-	defer span.Finish()
+	defer span.End()
 	return m.ds.GetApps(ctx, filter)
 }
 
 func (m *metricds) InsertApp(ctx context.Context, app *models.App) (*models.App, error) {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "ds_insert_app")
+	ctx, span := trace.StartSpan(ctx, "ds_insert_app")
-	defer span.Finish()
+	defer span.End()
 	return m.ds.InsertApp(ctx, app)
 }
 
 func (m *metricds) UpdateApp(ctx context.Context, app *models.App) (*models.App, error) {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "ds_update_app")
+	ctx, span := trace.StartSpan(ctx, "ds_update_app")
-	defer span.Finish()
+	defer span.End()
 	return m.ds.UpdateApp(ctx, app)
 }
 
 func (m *metricds) RemoveApp(ctx context.Context, appName string) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "ds_remove_app")
+	ctx, span := trace.StartSpan(ctx, "ds_remove_app")
-	defer span.Finish()
+	defer span.End()
 	return m.ds.RemoveApp(ctx, appName)
 }
 
 func (m *metricds) GetRoute(ctx context.Context, appName, routePath string) (*models.Route, error) {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "ds_get_route")
+	ctx, span := trace.StartSpan(ctx, "ds_get_route")
-	defer span.Finish()
+	defer span.End()
 	return m.ds.GetRoute(ctx, appName, routePath)
 }
 
 func (m *metricds) GetRoutesByApp(ctx context.Context, appName string, filter *models.RouteFilter) (routes []*models.Route, err error) {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "ds_get_routes_by_app")
+	ctx, span := trace.StartSpan(ctx, "ds_get_routes_by_app")
-	defer span.Finish()
+	defer span.End()
 	return m.ds.GetRoutesByApp(ctx, appName, filter)
 }
 
 func (m *metricds) InsertRoute(ctx context.Context, route *models.Route) (*models.Route, error) {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "ds_insert_route")
+	ctx, span := trace.StartSpan(ctx, "ds_insert_route")
-	defer span.Finish()
+	defer span.End()
 	return m.ds.InsertRoute(ctx, route)
 }
 
 func (m *metricds) UpdateRoute(ctx context.Context, route *models.Route) (*models.Route, error) {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "ds_update_route")
+	ctx, span := trace.StartSpan(ctx, "ds_update_route")
-	defer span.Finish()
+	defer span.End()
 	return m.ds.UpdateRoute(ctx, route)
 }
 
 func (m *metricds) RemoveRoute(ctx context.Context, appName, routePath string) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "ds_remove_route")
+	ctx, span := trace.StartSpan(ctx, "ds_remove_route")
-	defer span.Finish()
+	defer span.End()
 	return m.ds.RemoveRoute(ctx, appName, routePath)
 }
 
 func (m *metricds) InsertCall(ctx context.Context, call *models.Call) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "ds_insert_call")
+	ctx, span := trace.StartSpan(ctx, "ds_insert_call")
-	defer span.Finish()
+	defer span.End()
 	return m.ds.InsertCall(ctx, call)
 }
 
 func (m *metricds) UpdateCall(ctx context.Context, from *models.Call, to *models.Call) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "ds_update_call")
+	ctx, span := trace.StartSpan(ctx, "ds_update_call")
-	defer span.Finish()
+	defer span.End()
 	return m.ds.UpdateCall(ctx, from, to)
 }
 
 func (m *metricds) GetCall(ctx context.Context, appName, callID string) (*models.Call, error) {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "ds_get_call")
+	ctx, span := trace.StartSpan(ctx, "ds_get_call")
-	defer span.Finish()
+	defer span.End()
 	return m.ds.GetCall(ctx, appName, callID)
 }
 
 func (m *metricds) GetCalls(ctx context.Context, filter *models.CallFilter) ([]*models.Call, error) {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "ds_get_calls")
+	ctx, span := trace.StartSpan(ctx, "ds_get_calls")
-	defer span.Finish()
+	defer span.End()
 	return m.ds.GetCalls(ctx, filter)
 }
 
 func (m *metricds) InsertLog(ctx context.Context, appName, callID string, callLog io.Reader) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "ds_insert_log")
+	ctx, span := trace.StartSpan(ctx, "ds_insert_log")
-	defer span.Finish()
+	defer span.End()
 	return m.ds.InsertLog(ctx, appName, callID, callLog)
 }
 
 func (m *metricds) GetLog(ctx context.Context, appName, callID string) (io.Reader, error) {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "ds_get_log")
+	ctx, span := trace.StartSpan(ctx, "ds_get_log")
-	defer span.Finish()
+	defer span.End()
 	return m.ds.GetLog(ctx, appName, callID)
 }
 

api/datastore/sql/sql_test.go

@@ -6,6 +6,7 @@ import (
 	"testing"
 
 	"context"
+
 	"github.com/fnproject/fn/api/datastore/internal/datastoretest"
 	"github.com/fnproject/fn/api/datastore/internal/datastoreutil"
 	"github.com/fnproject/fn/api/models"

api/logs/log.go

@@ -5,6 +5,7 @@ import (
 	"net/url"
 
 	"context"
+
 	"github.com/fnproject/fn/api/common"
 	"github.com/fnproject/fn/api/datastore/sql"
 	"github.com/fnproject/fn/api/logs/s3"

api/logs/log_test.go

@@ -6,6 +6,7 @@ import (
 	"testing"
 
 	"context"
+
 	"github.com/fnproject/fn/api/datastore/sql"
 	logTesting "github.com/fnproject/fn/api/logs/testing"
 )

api/logs/s3/s3.go

@@ -17,10 +17,12 @@ import (
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/aws/aws-sdk-go/service/s3/s3manager"
-	"github.com/fnproject/fn/api/common"
 	"github.com/fnproject/fn/api/models"
-	"github.com/opentracing/opentracing-go"
 	"github.com/sirupsen/logrus"
+	"go.opencensus.io/stats"
+	"go.opencensus.io/stats/view"
+	"go.opencensus.io/tag"
+	"go.opencensus.io/trace"
 )
 
 // TODO we should encrypt these, user will have to supply a key though (or all
@@ -124,8 +126,8 @@ func path(appName, callID string) string {
 }
 
 func (s *store) InsertLog(ctx context.Context, appName, callID string, callLog io.Reader) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "s3_insert_log")
+	ctx, span := trace.StartSpan(ctx, "s3_insert_log")
-	defer span.Finish()
+	defer span.End()
 
 	// wrap original reader in a decorator to keep track of read bytes without buffering
 	cr := &countingReader{r: callLog}
@@ -144,13 +146,13 @@ func (s *store) InsertLog(ctx context.Context, appName, callID string, callLog i
 		return fmt.Errorf("failed to write log, %v", err)
 	}
 
-	common.PublishHistogramToSpan(span, "s3_log_upload_size", float64(cr.count))
+	stats.Record(ctx, uploadSizeMeasure.M(int64(cr.count)))
 	return nil
 }
 
 func (s *store) GetLog(ctx context.Context, appName, callID string) (io.Reader, error) {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "s3_get_log")
+	ctx, span := trace.StartSpan(ctx, "s3_get_log")
-	defer span.Finish()
+	defer span.End()
 
 	objectName := path(appName, callID)
 	logrus.WithFields(logrus.Fields{"bucketName": s.bucket, "key": objectName}).Debug("Downloading log")
@@ -169,6 +171,65 @@ func (s *store) GetLog(ctx context.Context, appName, callID string) (io.Reader,
 		return nil, fmt.Errorf("failed to read log, %v", err)
 	}
 
-	common.PublishHistogramToSpan(span, "s3_log_download_size", float64(size))
+	stats.Record(ctx, downloadSizeMeasure.M(size))
 	return bytes.NewReader(target.Bytes()), nil
 }
+
+var (
+	uploadSizeMeasure   *stats.Int64Measure
+	downloadSizeMeasure *stats.Int64Measure
+)
+
+func init() {
+	// TODO(reed): do we have to do this? the measurements will be tagged on the context, will they be propagated
+	// or we have to white list them in the view for them to show up? test...
+	var err error
+	appKey, err := tag.NewKey("fn_appname")
+	if err != nil {
+		logrus.Fatal(err)
+	}
+	pathKey, err := tag.NewKey("fn_path")
+	if err != nil {
+		logrus.Fatal(err)
+	}
+
+	{
+		uploadSizeMeasure, err = stats.Int64("s3_log_upload_size", "uploaded log size", "byte")
+		if err != nil {
+			logrus.Fatal(err)
+		}
+		v, err := view.New(
+			"s3_log_upload_size",
+			"uploaded log size",
+			[]tag.Key{appKey, pathKey},
+			uploadSizeMeasure,
+			view.DistributionAggregation{},
+		)
+		if err != nil {
+			logrus.Fatalf("cannot create view: %v", err)
+		}
+		if err := v.Subscribe(); err != nil {
+			logrus.Fatal(err)
+		}
+	}
+
+	{
+		downloadSizeMeasure, err = stats.Int64("s3_log_download_size", "downloaded log size", "byte")
+		if err != nil {
+			logrus.Fatal(err)
+		}
+		v, err := view.New(
+			"s3_log_download_size",
+			"downloaded log size",
+			[]tag.Key{appKey, pathKey},
+			downloadSizeMeasure,
+			view.DistributionAggregation{},
+		)
+		if err != nil {
+			logrus.Fatalf("cannot create view: %v", err)
+		}
+		if err := v.Subscribe(); err != nil {
+			logrus.Fatal(err)
+		}
+	}
+}

api/mqs/new.go

@@ -5,8 +5,9 @@ import (
 	"fmt"
 	"net/url"
 
+	"go.opencensus.io/trace"
+
 	"github.com/fnproject/fn/api/models"
-	"github.com/opentracing/opentracing-go"
 	"github.com/sirupsen/logrus"
 )
 
@@ -43,19 +44,19 @@ type metricMQ struct {
 }
 
 func (m *metricMQ) Push(ctx context.Context, t *models.Call) (*models.Call, error) {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "mq_push")
+	ctx, span := trace.StartSpan(ctx, "mq_push")
-	defer span.Finish()
+	defer span.End()
 	return m.mq.Push(ctx, t)
 }
 
 func (m *metricMQ) Reserve(ctx context.Context) (*models.Call, error) {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "mq_reserve")
+	ctx, span := trace.StartSpan(ctx, "mq_reserve")
-	defer span.Finish()
+	defer span.End()
 	return m.mq.Reserve(ctx)
 }
 
 func (m *metricMQ) Delete(ctx context.Context, t *models.Call) error {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "mq_delete")
+	ctx, span := trace.StartSpan(ctx, "mq_delete")
-	defer span.Finish()
+	defer span.End()
 	return m.mq.Delete(ctx, t)
 }

api/server/call_logs.go

@@ -6,10 +6,11 @@ import (
 	"net/http"
 
 	"errors"
+	"strings"
+
 	"github.com/fnproject/fn/api"
 	"github.com/fnproject/fn/api/models"
 	"github.com/gin-gonic/gin"
-	"strings"
 )
 
 // note: for backward compatibility, will go away later

api/server/fntracer.go

@@ -1,60 +0,0 @@
-package server
-
-import (
-	"github.com/opentracing/opentracing-go"
-	"strings"
-)
-
-// FnTracer is a custom Tracer which wraps another another tracer
-// its main purpose is to wrap the underlying Span in a FnSpan,
-// which adds some extra behaviour required for sending tracing spans to prometheus
-type FnTracer struct {
-	opentracing.Tracer
-}
-
-// NewFnTracer returns a new FnTracer which wraps the specified Tracer
-func NewFnTracer(t opentracing.Tracer) opentracing.Tracer {
-	return &FnTracer{t}
-}
-
-// FnTracer implements opentracing.Tracer
-// Override StartSpan to wrap the returned Span in a FnSpan
-func (fnt FnTracer) StartSpan(operationName string, opts ...opentracing.StartSpanOption) opentracing.Span {
-	return NewFnSpan(fnt.Tracer.StartSpan(operationName, opts...))
-}
-
-// FnSpan is a custom Span that wraps another span
-// which adds some extra behaviour required for sending tracing spans to prometheus
-type FnSpan struct {
-	opentracing.Span
-}
-
-// NewFnSpan returns a new FnSpan which wraps the specified Span
-func NewFnSpan(s opentracing.Span) opentracing.Span {
-	return &FnSpan{s}
-}
-
-// FnSpan implements opentracing.Span
-func (fns FnSpan) Finish() {
-	fns.copyBaggageItemsToTags()
-	fns.Span.Finish()
-}
-
-// FnSpan implements opentracing.Span
-func (fns FnSpan) FinishWithOptions(opts opentracing.FinishOptions) {
-	fns.copyBaggageItemsToTags()
-	fns.Span.FinishWithOptions(opts)
-}
-
-func (fns FnSpan) copyBaggageItemsToTags() {
-	// copy baggage items (which are inherited from the parent) with keys starting with "fn" to tags
-	// the PrometheusCollector will send these to Prometheus
-	// need to do this because the collector can't access baggage items, but it can access tags
-	// whereas here we can access the parent's baggage items, but not its tags
-	fns.Context().ForeachBaggageItem(func(k, v string) bool {
-		if strings.HasPrefix(k, "fn") {
-			fns.SetTag(k, v)
-		}
-		return true
-	})
-}

api/server/gin_middlewares.go

@@ -13,9 +13,9 @@ import (
 	"github.com/fnproject/fn/fnext"
 	"github.com/gin-contrib/cors"
 	"github.com/gin-gonic/gin"
-	opentracing "github.com/opentracing/opentracing-go"
-	"github.com/opentracing/opentracing-go/ext"
 	"github.com/sirupsen/logrus"
+	"go.opencensus.io/tag"
+	"go.opencensus.io/trace"
 )
 
 func optionalCorsWrap(r *gin.Engine) {
@@ -37,20 +37,28 @@ func optionalCorsWrap(r *gin.Engine) {
 
 // we should use http grr
 func traceWrap(c *gin.Context) {
-	// try to grab a span from the request if made from another service, ignore err if not
+	appKey, err := tag.NewKey("fn_appname")
-	wireContext, _ := opentracing.GlobalTracer().Extract(
+	if err != nil {
-		opentracing.HTTPHeaders,
+		logrus.Fatal(err)
-		opentracing.HTTPHeadersCarrier(c.Request.Header))
+	}
+	pathKey, err := tag.NewKey("fn_path")
+	if err != nil {
+		logrus.Fatal(err)
+	}
+	ctx, err := tag.New(c.Request.Context(),
+		tag.Insert(appKey, c.Param(api.CApp)),
+		tag.Insert(pathKey, c.Param(api.CRoute)),
+	)
+	if err != nil {
+		logrus.Fatal(err)
+	}
 
-	// Create the span referring to the RPC client if available.
+	// TODO inspect opencensus more and see if we need to define a header ourselves
-	// If wireContext == nil, a root span will be created.
+	// to trigger per-request spans (we will want this), we can set sampler here per request.
-	// TODO we should add more tags?
+
-	serverSpan := opentracing.StartSpan("serve_http", ext.RPCServerOption(wireContext), opentracing.Tag{Key: "path", Value: c.Request.URL.Path})
+	ctx, serverSpan := trace.StartSpan(ctx, "serve_http")
-	serverSpan.SetBaggageItem("fn_appname", c.Param(api.CApp))
+	defer serverSpan.End()
-	serverSpan.SetBaggageItem("fn_path", c.Param(api.CRoute))
-	defer serverSpan.Finish()
 
-	ctx := opentracing.ContextWithSpan(c.Request.Context(), serverSpan)
 	c.Request = c.Request.WithContext(ctx)
 	c.Next()
 }

api/server/prom_zip_collector.go

@@ -1,327 +0,0 @@
-package server
-
-import (
-	"github.com/fnproject/fn/api/common"
-	"github.com/openzipkin/zipkin-go-opentracing"
-	"github.com/openzipkin/zipkin-go-opentracing/thrift/gen-go/zipkincore"
-	"github.com/prometheus/client_golang/prometheus"
-	"strconv"
-	"strings"
-	"sync"
-	"time"
-)
-
-// PrometheusCollector is a custom Collector
-// which sends ZipKin traces to Prometheus
-type PrometheusCollector struct {
-	lock sync.Mutex
-	// Each span name is published as a separate Histogram metric
-	// Using metric names of the form fn_span_<span-name>_duration_seconds
-
-	// In this map, the key is the name of a tracing span,
-	// and the corresponding value is a HistogramVec metric used to report the duration of spans with this name to Prometheus
-	histogramVecMap map[string]*prometheus.HistogramVec
-
-	// In this map, the key is the name of a tracing span,
-	// and the corresponding value is a CounterVec metric used to report the duration of spans with this name to Prometheus
-	counterVecMap map[string]*prometheus.CounterVec
-
-	// In this map, the key is the name of a tracing span,
-	// and the corresponding value is a GaugeVec metric used to report the duration of spans with this name to Prometheus
-	gaugeVecMap map[string]*prometheus.GaugeVec
-
-	// In this map, the key is the name of a tracing span,
-	// and the corresponding value is an array containing the label keys that were specified when the HistogramVec metric was created
-	registeredLabelKeysMap map[string][]string
-}
-
-// NewPrometheusCollector returns a new PrometheusCollector
-func NewPrometheusCollector() (zipkintracer.Collector, error) {
-	pc := &PrometheusCollector{
-		histogramVecMap:        make(map[string]*prometheus.HistogramVec),
-		counterVecMap:          make(map[string]*prometheus.CounterVec),
-		gaugeVecMap:            make(map[string]*prometheus.GaugeVec),
-		registeredLabelKeysMap: make(map[string][]string),
-	}
-	return pc, nil
-}
-
-// PrometheusCollector implements Collector.
-func (pc *PrometheusCollector) Collect(span *zipkincore.Span) error {
-
-	spanName := span.GetName()
-
-	// extract any label values from the span
-	labelKeysFromSpan, labelValuesFromSpan := getLabels(span)
-
-	// report the duration of this span as a histogram
-	// (unless the span name ends with SpannameSuffixDummy to denote it as being purely the carrier of a metric value and so of no interest in itself)
-	if !strings.HasSuffix(spanName, common.SpannameSuffixDummy) {
-
-		// get the HistogramVec for this span name
-		histogramVec, labelValuesToUse := pc.getHistogramVec(
-			("fn_span_" + spanName + "_duration_seconds"), ("Span " + spanName + " duration, by span name"), labelKeysFromSpan, labelValuesFromSpan)
-
-		// now report the span duration value
-		histogramVec.With(labelValuesToUse).Observe((time.Duration(span.GetDuration()) * time.Microsecond).Seconds())
-
-	}
-
-	// now extract any logged histogram metric values from the span
-	for key, value := range getLoggedHistogramMetrics(span) {
-
-		// get the HistogramVec for this metric
-		thisMetricHistogramVec, labelValuesToUse := pc.getHistogramVec(
-			key, ("Metric " + key), labelKeysFromSpan, labelValuesFromSpan)
-
-		// now report the metric value
-		thisMetricHistogramVec.With(labelValuesToUse).Observe(value)
-	}
-
-	// now extract any logged counter metric values from the span
-	for key, value := range getLoggedCounterMetrics(span) {
-
-		// get the CounterVec for this metric
-		thisMetricCounterVec, labelValuesToUse := pc.getCounterVec(
-			key, ("Metric " + key), labelKeysFromSpan, labelValuesFromSpan)
-
-		// now report the metric value
-		thisMetricCounterVec.With(labelValuesToUse).Add(value)
-	}
-
-	// now extract any logged gauge metric values from the span
-	for key, value := range getLoggedGaugeMetrics(span) {
-
-		// get the GaugeVec for this metric
-		thisMetricGaugeVec, labelValuesToUse := pc.getGaugeVec(
-			key, ("Metric " + key), labelKeysFromSpan, labelValuesFromSpan)
-
-		// now report the metric value
-		thisMetricGaugeVec.With(labelValuesToUse).Add(value)
-
-	}
-
-	return nil
-}
-
-// Return (and create, if necessary) a HistogramVec for the specified Prometheus metric
-func (pc *PrometheusCollector) getHistogramVec(
-	metricName string, metricHelp string, labelKeysFromSpan []string, labelValuesFromSpan map[string]string) (
-	*prometheus.HistogramVec, map[string]string) {
-
-	var labelValuesToUse map[string]string
-
-	pc.lock.Lock()
-	defer pc.lock.Unlock()
-
-	histogramVec, found := pc.histogramVecMap[metricName]
-	if !found {
-		// create a new HistogramVec
-		histogramVec = prometheus.NewHistogramVec(
-			prometheus.HistogramOpts{
-				Name: metricName,
-				Help: metricHelp,
-			},
-			labelKeysFromSpan,
-		)
-		pc.histogramVecMap[metricName] = histogramVec
-		pc.registeredLabelKeysMap[metricName] = labelKeysFromSpan
-		prometheus.MustRegister(histogramVec)
-		labelValuesToUse = labelValuesFromSpan
-	} else {
-		// found an existing HistogramVec
-		// need to be careful here, since we must supply the same label keys as when we first created the metric
-		// otherwise we will get a "inconsistent label cardinality" panic
-		// that's why we saved the original label keys in the registeredLabelKeysMap map
-		// so we can use that to construct a map of label key/value pairs to set on the metric
-		labelValuesToUse = make(map[string]string)
-		for _, thisRegisteredLabelKey := range pc.registeredLabelKeysMap[metricName] {
-			if value, found := labelValuesFromSpan[thisRegisteredLabelKey]; found {
-				labelValuesToUse[thisRegisteredLabelKey] = value
-			} else {
-				labelValuesToUse[thisRegisteredLabelKey] = ""
-			}
-		}
-	}
-	return histogramVec, labelValuesToUse
-}
-
-// Return (and create, if necessary) a CounterVec for the specified Prometheus metric
-func (pc *PrometheusCollector) getCounterVec(
-	metricName string, metricHelp string, labelKeysFromSpan []string, labelValuesFromSpan map[string]string) (
-	*prometheus.CounterVec, map[string]string) {
-
-	var labelValuesToUse map[string]string
-
-	pc.lock.Lock()
-	defer pc.lock.Unlock()
-
-	counterVec, found := pc.counterVecMap[metricName]
-	if !found {
-		// create a new CounterVec
-		counterVec = prometheus.NewCounterVec(
-			prometheus.CounterOpts{
-				Name: metricName,
-				Help: metricHelp,
-			},
-			labelKeysFromSpan,
-		)
-		pc.counterVecMap[metricName] = counterVec
-		pc.registeredLabelKeysMap[metricName] = labelKeysFromSpan
-		prometheus.MustRegister(counterVec)
-		labelValuesToUse = labelValuesFromSpan
-	} else {
-		// found an existing CounterVec
-		// need to be careful here, since we must supply the same label keys as when we first created the metric
-		// otherwise we will get a "inconsistent label cardinality" panic
-		// that's why we saved the original label keys in the registeredLabelKeysMap map
-		// so we can use that to construct a map of label key/value pairs to set on the metric
-		labelValuesToUse = make(map[string]string)
-		for _, thisRegisteredLabelKey := range pc.registeredLabelKeysMap[metricName] {
-			if value, found := labelValuesFromSpan[thisRegisteredLabelKey]; found {
-				labelValuesToUse[thisRegisteredLabelKey] = value
-			} else {
-				labelValuesToUse[thisRegisteredLabelKey] = ""
-			}
-		}
-	}
-	return counterVec, labelValuesToUse
-}
-
-// Return (and create, if necessary) a GaugeVec for the specified Prometheus metric
-func (pc *PrometheusCollector) getGaugeVec(
-	metricName string, metricHelp string, labelKeysFromSpan []string, labelValuesFromSpan map[string]string) (
-	*prometheus.GaugeVec, map[string]string) {
-
-	var labelValuesToUse map[string]string
-
-	pc.lock.Lock()
-	defer pc.lock.Unlock()
-
-	gaugeVec, found := pc.gaugeVecMap[metricName]
-	if !found {
-		// create a new GaugeVec
-		gaugeVec = prometheus.NewGaugeVec(
-			prometheus.GaugeOpts{
-				Name: metricName,
-				Help: metricHelp,
-			},
-			labelKeysFromSpan,
-		)
-		pc.gaugeVecMap[metricName] = gaugeVec
-		pc.registeredLabelKeysMap[metricName] = labelKeysFromSpan
-		prometheus.MustRegister(gaugeVec)
-		labelValuesToUse = labelValuesFromSpan
-	} else {
-		// found an existing GaugeVec
-		// need to be careful here, since we must supply the same label keys as when we first created the metric
-		// otherwise we will get a "inconsistent label cardinality" panic
-		// that's why we saved the original label keys in the registeredLabelKeysMap map
-		// so we can use that to construct a map of label key/value pairs to set on the metric
-		labelValuesToUse = make(map[string]string)
-		for _, thisRegisteredLabelKey := range pc.registeredLabelKeysMap[metricName] {
-			if value, found := labelValuesFromSpan[thisRegisteredLabelKey]; found {
-				labelValuesToUse[thisRegisteredLabelKey] = value
-			} else {
-				labelValuesToUse[thisRegisteredLabelKey] = ""
-			}
-		}
-	}
-	return gaugeVec, labelValuesToUse
-}
-
-// extract from the specified span the key/value pairs that we want to add as labels to the Prometheus metric for this span
-// returns an array of keys, and a map of key-value pairs
-func getLabels(span *zipkincore.Span) ([]string, map[string]string) {
-
-	var keys []string
-	labelMap := make(map[string]string)
-
-	// extract any tags whose key starts with "fn" from the span
-	binaryAnnotations := span.GetBinaryAnnotations()
-	for _, thisBinaryAnnotation := range binaryAnnotations {
-		key := thisBinaryAnnotation.GetKey()
-		if thisBinaryAnnotation.GetAnnotationType() == zipkincore.AnnotationType_STRING && strings.HasPrefix(key, "fn") {
-			keys = append(keys, key)
-			value := string(thisBinaryAnnotation.GetValue()[:])
-			labelMap[key] = value
-		}
-	}
-
-	return keys, labelMap
-}
-
-// extract from the span the logged histogram metric values.
-// These are the ones whose names start with FieldnamePrefixHistogram,
-// and whose values we assume are float64
-func getLoggedHistogramMetrics(span *zipkincore.Span) map[string]float64 {
-
-	keyValueMap := make(map[string]float64)
-
-	// extract any annotations whose Value starts with FieldnamePrefixHistogram
-	annotations := span.GetAnnotations()
-	for _, thisAnnotation := range annotations {
-		if strings.HasPrefix(thisAnnotation.GetValue(), common.FieldnamePrefixHistogram) {
-			keyvalue := strings.Split(thisAnnotation.GetValue(), "=")
-			if len(keyvalue) == 2 {
-				if value, err := strconv.ParseFloat(keyvalue[1], 64); err == nil {
-					key := strings.TrimSpace(keyvalue[0])
-					key = common.FnPrefix + key[len(common.FieldnamePrefixHistogram):] // strip off fieldname prefix and then prepend "fn_" to the front
-					keyValueMap[key] = value
-				}
-			}
-		}
-	}
-	return keyValueMap
-}
-
-// extract from the span the logged counter metric values.
-// These are the ones whose names start with FieldnamePrefixCounter,
-// and whose values we assume are float64
-func getLoggedCounterMetrics(span *zipkincore.Span) map[string]float64 {
-
-	keyValueMap := make(map[string]float64)
-
-	// extract any annotations whose Value starts with FieldnamePrefixCounter
-	annotations := span.GetAnnotations()
-	for _, thisAnnotation := range annotations {
-		if strings.HasPrefix(thisAnnotation.GetValue(), common.FieldnamePrefixCounter) {
-			keyvalue := strings.Split(thisAnnotation.GetValue(), "=")
-			if len(keyvalue) == 2 {
-				if value, err := strconv.ParseFloat(keyvalue[1], 64); err == nil {
-					key := strings.TrimSpace(keyvalue[0])
-					key = common.FnPrefix + key[len(common.FieldnamePrefixCounter):] // strip off fieldname prefix and then prepend "fn_" to the front
-					keyValueMap[key] = value
-				}
-			}
-		}
-	}
-	return keyValueMap
-}
-
-// extract from the span the logged gauge metric values.
-// These are the ones whose names start with FieldnamePrefixGauge,
-// and whose values we assume are float64
-func getLoggedGaugeMetrics(span *zipkincore.Span) map[string]float64 {
-
-	keyValueMap := make(map[string]float64)
-
-	// extract any annotations whose Value starts with FieldnamePrefixGauge
-	annotations := span.GetAnnotations()
-	for _, thisAnnotation := range annotations {
-		if strings.HasPrefix(thisAnnotation.GetValue(), common.FieldnamePrefixGauge) {
-			keyvalue := strings.Split(thisAnnotation.GetValue(), "=")
-			if len(keyvalue) == 2 {
-				if value, err := strconv.ParseFloat(keyvalue[1], 64); err == nil {
-					key := strings.TrimSpace(keyvalue[0])
-					key = common.FnPrefix + key[len(common.FieldnamePrefixGauge):] // strip off fieldname prefix and then prepend "fn_" to the front
-					keyValueMap[key] = value
-				}
-			}
-		}
-	}
-	return keyValueMap
-}
-
-// PrometheusCollector implements Collector.
-func (*PrometheusCollector) Close() error { return nil }

api/server/prometheus_metrics.go

@@ -1,9 +0,0 @@
-package server
-
-import (
-	"github.com/gin-gonic/gin"
-)
-
-func (s *Server) handlePrometheusMetrics(c *gin.Context) {
-	s.agent.PromHandler().ServeHTTP(c.Writer, c.Request)
-}

api/server/server.go

@@ -24,9 +24,13 @@ import (
 	"github.com/fnproject/fn/api/version"
 	"github.com/fnproject/fn/fnext"
 	"github.com/gin-gonic/gin"
-	opentracing "github.com/opentracing/opentracing-go"
+	zipkinhttp "github.com/openzipkin/zipkin-go/reporter/http"
-	zipkintracer "github.com/openzipkin/zipkin-go-opentracing"
 	"github.com/sirupsen/logrus"
+	"go.opencensus.io/exporter/prometheus"
+	"go.opencensus.io/exporter/zipkin"
+	"go.opencensus.io/plugin/ochttp"
+	"go.opencensus.io/stats/view"
+	"go.opencensus.io/trace"
 )
 
 const (
@@ -79,6 +83,7 @@ type Server struct {
 	appListeners    *appListeners
 	rootMiddlewares []fnext.Middleware
 	apiMiddlewares  []fnext.Middleware
+	promExporter    *prometheus.Exporter
 }
 
 func nodeTypeFromString(value string) ServerNodeType {
@@ -226,8 +231,8 @@ func WithAgent(agent agent.Agent) ServerOption {
 // New creates a new Functions server with the opts given. For convenience, users may
 // prefer to use NewFromEnv but New is more flexible if needed.
 func New(ctx context.Context, opts ...ServerOption) *Server {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "server_init")
+	ctx, span := trace.StartSpan(ctx, "server_init")
-	defer span.Finish()
+	defer span.End()
 
 	log := common.Logger(ctx)
 	s := &Server{
@@ -290,50 +295,35 @@ func New(ctx context.Context, opts ...ServerOption) *Server {
 func WithTracer(zipkinURL string) ServerOption {
 	return func(ctx context.Context, s *Server) error {
 		var (
-			debugMode          = false
+			// TODO add server identifier to this crap
-			serviceName        = "fnserver"
+			//debugMode          = false
-			serviceHostPort    = "localhost:8080" // meh
+			//serviceName        = "fnserver"
+			//serviceHostPort    = "localhost:8080" // meh
 			zipkinHTTPEndpoint = zipkinURL
-			// ex: "http://zipkin:9411/api/v1/spans"
+			// ex: "http://zipkin:9411/api/v2/spans"
 		)
 
-		var collector zipkintracer.Collector
-
-		// custom Zipkin collector to send tracing spans to Prometheus
-		promCollector, promErr := NewPrometheusCollector()
-		if promErr != nil {
-			logrus.WithError(promErr).Fatalln("couldn't start Prometheus trace collector")
-		}
-
-		logger := zipkintracer.LoggerFunc(func(i ...interface{}) error { logrus.Error(i...); return nil })
-
 		if zipkinHTTPEndpoint != "" {
-			// Custom PrometheusCollector and Zipkin HTTPCollector
+			reporter := zipkinhttp.NewReporter(zipkinURL, zipkinhttp.MaxBacklog(10000))
-			httpCollector, zipErr := zipkintracer.NewHTTPCollector(zipkinHTTPEndpoint,
+			exporter := zipkin.NewExporter(reporter, nil)
-				zipkintracer.HTTPLogger(logger), zipkintracer.HTTPMaxBacklog(1000),
+			trace.RegisterExporter(exporter)
-			)
+			logrus.WithFields(logrus.Fields{"url": zipkinHTTPEndpoint}).Info("exporting spans to zipkin")
-			if zipErr != nil {
+
-				logrus.WithError(zipErr).Fatalln("couldn't start Zipkin trace collector")
+			// TODO don't do this. testing parity.
-			}
+			trace.SetDefaultSampler(trace.AlwaysSample())
-			collector = zipkintracer.MultiCollector{httpCollector, promCollector}
-		} else {
-			// Custom PrometheusCollector only
-			collector = promCollector
 		}
 
-		ziptracer, err := zipkintracer.NewTracer(zipkintracer.NewRecorder(collector, debugMode, serviceHostPort, serviceName),
+		// TODO we can keep this on *Server and unregister it in Close()... can finagle later. same for tracer
-			zipkintracer.ClientServerSameSpan(true),
+		exporter, err := prometheus.NewExporter(prometheus.Options{
-			zipkintracer.TraceID128Bit(true),
+			Namespace: "fn",
-		)
+			OnError:   func(err error) { logrus.WithError(err).Error("opencensus prometheus exporter err") },
+		})
 		if err != nil {
-			logrus.WithError(err).Fatalln("couldn't start tracer")
+			logrus.Fatal(err)
 		}
+		s.promExporter = exporter
+		view.RegisterExporter(exporter)
 
-		// wrap the Zipkin tracer in a FnTracer which will also send spans to Prometheus
-		fntracer := NewFnTracer(ziptracer)
-
-		opentracing.SetGlobalTracer(fntracer)
-		logrus.WithFields(logrus.Fields{"url": zipkinHTTPEndpoint}).Info("started tracer")
 		return nil
 	}
 }
@@ -404,7 +394,8 @@ func (s *Server) startGears(ctx context.Context, cancel context.CancelFunc) {
 
 	server := http.Server{
 		Addr:    listen,
-		Handler: s.Router,
+		Handler: &ochttp.Handler{Handler: s.Router},
+
 		// TODO we should set read/write timeouts
 	}
 
@@ -438,8 +429,11 @@ func (s *Server) bindHandlers(ctx context.Context) {
 
 	engine.GET("/", handlePing)
 	engine.GET("/version", handleVersion)
-	// TODO: move the following under v1
+
-	engine.GET("/metrics", s.handlePrometheusMetrics)
+	// TODO: move under v1 ?
+	if s.promExporter != nil {
+		engine.GET("/metrics", gin.WrapH(s.promExporter))
+	}
 
 	profilerSetup(engine, "/debug")
 

test/fn-api-tests/routes_test.go

@@ -3,9 +3,10 @@ package tests
 import (
 	"testing"
 
+	"reflect"
+
 	"github.com/fnproject/fn/api/id"
 	"github.com/fnproject/fn_go/models"
-	"reflect"
 )
 
 func TestCreateRouteEmptyType(t *testing.T) {

vendor/github.com/Shopify/sarama/.github/CONTRIBUTING.md

@@ -1,31 +0,0 @@
-# Contributing
-
-Contributions are always welcome, both reporting issues and submitting pull requests!
-
-### Reporting issues
-
-Please make sure to include any potentially useful information in the issue, so we can pinpoint the issue faster without going back and forth.
-
-- What SHA of Sarama are you running? If this is not the latest SHA on the master branch, please try if the problem persists with the latest version.
-- You can set `sarama.Logger` to a [log.Logger](http://golang.org/pkg/log/#Logger) instance to capture debug output. Please include it in your issue description.
-- Also look at the logs of the Kafka broker you are connected to. If you see anything out of the ordinary, please include it.
-
-Also, please include the following information about your environment, so we can help you faster:
-
-- What version of Kafka are you using?
-- What version of Go are you using?
-- What are the values of your Producer/Consumer/Client configuration?
-
-
-### Submitting pull requests
-
-We will gladly accept bug fixes, or additions to this library. Please fork this library, commit & push your changes, and open a pull request. Because this library is in production use by many people and applications, we code review all additions. To make the review process go as smooth as possible, please consider the following.
-
-- If you plan to work on something major, please open an issue to discuss the design first.
-- Don't break backwards compatibility. If you really have to, open an issue to discuss this first.
-- Make sure to use the `go fmt` command to format your code according to the standards. Even better, set up your editor to do this for you when saving.
-- Run [go vet](https://godoc.org/golang.org/x/tools/cmd/vet) to detect any suspicious constructs in your code that could be bugs.
-- Explicitly handle all error return values. If you really want to ignore an error value, you can assign it to `_`.You can use [errcheck](https://github.com/kisielk/errcheck) to verify whether you have handled all errors.
-- You may also want to run [golint](https://github.com/golang/lint) as well to detect style problems.
-- Add tests that cover the changes you made. Make sure to run `go test` with the `-race` argument to test for race conditions.
-- Make sure your code is supported by all the Go versions we support. You can rely on [Travis CI](https://travis-ci.org/Shopify/sarama) for testing older Go versions

vendor/github.com/Shopify/sarama/.github/ISSUE_TEMPLATE.md

@@ -1,20 +0,0 @@
-##### Versions
-
-*Please specify real version numbers or git SHAs, not just "Latest" since that changes fairly regularly.*
-Sarama Version:
-Kafka Version:
-Go Version:
-
-##### Configuration
-
-What configuration values are you using for Sarama and Kafka?
-
-##### Logs
-
-When filing an issue please provide logs from Sarama and Kafka if at all
-possible. You can set `sarama.Logger` to a `log.Logger` to capture Sarama debug
-output.
-
-##### Problem Description
-
-

vendor/github.com/Shopify/sarama/.gitignore

@@ -1,26 +0,0 @@
-# Compiled Object files, Static and Dynamic libs (Shared Objects)
-*.o
-*.a
-*.so
-*.test
-
-# Folders
-_obj
-_test
-.vagrant
-
-# Architecture specific extensions/prefixes
-*.[568vq]
-[568vq].out
-
-*.cgo1.go
-*.cgo2.c
-_cgo_defun.c
-_cgo_gotypes.go
-_cgo_export.*
-
-_testmain.go
-
-*.exe
-
-coverage.txt

vendor/github.com/Shopify/sarama/.travis.yml

@@ -1,37 +0,0 @@
-language: go
-go:
-- 1.7.x
-- 1.8.x
-- 1.9.x
-
-env:
-  global:
-  - KAFKA_PEERS=localhost:9091,localhost:9092,localhost:9093,localhost:9094,localhost:9095
-  - TOXIPROXY_ADDR=http://localhost:8474
-  - KAFKA_INSTALL_ROOT=/home/travis/kafka
-  - KAFKA_HOSTNAME=localhost
-  - DEBUG=true
-  matrix:
-  - KAFKA_VERSION=0.10.2.1
-  - KAFKA_VERSION=0.11.0.2
-  - KAFKA_VERSION=1.0.0
-
-before_install:
-- export REPOSITORY_ROOT=${TRAVIS_BUILD_DIR}
-- vagrant/install_cluster.sh
-- vagrant/boot_cluster.sh
-- vagrant/create_topics.sh
-
-install:
-- make install_dependencies
-
-script:
-- make test
-- make vet
-- make errcheck
-- make fmt
-
-after_success:
-  - bash <(curl -s https://codecov.io/bash)
-
-sudo: false

vendor/github.com/Shopify/sarama/CHANGELOG.md

@@ -1,461 +0,0 @@
-# Changelog
-
-#### Version 1.15.0 (2017-12-08)
-
-New Features:
- - Claim official support for Kafka 1.0, though it did already work
-   ([#984](https://github.com/Shopify/sarama/pull/984)).
- - Helper methods for Kafka version numbers to/from strings
-   ([#989](https://github.com/Shopify/sarama/pull/989)).
- - Implement CreatePartitions request/response
-   ([#985](https://github.com/Shopify/sarama/pull/985)).
-
-Improvements:
- - Add error codes 45-60
-   ([#986](https://github.com/Shopify/sarama/issues/986)).
-
-Bug Fixes:
- - Fix slow consuming for certain Kafka 0.11/1.0 configurations
-   ([#982](https://github.com/Shopify/sarama/pull/982)).
- - Correctly determine when a FetchResponse contains the new message format
-   ([#990](https://github.com/Shopify/sarama/pull/990)).
- - Fix producing with multiple headers
-   ([#996](https://github.com/Shopify/sarama/pull/996)).
- - Fix handling of truncated record batches
-   ([#998](https://github.com/Shopify/sarama/pull/998)).
- - Fix leaking metrics when closing brokers
-   ([#991](https://github.com/Shopify/sarama/pull/991)).
-
-#### Version 1.14.0 (2017-11-13)
-
-New Features:
- - Add support for the new Kafka 0.11 record-batch format, including the wire
-   protocol and the necessary behavioural changes in the producer and consumer.
-   Transactions and idempotency are not yet supported, but producing and
-   consuming should work with all the existing bells and whistles (batching,
-   compression, etc) as well as the new custom headers. Thanks to Vlad Hanciuta
-   of Arista Networks for this work. Part of
-   ([#901](https://github.com/Shopify/sarama/issues/901)).
-
-Bug Fixes:
- - Fix encoding of ProduceResponse versions in test
-   ([#970](https://github.com/Shopify/sarama/pull/970)).
- - Return partial replicas list when we have it
-   ([#975](https://github.com/Shopify/sarama/pull/975)).
-
-#### Version 1.13.0 (2017-10-04)
-
-New Features:
- - Support for FetchRequest version 3
-   ([#905](https://github.com/Shopify/sarama/pull/905)).
- - Permit setting version on mock FetchResponses
-   ([#939](https://github.com/Shopify/sarama/pull/939)).
- - Add a configuration option to support storing only minimal metadata for
-   extremely large clusters
-   ([#937](https://github.com/Shopify/sarama/pull/937)).
- - Add `PartitionOffsetManager.ResetOffset` for backtracking tracked offsets
-   ([#932](https://github.com/Shopify/sarama/pull/932)).
-
-Improvements:
- - Provide the block-level timestamp when consuming compressed messages
-   ([#885](https://github.com/Shopify/sarama/issues/885)).
- - `Client.Replicas` and `Client.InSyncReplicas` now respect the order returned
-   by the broker, which can be meaningful
-   ([#930](https://github.com/Shopify/sarama/pull/930)).
- - Use a `Ticker` to reduce consumer timer overhead at the cost of higher
-   variance in the actual timeout
-   ([#933](https://github.com/Shopify/sarama/pull/933)).
-
-Bug Fixes:
- - Gracefully handle messages with negative timestamps
-   ([#907](https://github.com/Shopify/sarama/pull/907)).
- - Raise a proper error when encountering an unknown message version
-   ([#940](https://github.com/Shopify/sarama/pull/940)).
-
-#### Version 1.12.0 (2017-05-08)
-
-New Features:
- - Added support for the `ApiVersions` request and response pair, and Kafka
-   version 0.10.2 ([#867](https://github.com/Shopify/sarama/pull/867)). Note
-   that you still need to specify the Kafka version in the Sarama configuration
-   for the time being.
- - Added a `Brokers` method to the Client which returns the complete set of
-   active brokers ([#813](https://github.com/Shopify/sarama/pull/813)).
- - Added an `InSyncReplicas` method to the Client which returns the set of all
-   in-sync broker IDs for the given partition, now that the Kafka versions for
-   which this was misleading are no longer in our supported set
-   ([#872](https://github.com/Shopify/sarama/pull/872)).
- - Added a `NewCustomHashPartitioner` method which allows constructing a hash
-   partitioner with a custom hash method in case the default (FNV-1a) is not
-   suitable
-   ([#837](https://github.com/Shopify/sarama/pull/837),
-    [#841](https://github.com/Shopify/sarama/pull/841)).
-
-Improvements:
- - Recognize more Kafka error codes
-   ([#859](https://github.com/Shopify/sarama/pull/859)).
-
-Bug Fixes:
- - Fix an issue where decoding a malformed FetchRequest would not return the
-   correct error ([#818](https://github.com/Shopify/sarama/pull/818)).
- - Respect ordering of group protocols in JoinGroupRequests. This fix is
-   transparent if you're using the `AddGroupProtocol` or
-   `AddGroupProtocolMetadata` helpers; otherwise you will need to switch from
-   the `GroupProtocols` field (now deprecated) to use `OrderedGroupProtocols`
-   ([#812](https://github.com/Shopify/sarama/issues/812)).
- - Fix an alignment-related issue with atomics on 32-bit architectures
-   ([#859](https://github.com/Shopify/sarama/pull/859)).
-
-#### Version 1.11.0 (2016-12-20)
-
-_Important:_ As of Sarama 1.11 it is necessary to set the config value of
-`Producer.Return.Successes` to true in order to use the SyncProducer. Previous
-versions would silently override this value when instantiating a SyncProducer
-which led to unexpected values and data races.
-
-New Features:
- - Metrics! Thanks to Sébastien Launay for all his work on this feature
-   ([#701](https://github.com/Shopify/sarama/pull/701),
-    [#746](https://github.com/Shopify/sarama/pull/746),
-    [#766](https://github.com/Shopify/sarama/pull/766)).
- - Add support for LZ4 compression
-   ([#786](https://github.com/Shopify/sarama/pull/786)).
- - Add support for ListOffsetRequest v1 and Kafka 0.10.1
-   ([#775](https://github.com/Shopify/sarama/pull/775)).
- - Added a `HighWaterMarks` method to the Consumer which aggregates the
-   `HighWaterMarkOffset` values of its child topic/partitions
-   ([#769](https://github.com/Shopify/sarama/pull/769)).
-
-Bug Fixes:
- - Fixed producing when using timestamps, compression and Kafka 0.10
-   ([#759](https://github.com/Shopify/sarama/pull/759)).
- - Added missing decoder methods to DescribeGroups response
-   ([#756](https://github.com/Shopify/sarama/pull/756)).
- - Fix producer shutdown when `Return.Errors` is disabled
-   ([#787](https://github.com/Shopify/sarama/pull/787)).
- - Don't mutate configuration in SyncProducer
-   ([#790](https://github.com/Shopify/sarama/pull/790)).
- - Fix crash on SASL initialization failure
-   ([#795](https://github.com/Shopify/sarama/pull/795)).
-
-#### Version 1.10.1 (2016-08-30)
-
-Bug Fixes:
- - Fix the documentation for `HashPartitioner` which was incorrect
-   ([#717](https://github.com/Shopify/sarama/pull/717)).
- - Permit client creation even when it is limited by ACLs
-   ([#722](https://github.com/Shopify/sarama/pull/722)).
- - Several fixes to the consumer timer optimization code, regressions introduced
-   in v1.10.0. Go's timers are finicky
-   ([#730](https://github.com/Shopify/sarama/pull/730),
-    [#733](https://github.com/Shopify/sarama/pull/733),
-    [#734](https://github.com/Shopify/sarama/pull/734)).
- - Handle consuming compressed relative offsets with Kafka 0.10
-   ([#735](https://github.com/Shopify/sarama/pull/735)).
-
-#### Version 1.10.0 (2016-08-02)
-
-_Important:_ As of Sarama 1.10 it is necessary to tell Sarama the version of
-Kafka you are running against (via the `config.Version` value) in order to use
-features that may not be compatible with old Kafka versions. If you don't
-specify this value it will default to 0.8.2 (the minimum supported), and trying
-to use more recent features (like the offset manager) will fail with an error.
-
-_Also:_ The offset-manager's behaviour has been changed to match the upstream
-java consumer (see [#705](https://github.com/Shopify/sarama/pull/705) and
-[#713](https://github.com/Shopify/sarama/pull/713)). If you use the
-offset-manager, please ensure that you are committing one *greater* than the
-last consumed message offset or else you may end up consuming duplicate
-messages.
-
-New Features:
- - Support for Kafka 0.10
-   ([#672](https://github.com/Shopify/sarama/pull/672),
-    [#678](https://github.com/Shopify/sarama/pull/678),
-    [#681](https://github.com/Shopify/sarama/pull/681), and others).
- - Support for configuring the target Kafka version
-   ([#676](https://github.com/Shopify/sarama/pull/676)).
- - Batch producing support in the SyncProducer
-   ([#677](https://github.com/Shopify/sarama/pull/677)).
- - Extend producer mock to allow setting expectations on message contents
-   ([#667](https://github.com/Shopify/sarama/pull/667)).
-
-Improvements:
- - Support `nil` compressed messages for deleting in compacted topics
-   ([#634](https://github.com/Shopify/sarama/pull/634)).
- - Pre-allocate decoding errors, greatly reducing heap usage and GC time against
-   misbehaving brokers ([#690](https://github.com/Shopify/sarama/pull/690)).
- - Re-use consumer expiry timers, removing one allocation per consumed message
-   ([#707](https://github.com/Shopify/sarama/pull/707)).
-
-Bug Fixes:
- - Actually default the client ID to "sarama" like we say we do
-   ([#664](https://github.com/Shopify/sarama/pull/664)).
- - Fix a rare issue where `Client.Leader` could return the wrong error
-   ([#685](https://github.com/Shopify/sarama/pull/685)).
- - Fix a possible tight loop in the consumer
-   ([#693](https://github.com/Shopify/sarama/pull/693)).
- - Match upstream's offset-tracking behaviour
-   ([#705](https://github.com/Shopify/sarama/pull/705)).
- - Report UnknownTopicOrPartition errors from the offset manager
-   ([#706](https://github.com/Shopify/sarama/pull/706)).
- - Fix possible negative partition value from the HashPartitioner
-   ([#709](https://github.com/Shopify/sarama/pull/709)).
-
-#### Version 1.9.0 (2016-05-16)
-
-New Features:
- - Add support for custom offset manager retention durations
-   ([#602](https://github.com/Shopify/sarama/pull/602)).
- - Publish low-level mocks to enable testing of third-party producer/consumer
-   implementations ([#570](https://github.com/Shopify/sarama/pull/570)).
- - Declare support for Golang 1.6
-   ([#611](https://github.com/Shopify/sarama/pull/611)).
- - Support for SASL plain-text auth
-   ([#648](https://github.com/Shopify/sarama/pull/648)).
-
-Improvements:
- - Simplified broker locking scheme slightly
-   ([#604](https://github.com/Shopify/sarama/pull/604)).
- - Documentation cleanup
-   ([#605](https://github.com/Shopify/sarama/pull/605),
-    [#621](https://github.com/Shopify/sarama/pull/621),
-    [#654](https://github.com/Shopify/sarama/pull/654)).
-
-Bug Fixes:
- - Fix race condition shutting down the OffsetManager
-   ([#658](https://github.com/Shopify/sarama/pull/658)).
-
-#### Version 1.8.0 (2016-02-01)
-
-New Features:
- - Full support for Kafka 0.9:
-   - All protocol messages and fields
-   ([#586](https://github.com/Shopify/sarama/pull/586),
-   [#588](https://github.com/Shopify/sarama/pull/588),
-   [#590](https://github.com/Shopify/sarama/pull/590)).
-   - Verified that TLS support works
-   ([#581](https://github.com/Shopify/sarama/pull/581)).
-   - Fixed the OffsetManager compatibility
-   ([#585](https://github.com/Shopify/sarama/pull/585)).
-
-Improvements:
- - Optimize for fewer system calls when reading from the network
-   ([#584](https://github.com/Shopify/sarama/pull/584)).
- - Automatically retry `InvalidMessage` errors to match upstream behaviour
-   ([#589](https://github.com/Shopify/sarama/pull/589)).
-
-#### Version 1.7.0 (2015-12-11)
-
-New Features:
- - Preliminary support for Kafka 0.9
-   ([#572](https://github.com/Shopify/sarama/pull/572)). This comes with several
-   caveats:
-   - Protocol-layer support is mostly in place
-     ([#577](https://github.com/Shopify/sarama/pull/577)), however Kafka 0.9
-     renamed some messages and fields, which we did not in order to preserve API
-     compatibility.
-   - The producer and consumer work against 0.9, but the offset manager does
-     not ([#573](https://github.com/Shopify/sarama/pull/573)).
-   - TLS support may or may not work
-     ([#581](https://github.com/Shopify/sarama/pull/581)).
-
-Improvements:
- - Don't wait for request timeouts on dead brokers, greatly speeding recovery
-   when the TCP connection is left hanging
-   ([#548](https://github.com/Shopify/sarama/pull/548)).
- - Refactored part of the producer. The new version provides a much more elegant
-   solution to [#449](https://github.com/Shopify/sarama/pull/449). It is also
-   slightly more efficient, and much more precise in calculating batch sizes
-   when compression is used
-   ([#549](https://github.com/Shopify/sarama/pull/549),
-   [#550](https://github.com/Shopify/sarama/pull/550),
-   [#551](https://github.com/Shopify/sarama/pull/551)).
-
-Bug Fixes:
- - Fix race condition in consumer test mock
-   ([#553](https://github.com/Shopify/sarama/pull/553)).
-
-#### Version 1.6.1 (2015-09-25)
-
-Bug Fixes:
- - Fix panic that could occur if a user-supplied message value failed to encode
-   ([#449](https://github.com/Shopify/sarama/pull/449)).
-
-#### Version 1.6.0 (2015-09-04)
-
-New Features:
- - Implementation of a consumer offset manager using the APIs introduced in
-   Kafka 0.8.2. The API is designed mainly for integration into a future
-   high-level consumer, not for direct use, although it is *possible* to use it
-   directly.
-   ([#461](https://github.com/Shopify/sarama/pull/461)).
-
-Improvements:
- - CRC32 calculation is much faster on machines with SSE4.2 instructions,
-   removing a major hotspot from most profiles
-   ([#255](https://github.com/Shopify/sarama/pull/255)).
-
-Bug Fixes:
- - Make protocol decoding more robust against some malformed packets generated
-   by go-fuzz ([#523](https://github.com/Shopify/sarama/pull/523),
-   [#525](https://github.com/Shopify/sarama/pull/525)) or found in other ways
-   ([#528](https://github.com/Shopify/sarama/pull/528)).
- - Fix a potential race condition panic in the consumer on shutdown
-   ([#529](https://github.com/Shopify/sarama/pull/529)).
-
-#### Version 1.5.0 (2015-08-17)
-
-New Features:
- - TLS-encrypted network connections are now supported. This feature is subject
-   to change when Kafka releases built-in TLS support, but for now this is
-   enough to work with TLS-terminating proxies
-   ([#154](https://github.com/Shopify/sarama/pull/154)).
-
-Improvements:
- - The consumer will not block if a single partition is not drained by the user;
-   all other partitions will continue to consume normally
-   ([#485](https://github.com/Shopify/sarama/pull/485)).
- - Formatting of error strings has been much improved
-   ([#495](https://github.com/Shopify/sarama/pull/495)).
- - Internal refactoring of the producer for code cleanliness and to enable
-   future work ([#300](https://github.com/Shopify/sarama/pull/300)).
-
-Bug Fixes:
- - Fix a potential deadlock in the consumer on shutdown
-   ([#475](https://github.com/Shopify/sarama/pull/475)).
-
-#### Version 1.4.3 (2015-07-21)
-
-Bug Fixes:
- - Don't include the partitioner in the producer's "fetch partitions"
-   circuit-breaker ([#466](https://github.com/Shopify/sarama/pull/466)).
- - Don't retry messages until the broker is closed when abandoning a broker in
-   the producer ([#468](https://github.com/Shopify/sarama/pull/468)).
- - Update the import path for snappy-go, it has moved again and the API has
-   changed slightly ([#486](https://github.com/Shopify/sarama/pull/486)).
-
-#### Version 1.4.2 (2015-05-27)
-
-Bug Fixes:
- - Update the import path for snappy-go, it has moved from google code to github
-   ([#456](https://github.com/Shopify/sarama/pull/456)).
-
-#### Version 1.4.1 (2015-05-25)
-
-Improvements:
- - Optimizations when decoding snappy messages, thanks to John Potocny
-   ([#446](https://github.com/Shopify/sarama/pull/446)).
-
-Bug Fixes:
- - Fix hypothetical race conditions on producer shutdown
-   ([#450](https://github.com/Shopify/sarama/pull/450),
-   [#451](https://github.com/Shopify/sarama/pull/451)).
-
-#### Version 1.4.0 (2015-05-01)
-
-New Features:
- - The consumer now implements `Topics()` and `Partitions()` methods to enable
-   users to dynamically choose what topics/partitions to consume without
-   instantiating a full client
-   ([#431](https://github.com/Shopify/sarama/pull/431)).
- - The partition-consumer now exposes the high water mark offset value returned
-   by the broker via the `HighWaterMarkOffset()` method ([#339](https://github.com/Shopify/sarama/pull/339)).
- - Added a `kafka-console-consumer` tool capable of handling multiple
-   partitions, and deprecated the now-obsolete `kafka-console-partitionConsumer`
-   ([#439](https://github.com/Shopify/sarama/pull/439),
-   [#442](https://github.com/Shopify/sarama/pull/442)).
-
-Improvements:
- - The producer's logging during retry scenarios is more consistent, more
-   useful, and slightly less verbose
-   ([#429](https://github.com/Shopify/sarama/pull/429)).
- - The client now shuffles its initial list of seed brokers in order to prevent
-   thundering herd on the first broker in the list
-   ([#441](https://github.com/Shopify/sarama/pull/441)).
-
-Bug Fixes:
- - The producer now correctly manages its state if retries occur when it is
-   shutting down, fixing several instances of confusing behaviour and at least
-   one potential deadlock ([#419](https://github.com/Shopify/sarama/pull/419)).
- - The consumer now handles messages for different partitions asynchronously,
-   making it much more resilient to specific user code ordering
-   ([#325](https://github.com/Shopify/sarama/pull/325)).
-
-#### Version 1.3.0 (2015-04-16)
-
-New Features:
- - The client now tracks consumer group coordinators using
-   ConsumerMetadataRequests similar to how it tracks partition leadership using
-   regular MetadataRequests ([#411](https://github.com/Shopify/sarama/pull/411)).
-   This adds two methods to the client API:
-   - `Coordinator(consumerGroup string) (*Broker, error)`
-   - `RefreshCoordinator(consumerGroup string) error`
-
-Improvements:
- - ConsumerMetadataResponses now automatically create a Broker object out of the
-   ID/address/port combination for the Coordinator; accessing the fields
-   individually has been deprecated
-   ([#413](https://github.com/Shopify/sarama/pull/413)).
- - Much improved handling of `OffsetOutOfRange` errors in the consumer.
-   Consumers will fail to start if the provided offset is out of range
-   ([#418](https://github.com/Shopify/sarama/pull/418))
-   and they will automatically shut down if the offset falls out of range
-   ([#424](https://github.com/Shopify/sarama/pull/424)).
- - Small performance improvement in encoding and decoding protocol messages
-   ([#427](https://github.com/Shopify/sarama/pull/427)).
-
-Bug Fixes:
- - Fix a rare race condition in the client's background metadata refresher if
-   it happens to be activated while the client is being closed
-   ([#422](https://github.com/Shopify/sarama/pull/422)).
-
-#### Version 1.2.0 (2015-04-07)
-
-Improvements:
- - The producer's behaviour when `Flush.Frequency` is set is now more intuitive
-   ([#389](https://github.com/Shopify/sarama/pull/389)).
- - The producer is now somewhat more memory-efficient during and after retrying
-   messages due to an improved queue implementation
-   ([#396](https://github.com/Shopify/sarama/pull/396)).
- - The consumer produces much more useful logging output when leadership
-   changes ([#385](https://github.com/Shopify/sarama/pull/385)).
- - The client's `GetOffset` method will now automatically refresh metadata and
-   retry once in the event of stale information or similar
-   ([#394](https://github.com/Shopify/sarama/pull/394)).
- - Broker connections now have support for using TCP keepalives
-   ([#407](https://github.com/Shopify/sarama/issues/407)).
-
-Bug Fixes:
- - The OffsetCommitRequest message now correctly implements all three possible
-   API versions ([#390](https://github.com/Shopify/sarama/pull/390),
-   [#400](https://github.com/Shopify/sarama/pull/400)).
-
-#### Version 1.1.0 (2015-03-20)
-
-Improvements:
- - Wrap the producer's partitioner call in a circuit-breaker so that repeatedly
-   broken topics don't choke throughput
-   ([#373](https://github.com/Shopify/sarama/pull/373)).
-
-Bug Fixes:
- - Fix the producer's internal reference counting in certain unusual scenarios
-   ([#367](https://github.com/Shopify/sarama/pull/367)).
- - Fix the consumer's internal reference counting in certain unusual scenarios
-   ([#369](https://github.com/Shopify/sarama/pull/369)).
- - Fix a condition where the producer's internal control messages could have
-   gotten stuck ([#368](https://github.com/Shopify/sarama/pull/368)).
- - Fix an issue where invalid partition lists would be cached when asking for
-   metadata for a non-existant topic ([#372](https://github.com/Shopify/sarama/pull/372)).
-
-
-#### Version 1.0.0 (2015-03-17)
-
-Version 1.0.0 is the first tagged version, and is almost a complete rewrite. The primary differences with previous untagged versions are:
-
-- The producer has been rewritten; there is now a `SyncProducer` with a blocking API, and an `AsyncProducer` that is non-blocking.
-- The consumer has been rewritten to only open one connection per broker instead of one connection per partition.
-- The main types of Sarama are now interfaces to make depedency injection easy; mock implementations for `Consumer`, `SyncProducer` and `AsyncProducer` are provided in the `github.com/Shopify/sarama/mocks` package.
-- For most uses cases, it is no longer necessary to open a `Client`; this will be done for you.
-- All the configuration values have been unified in the `Config` struct.
-- Much improved test suite.

vendor/github.com/Shopify/sarama/LICENSE

@@ -1,20 +0,0 @@
-Copyright (c) 2013 Evan Huus
-
-Permission is hereby granted, free of charge, to any person obtaining
-a copy of this software and associated documentation files (the
-"Software"), to deal in the Software without restriction, including
-without limitation the rights to use, copy, modify, merge, publish,
-distribute, sublicense, and/or sell copies of the Software, and to
-permit persons to whom the Software is furnished to do so, subject to
-the following conditions:
-
-The above copyright notice and this permission notice shall be
-included in all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
-LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

vendor/github.com/Shopify/sarama/Makefile

@@ -1,29 +0,0 @@
-default: fmt vet errcheck test
-
-# Taken from https://github.com/codecov/example-go#caveat-multiple-files
-test:
-	echo "" > coverage.txt
-	for d in `go list ./... | grep -v vendor`; do \
-		go test -v -timeout 60s -race -coverprofile=profile.out -covermode=atomic $$d; \
-		if [ -f profile.out ]; then \
-			cat profile.out >> coverage.txt; \
-			rm profile.out; \
-		fi \
-	done
-
-vet:
-	go vet ./...
-
-errcheck:
-	errcheck github.com/Shopify/sarama/...
-
-fmt:
-	@if [ -n "$$(go fmt ./...)" ]; then echo 'Please run go fmt on your code.' && exit 1; fi
-
-install_dependencies: install_errcheck get
-
-install_errcheck:
-	go get github.com/kisielk/errcheck
-
-get:
-	go get -t

vendor/github.com/Shopify/sarama/README.md

@@ -1,39 +0,0 @@
-sarama
-======
-
-[![GoDoc](https://godoc.org/github.com/Shopify/sarama?status.png)](https://godoc.org/github.com/Shopify/sarama)
-[![Build Status](https://travis-ci.org/Shopify/sarama.svg?branch=master)](https://travis-ci.org/Shopify/sarama)
-[![Coverage](https://codecov.io/gh/Shopify/sarama/branch/master/graph/badge.svg)](https://codecov.io/gh/Shopify/sarama)
-
-Sarama is an MIT-licensed Go client library for [Apache Kafka](https://kafka.apache.org/) version 0.8 (and later).
-
-### Getting started
-
-- API documentation and examples are available via [godoc](https://godoc.org/github.com/Shopify/sarama).
-- Mocks for testing are available in the [mocks](./mocks) subpackage.
-- The [examples](./examples) directory contains more elaborate example applications.
-- The [tools](./tools) directory contains command line tools that can be useful for testing, diagnostics, and instrumentation.
-
-You might also want to look at the [Frequently Asked Questions](https://github.com/Shopify/sarama/wiki/Frequently-Asked-Questions).
-
-### Compatibility and API stability
-
-Sarama provides a "2 releases + 2 months" compatibility guarantee: we support
-the two latest stable releases of Kafka and Go, and we provide a two month
-grace period for older releases. This means we currently officially support
-Go 1.9 through 1.7, and Kafka 1.0 through 0.10, although older releases are
-still likely to work.
-
-Sarama follows semantic versioning and provides API stability via the gopkg.in service.
-You can import a version with a guaranteed stable API via http://gopkg.in/Shopify/sarama.v1.
-A changelog is available [here](CHANGELOG.md).
-
-### Contributing
-
-* Get started by checking our [contribution guidelines](https://github.com/Shopify/sarama/blob/master/.github/CONTRIBUTING.md).
-* Read the [Sarama wiki](https://github.com/Shopify/sarama/wiki) for more
-  technical and design details.
-* The [Kafka Protocol Specification](https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol)
-  contains a wealth of useful information.
-* For more general issues, there is [a google group](https://groups.google.com/forum/#!forum/kafka-clients) for Kafka client developers.
-* If you have any questions, just ask!

vendor/github.com/Shopify/sarama/Vagrantfile

@@ -1,20 +0,0 @@
-# -*- mode: ruby -*-
-# vi: set ft=ruby :
-
-# Vagrantfile API/syntax version. Don't touch unless you know what you're doing!
-VAGRANTFILE_API_VERSION = "2"
-
-# We have 5 * 192MB ZK processes and 5 * 320MB Kafka processes => 2560MB
-MEMORY = 3072
-
-Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
-  config.vm.box = "ubuntu/trusty64"
-
-  config.vm.provision :shell, path: "vagrant/provision.sh"
-
-  config.vm.network "private_network", ip: "192.168.100.67"
-
-  config.vm.provider "virtualbox" do |v|
-    v.memory = MEMORY
-  end
-end

vendor/github.com/Shopify/sarama/api_versions_request.go

@@ -1,24 +0,0 @@
-package sarama
-
-type ApiVersionsRequest struct {
-}
-
-func (r *ApiVersionsRequest) encode(pe packetEncoder) error {
-	return nil
-}
-
-func (r *ApiVersionsRequest) decode(pd packetDecoder, version int16) (err error) {
-	return nil
-}
-
-func (r *ApiVersionsRequest) key() int16 {
-	return 18
-}
-
-func (r *ApiVersionsRequest) version() int16 {
-	return 0
-}
-
-func (r *ApiVersionsRequest) requiredVersion() KafkaVersion {
-	return V0_10_0_0
-}

vendor/github.com/Shopify/sarama/api_versions_request_test.go

@@ -1,14 +0,0 @@
-package sarama
-
-import "testing"
-
-var (
-	apiVersionRequest = []byte{}
-)
-
-func TestApiVersionsRequest(t *testing.T) {
-	var request *ApiVersionsRequest
-
-	request = new(ApiVersionsRequest)
-	testRequest(t, "basic", request, apiVersionRequest)
-}

vendor/github.com/Shopify/sarama/api_versions_response.go

@@ -1,87 +0,0 @@
-package sarama
-
-type ApiVersionsResponseBlock struct {
-	ApiKey     int16
-	MinVersion int16
-	MaxVersion int16
-}
-
-func (b *ApiVersionsResponseBlock) encode(pe packetEncoder) error {
-	pe.putInt16(b.ApiKey)
-	pe.putInt16(b.MinVersion)
-	pe.putInt16(b.MaxVersion)
-	return nil
-}
-
-func (b *ApiVersionsResponseBlock) decode(pd packetDecoder) error {
-	var err error
-
-	if b.ApiKey, err = pd.getInt16(); err != nil {
-		return err
-	}
-
-	if b.MinVersion, err = pd.getInt16(); err != nil {
-		return err
-	}
-
-	if b.MaxVersion, err = pd.getInt16(); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-type ApiVersionsResponse struct {
-	Err         KError
-	ApiVersions []*ApiVersionsResponseBlock
-}
-
-func (r *ApiVersionsResponse) encode(pe packetEncoder) error {
-	pe.putInt16(int16(r.Err))
-	if err := pe.putArrayLength(len(r.ApiVersions)); err != nil {
-		return err
-	}
-	for _, apiVersion := range r.ApiVersions {
-		if err := apiVersion.encode(pe); err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-func (r *ApiVersionsResponse) decode(pd packetDecoder, version int16) error {
-	kerr, err := pd.getInt16()
-	if err != nil {
-		return err
-	}
-
-	r.Err = KError(kerr)
-
-	numBlocks, err := pd.getArrayLength()
-	if err != nil {
-		return err
-	}
-
-	r.ApiVersions = make([]*ApiVersionsResponseBlock, numBlocks)
-	for i := 0; i < numBlocks; i++ {
-		block := new(ApiVersionsResponseBlock)
-		if err := block.decode(pd); err != nil {
-			return err
-		}
-		r.ApiVersions[i] = block
-	}
-
-	return nil
-}
-
-func (r *ApiVersionsResponse) key() int16 {
-	return 18
-}
-
-func (r *ApiVersionsResponse) version() int16 {
-	return 0
-}
-
-func (r *ApiVersionsResponse) requiredVersion() KafkaVersion {
-	return V0_10_0_0
-}

vendor/github.com/Shopify/sarama/api_versions_response_test.go

@@ -1,32 +0,0 @@
-package sarama
-
-import "testing"
-
-var (
-	apiVersionResponse = []byte{
-		0x00, 0x00,
-		0x00, 0x00, 0x00, 0x01,
-		0x00, 0x03,
-		0x00, 0x02,
-		0x00, 0x01,
-	}
-)
-
-func TestApiVersionsResponse(t *testing.T) {
-	var response *ApiVersionsResponse
-
-	response = new(ApiVersionsResponse)
-	testVersionDecodable(t, "no error", response, apiVersionResponse, 0)
-	if response.Err != ErrNoError {
-		t.Error("Decoding error failed: no error expected but found", response.Err)
-	}
-	if response.ApiVersions[0].ApiKey != 0x03 {
-		t.Error("Decoding error: expected 0x03 but got", response.ApiVersions[0].ApiKey)
-	}
-	if response.ApiVersions[0].MinVersion != 0x02 {
-		t.Error("Decoding error: expected 0x02 but got", response.ApiVersions[0].MinVersion)
-	}
-	if response.ApiVersions[0].MaxVersion != 0x01 {
-		t.Error("Decoding error: expected 0x01 but got", response.ApiVersions[0].MaxVersion)
-	}
-}

vendor/github.com/Shopify/sarama/async_producer.go

@@ -1,921 +0,0 @@
-package sarama
-
-import (
-	"encoding/binary"
-	"fmt"
-	"sync"
-	"time"
-
-	"github.com/eapache/go-resiliency/breaker"
-	"github.com/eapache/queue"
-)
-
-// AsyncProducer publishes Kafka messages using a non-blocking API. It routes messages
-// to the correct broker for the provided topic-partition, refreshing metadata as appropriate,
-// and parses responses for errors. You must read from the Errors() channel or the
-// producer will deadlock. You must call Close() or AsyncClose() on a producer to avoid
-// leaks: it will not be garbage-collected automatically when it passes out of
-// scope.
-type AsyncProducer interface {
-
-	// AsyncClose triggers a shutdown of the producer. The shutdown has completed
-	// when both the Errors and Successes channels have been closed. When calling
-	// AsyncClose, you *must* continue to read from those channels in order to
-	// drain the results of any messages in flight.
-	AsyncClose()
-
-	// Close shuts down the producer and waits for any buffered messages to be
-	// flushed. You must call this function before a producer object passes out of
-	// scope, as it may otherwise leak memory. You must call this before calling
-	// Close on the underlying client.
-	Close() error
-
-	// Input is the input channel for the user to write messages to that they
-	// wish to send.
-	Input() chan<- *ProducerMessage
-
-	// Successes is the success output channel back to the user when Return.Successes is
-	// enabled. If Return.Successes is true, you MUST read from this channel or the
-	// Producer will deadlock. It is suggested that you send and read messages
-	// together in a single select statement.
-	Successes() <-chan *ProducerMessage
-
-	// Errors is the error output channel back to the user. You MUST read from this
-	// channel or the Producer will deadlock when the channel is full. Alternatively,
-	// you can set Producer.Return.Errors in your config to false, which prevents
-	// errors to be returned.
-	Errors() <-chan *ProducerError
-}
-
-type asyncProducer struct {
-	client    Client
-	conf      *Config
-	ownClient bool
-
-	errors                    chan *ProducerError
-	input, successes, retries chan *ProducerMessage
-	inFlight                  sync.WaitGroup
-
-	brokers    map[*Broker]chan<- *ProducerMessage
-	brokerRefs map[chan<- *ProducerMessage]int
-	brokerLock sync.Mutex
-}
-
-// NewAsyncProducer creates a new AsyncProducer using the given broker addresses and configuration.
-func NewAsyncProducer(addrs []string, conf *Config) (AsyncProducer, error) {
-	client, err := NewClient(addrs, conf)
-	if err != nil {
-		return nil, err
-	}
-
-	p, err := NewAsyncProducerFromClient(client)
-	if err != nil {
-		return nil, err
-	}
-	p.(*asyncProducer).ownClient = true
-	return p, nil
-}
-
-// NewAsyncProducerFromClient creates a new Producer using the given client. It is still
-// necessary to call Close() on the underlying client when shutting down this producer.
-func NewAsyncProducerFromClient(client Client) (AsyncProducer, error) {
-	// Check that we are not dealing with a closed Client before processing any other arguments
-	if client.Closed() {
-		return nil, ErrClosedClient
-	}
-
-	p := &asyncProducer{
-		client:     client,
-		conf:       client.Config(),
-		errors:     make(chan *ProducerError),
-		input:      make(chan *ProducerMessage),
-		successes:  make(chan *ProducerMessage),
-		retries:    make(chan *ProducerMessage),
-		brokers:    make(map[*Broker]chan<- *ProducerMessage),
-		brokerRefs: make(map[chan<- *ProducerMessage]int),
-	}
-
-	// launch our singleton dispatchers
-	go withRecover(p.dispatcher)
-	go withRecover(p.retryHandler)
-
-	return p, nil
-}
-
-type flagSet int8
-
-const (
-	syn      flagSet = 1 << iota // first message from partitionProducer to brokerProducer
-	fin                          // final message from partitionProducer to brokerProducer and back
-	shutdown                     // start the shutdown process
-)
-
-// ProducerMessage is the collection of elements passed to the Producer in order to send a message.
-type ProducerMessage struct {
-	Topic string // The Kafka topic for this message.
-	// The partitioning key for this message. Pre-existing Encoders include
-	// StringEncoder and ByteEncoder.
-	Key Encoder
-	// The actual message to store in Kafka. Pre-existing Encoders include
-	// StringEncoder and ByteEncoder.
-	Value Encoder
-
-	// The headers are key-value pairs that are transparently passed
-	// by Kafka between producers and consumers.
-	Headers []RecordHeader
-
-	// This field is used to hold arbitrary data you wish to include so it
-	// will be available when receiving on the Successes and Errors channels.
-	// Sarama completely ignores this field and is only to be used for
-	// pass-through data.
-	Metadata interface{}
-
-	// Below this point are filled in by the producer as the message is processed
-
-	// Offset is the offset of the message stored on the broker. This is only
-	// guaranteed to be defined if the message was successfully delivered and
-	// RequiredAcks is not NoResponse.
-	Offset int64
-	// Partition is the partition that the message was sent to. This is only
-	// guaranteed to be defined if the message was successfully delivered.
-	Partition int32
-	// Timestamp is the timestamp assigned to the message by the broker. This
-	// is only guaranteed to be defined if the message was successfully
-	// delivered, RequiredAcks is not NoResponse, and the Kafka broker is at
-	// least version 0.10.0.
-	Timestamp time.Time
-
-	retries int
-	flags   flagSet
-}
-
-const producerMessageOverhead = 26 // the metadata overhead of CRC, flags, etc.
-
-func (m *ProducerMessage) byteSize(version int) int {
-	var size int
-	if version >= 2 {
-		size = maximumRecordOverhead
-		for _, h := range m.Headers {
-			size += len(h.Key) + len(h.Value) + 2*binary.MaxVarintLen32
-		}
-	} else {
-		size = producerMessageOverhead
-	}
-	if m.Key != nil {
-		size += m.Key.Length()
-	}
-	if m.Value != nil {
-		size += m.Value.Length()
-	}
-	return size
-}
-
-func (m *ProducerMessage) clear() {
-	m.flags = 0
-	m.retries = 0
-}
-
-// ProducerError is the type of error generated when the producer fails to deliver a message.
-// It contains the original ProducerMessage as well as the actual error value.
-type ProducerError struct {
-	Msg *ProducerMessage
-	Err error
-}
-
-func (pe ProducerError) Error() string {
-	return fmt.Sprintf("kafka: Failed to produce message to topic %s: %s", pe.Msg.Topic, pe.Err)
-}
-
-// ProducerErrors is a type that wraps a batch of "ProducerError"s and implements the Error interface.
-// It can be returned from the Producer's Close method to avoid the need to manually drain the Errors channel
-// when closing a producer.
-type ProducerErrors []*ProducerError
-
-func (pe ProducerErrors) Error() string {
-	return fmt.Sprintf("kafka: Failed to deliver %d messages.", len(pe))
-}
-
-func (p *asyncProducer) Errors() <-chan *ProducerError {
-	return p.errors
-}
-
-func (p *asyncProducer) Successes() <-chan *ProducerMessage {
-	return p.successes
-}
-
-func (p *asyncProducer) Input() chan<- *ProducerMessage {
-	return p.input
-}
-
-func (p *asyncProducer) Close() error {
-	p.AsyncClose()
-
-	if p.conf.Producer.Return.Successes {
-		go withRecover(func() {
-			for range p.successes {
-			}
-		})
-	}
-
-	var errors ProducerErrors
-	if p.conf.Producer.Return.Errors {
-		for event := range p.errors {
-			errors = append(errors, event)
-		}
-	} else {
-		<-p.errors
-	}
-
-	if len(errors) > 0 {
-		return errors
-	}
-	return nil
-}
-
-func (p *asyncProducer) AsyncClose() {
-	go withRecover(p.shutdown)
-}
-
-// singleton
-// dispatches messages by topic
-func (p *asyncProducer) dispatcher() {
-	handlers := make(map[string]chan<- *ProducerMessage)
-	shuttingDown := false
-
-	for msg := range p.input {
-		if msg == nil {
-			Logger.Println("Something tried to send a nil message, it was ignored.")
-			continue
-		}
-
-		if msg.flags&shutdown != 0 {
-			shuttingDown = true
-			p.inFlight.Done()
-			continue
-		} else if msg.retries == 0 {
-			if shuttingDown {
-				// we can't just call returnError here because that decrements the wait group,
-				// which hasn't been incremented yet for this message, and shouldn't be
-				pErr := &ProducerError{Msg: msg, Err: ErrShuttingDown}
-				if p.conf.Producer.Return.Errors {
-					p.errors <- pErr
-				} else {
-					Logger.Println(pErr)
-				}
-				continue
-			}
-			p.inFlight.Add(1)
-		}
-
-		version := 1
-		if p.conf.Version.IsAtLeast(V0_11_0_0) {
-			version = 2
-		}
-		if msg.byteSize(version) > p.conf.Producer.MaxMessageBytes {
-			p.returnError(msg, ErrMessageSizeTooLarge)
-			continue
-		}
-
-		handler := handlers[msg.Topic]
-		if handler == nil {
-			handler = p.newTopicProducer(msg.Topic)
-			handlers[msg.Topic] = handler
-		}
-
-		handler <- msg
-	}
-
-	for _, handler := range handlers {
-		close(handler)
-	}
-}
-
-// one per topic
-// partitions messages, then dispatches them by partition
-type topicProducer struct {
-	parent *asyncProducer
-	topic  string
-	input  <-chan *ProducerMessage
-
-	breaker     *breaker.Breaker
-	handlers    map[int32]chan<- *ProducerMessage
-	partitioner Partitioner
-}
-
-func (p *asyncProducer) newTopicProducer(topic string) chan<- *ProducerMessage {
-	input := make(chan *ProducerMessage, p.conf.ChannelBufferSize)
-	tp := &topicProducer{
-		parent:      p,
-		topic:       topic,
-		input:       input,
-		breaker:     breaker.New(3, 1, 10*time.Second),
-		handlers:    make(map[int32]chan<- *ProducerMessage),
-		partitioner: p.conf.Producer.Partitioner(topic),
-	}
-	go withRecover(tp.dispatch)
-	return input
-}
-
-func (tp *topicProducer) dispatch() {
-	for msg := range tp.input {
-		if msg.retries == 0 {
-			if err := tp.partitionMessage(msg); err != nil {
-				tp.parent.returnError(msg, err)
-				continue
-			}
-		}
-
-		handler := tp.handlers[msg.Partition]
-		if handler == nil {
-			handler = tp.parent.newPartitionProducer(msg.Topic, msg.Partition)
-			tp.handlers[msg.Partition] = handler
-		}
-
-		handler <- msg
-	}
-
-	for _, handler := range tp.handlers {
-		close(handler)
-	}
-}
-
-func (tp *topicProducer) partitionMessage(msg *ProducerMessage) error {
-	var partitions []int32
-
-	err := tp.breaker.Run(func() (err error) {
-		if tp.partitioner.RequiresConsistency() {
-			partitions, err = tp.parent.client.Partitions(msg.Topic)
-		} else {
-			partitions, err = tp.parent.client.WritablePartitions(msg.Topic)
-		}
-		return
-	})
-
-	if err != nil {
-		return err
-	}
-
-	numPartitions := int32(len(partitions))
-
-	if numPartitions == 0 {
-		return ErrLeaderNotAvailable
-	}
-
-	choice, err := tp.partitioner.Partition(msg, numPartitions)
-
-	if err != nil {
-		return err
-	} else if choice < 0 || choice >= numPartitions {
-		return ErrInvalidPartition
-	}
-
-	msg.Partition = partitions[choice]
-
-	return nil
-}
-
-// one per partition per topic
-// dispatches messages to the appropriate broker
-// also responsible for maintaining message order during retries
-type partitionProducer struct {
-	parent    *asyncProducer
-	topic     string
-	partition int32
-	input     <-chan *ProducerMessage
-
-	leader  *Broker
-	breaker *breaker.Breaker
-	output  chan<- *ProducerMessage
-
-	// highWatermark tracks the "current" retry level, which is the only one where we actually let messages through,
-	// all other messages get buffered in retryState[msg.retries].buf to preserve ordering
-	// retryState[msg.retries].expectChaser simply tracks whether we've seen a fin message for a given level (and
-	// therefore whether our buffer is complete and safe to flush)
-	highWatermark int
-	retryState    []partitionRetryState
-}
-
-type partitionRetryState struct {
-	buf          []*ProducerMessage
-	expectChaser bool
-}
-
-func (p *asyncProducer) newPartitionProducer(topic string, partition int32) chan<- *ProducerMessage {
-	input := make(chan *ProducerMessage, p.conf.ChannelBufferSize)
-	pp := &partitionProducer{
-		parent:    p,
-		topic:     topic,
-		partition: partition,
-		input:     input,
-
-		breaker:    breaker.New(3, 1, 10*time.Second),
-		retryState: make([]partitionRetryState, p.conf.Producer.Retry.Max+1),
-	}
-	go withRecover(pp.dispatch)
-	return input
-}
-
-func (pp *partitionProducer) dispatch() {
-	// try to prefetch the leader; if this doesn't work, we'll do a proper call to `updateLeader`
-	// on the first message
-	pp.leader, _ = pp.parent.client.Leader(pp.topic, pp.partition)
-	if pp.leader != nil {
-		pp.output = pp.parent.getBrokerProducer(pp.leader)
-		pp.parent.inFlight.Add(1) // we're generating a syn message; track it so we don't shut down while it's still inflight
-		pp.output <- &ProducerMessage{Topic: pp.topic, Partition: pp.partition, flags: syn}
-	}
-
-	for msg := range pp.input {
-		if msg.retries > pp.highWatermark {
-			// a new, higher, retry level; handle it and then back off
-			pp.newHighWatermark(msg.retries)
-			time.Sleep(pp.parent.conf.Producer.Retry.Backoff)
-		} else if pp.highWatermark > 0 {
-			// we are retrying something (else highWatermark would be 0) but this message is not a *new* retry level
-			if msg.retries < pp.highWatermark {
-				// in fact this message is not even the current retry level, so buffer it for now (unless it's a just a fin)
-				if msg.flags&fin == fin {
-					pp.retryState[msg.retries].expectChaser = false
-					pp.parent.inFlight.Done() // this fin is now handled and will be garbage collected
-				} else {
-					pp.retryState[msg.retries].buf = append(pp.retryState[msg.retries].buf, msg)
-				}
-				continue
-			} else if msg.flags&fin == fin {
-				// this message is of the current retry level (msg.retries == highWatermark) and the fin flag is set,
-				// meaning this retry level is done and we can go down (at least) one level and flush that
-				pp.retryState[pp.highWatermark].expectChaser = false
-				pp.flushRetryBuffers()
-				pp.parent.inFlight.Done() // this fin is now handled and will be garbage collected
-				continue
-			}
-		}
-
-		// if we made it this far then the current msg contains real data, and can be sent to the next goroutine
-		// without breaking any of our ordering guarantees
-
-		if pp.output == nil {
-			if err := pp.updateLeader(); err != nil {
-				pp.parent.returnError(msg, err)
-				time.Sleep(pp.parent.conf.Producer.Retry.Backoff)
-				continue
-			}
-			Logger.Printf("producer/leader/%s/%d selected broker %d\n", pp.topic, pp.partition, pp.leader.ID())
-		}
-
-		pp.output <- msg
-	}
-
-	if pp.output != nil {
-		pp.parent.unrefBrokerProducer(pp.leader, pp.output)
-	}
-}
-
-func (pp *partitionProducer) newHighWatermark(hwm int) {
-	Logger.Printf("producer/leader/%s/%d state change to [retrying-%d]\n", pp.topic, pp.partition, hwm)
-	pp.highWatermark = hwm
-
-	// send off a fin so that we know when everything "in between" has made it
-	// back to us and we can safely flush the backlog (otherwise we risk re-ordering messages)
-	pp.retryState[pp.highWatermark].expectChaser = true
-	pp.parent.inFlight.Add(1) // we're generating a fin message; track it so we don't shut down while it's still inflight
-	pp.output <- &ProducerMessage{Topic: pp.topic, Partition: pp.partition, flags: fin, retries: pp.highWatermark - 1}
-
-	// a new HWM means that our current broker selection is out of date
-	Logger.Printf("producer/leader/%s/%d abandoning broker %d\n", pp.topic, pp.partition, pp.leader.ID())
-	pp.parent.unrefBrokerProducer(pp.leader, pp.output)
-	pp.output = nil
-}
-
-func (pp *partitionProducer) flushRetryBuffers() {
-	Logger.Printf("producer/leader/%s/%d state change to [flushing-%d]\n", pp.topic, pp.partition, pp.highWatermark)
-	for {
-		pp.highWatermark--
-
-		if pp.output == nil {
-			if err := pp.updateLeader(); err != nil {
-				pp.parent.returnErrors(pp.retryState[pp.highWatermark].buf, err)
-				goto flushDone
-			}
-			Logger.Printf("producer/leader/%s/%d selected broker %d\n", pp.topic, pp.partition, pp.leader.ID())
-		}
-
-		for _, msg := range pp.retryState[pp.highWatermark].buf {
-			pp.output <- msg
-		}
-
-	flushDone:
-		pp.retryState[pp.highWatermark].buf = nil
-		if pp.retryState[pp.highWatermark].expectChaser {
-			Logger.Printf("producer/leader/%s/%d state change to [retrying-%d]\n", pp.topic, pp.partition, pp.highWatermark)
-			break
-		} else if pp.highWatermark == 0 {
-			Logger.Printf("producer/leader/%s/%d state change to [normal]\n", pp.topic, pp.partition)
-			break
-		}
-	}
-}
-
-func (pp *partitionProducer) updateLeader() error {
-	return pp.breaker.Run(func() (err error) {
-		if err = pp.parent.client.RefreshMetadata(pp.topic); err != nil {
-			return err
-		}
-
-		if pp.leader, err = pp.parent.client.Leader(pp.topic, pp.partition); err != nil {
-			return err
-		}
-
-		pp.output = pp.parent.getBrokerProducer(pp.leader)
-		pp.parent.inFlight.Add(1) // we're generating a syn message; track it so we don't shut down while it's still inflight
-		pp.output <- &ProducerMessage{Topic: pp.topic, Partition: pp.partition, flags: syn}
-
-		return nil
-	})
-}
-
-// one per broker; also constructs an associated flusher
-func (p *asyncProducer) newBrokerProducer(broker *Broker) chan<- *ProducerMessage {
-	var (
-		input     = make(chan *ProducerMessage)
-		bridge    = make(chan *produceSet)
-		responses = make(chan *brokerProducerResponse)
-	)
-
-	bp := &brokerProducer{
-		parent:         p,
-		broker:         broker,
-		input:          input,
-		output:         bridge,
-		responses:      responses,
-		buffer:         newProduceSet(p),
-		currentRetries: make(map[string]map[int32]error),
-	}
-	go withRecover(bp.run)
-
-	// minimal bridge to make the network response `select`able
-	go withRecover(func() {
-		for set := range bridge {
-			request := set.buildRequest()
-
-			response, err := broker.Produce(request)
-
-			responses <- &brokerProducerResponse{
-				set: set,
-				err: err,
-				res: response,
-			}
-		}
-		close(responses)
-	})
-
-	return input
-}
-
-type brokerProducerResponse struct {
-	set *produceSet
-	err error
-	res *ProduceResponse
-}
-
-// groups messages together into appropriately-sized batches for sending to the broker
-// handles state related to retries etc
-type brokerProducer struct {
-	parent *asyncProducer
-	broker *Broker
-
-	input     <-chan *ProducerMessage
-	output    chan<- *produceSet
-	responses <-chan *brokerProducerResponse
-
-	buffer     *produceSet
-	timer      <-chan time.Time
-	timerFired bool
-
-	closing        error
-	currentRetries map[string]map[int32]error
-}
-
-func (bp *brokerProducer) run() {
-	var output chan<- *produceSet
-	Logger.Printf("producer/broker/%d starting up\n", bp.broker.ID())
-
-	for {
-		select {
-		case msg := <-bp.input:
-			if msg == nil {
-				bp.shutdown()
-				return
-			}
-
-			if msg.flags&syn == syn {
-				Logger.Printf("producer/broker/%d state change to [open] on %s/%d\n",
-					bp.broker.ID(), msg.Topic, msg.Partition)
-				if bp.currentRetries[msg.Topic] == nil {
-					bp.currentRetries[msg.Topic] = make(map[int32]error)
-				}
-				bp.currentRetries[msg.Topic][msg.Partition] = nil
-				bp.parent.inFlight.Done()
-				continue
-			}
-
-			if reason := bp.needsRetry(msg); reason != nil {
-				bp.parent.retryMessage(msg, reason)
-
-				if bp.closing == nil && msg.flags&fin == fin {
-					// we were retrying this partition but we can start processing again
-					delete(bp.currentRetries[msg.Topic], msg.Partition)
-					Logger.Printf("producer/broker/%d state change to [closed] on %s/%d\n",
-						bp.broker.ID(), msg.Topic, msg.Partition)
-				}
-
-				continue
-			}
-
-			if bp.buffer.wouldOverflow(msg) {
-				if err := bp.waitForSpace(msg); err != nil {
-					bp.parent.retryMessage(msg, err)
-					continue
-				}
-			}
-
-			if err := bp.buffer.add(msg); err != nil {
-				bp.parent.returnError(msg, err)
-				continue
-			}
-
-			if bp.parent.conf.Producer.Flush.Frequency > 0 && bp.timer == nil {
-				bp.timer = time.After(bp.parent.conf.Producer.Flush.Frequency)
-			}
-		case <-bp.timer:
-			bp.timerFired = true
-		case output <- bp.buffer:
-			bp.rollOver()
-		case response := <-bp.responses:
-			bp.handleResponse(response)
-		}
-
-		if bp.timerFired || bp.buffer.readyToFlush() {
-			output = bp.output
-		} else {
-			output = nil
-		}
-	}
-}
-
-func (bp *brokerProducer) shutdown() {
-	for !bp.buffer.empty() {
-		select {
-		case response := <-bp.responses:
-			bp.handleResponse(response)
-		case bp.output <- bp.buffer:
-			bp.rollOver()
-		}
-	}
-	close(bp.output)
-	for response := range bp.responses {
-		bp.handleResponse(response)
-	}
-
-	Logger.Printf("producer/broker/%d shut down\n", bp.broker.ID())
-}
-
-func (bp *brokerProducer) needsRetry(msg *ProducerMessage) error {
-	if bp.closing != nil {
-		return bp.closing
-	}
-
-	return bp.currentRetries[msg.Topic][msg.Partition]
-}
-
-func (bp *brokerProducer) waitForSpace(msg *ProducerMessage) error {
-	Logger.Printf("producer/broker/%d maximum request accumulated, waiting for space\n", bp.broker.ID())
-
-	for {
-		select {
-		case response := <-bp.responses:
-			bp.handleResponse(response)
-			// handling a response can change our state, so re-check some things
-			if reason := bp.needsRetry(msg); reason != nil {
-				return reason
-			} else if !bp.buffer.wouldOverflow(msg) {
-				return nil
-			}
-		case bp.output <- bp.buffer:
-			bp.rollOver()
-			return nil
-		}
-	}
-}
-
-func (bp *brokerProducer) rollOver() {
-	bp.timer = nil
-	bp.timerFired = false
-	bp.buffer = newProduceSet(bp.parent)
-}
-
-func (bp *brokerProducer) handleResponse(response *brokerProducerResponse) {
-	if response.err != nil {
-		bp.handleError(response.set, response.err)
-	} else {
-		bp.handleSuccess(response.set, response.res)
-	}
-
-	if bp.buffer.empty() {
-		bp.rollOver() // this can happen if the response invalidated our buffer
-	}
-}
-
-func (bp *brokerProducer) handleSuccess(sent *produceSet, response *ProduceResponse) {
-	// we iterate through the blocks in the request set, not the response, so that we notice
-	// if the response is missing a block completely
-	sent.eachPartition(func(topic string, partition int32, msgs []*ProducerMessage) {
-		if response == nil {
-			// this only happens when RequiredAcks is NoResponse, so we have to assume success
-			bp.parent.returnSuccesses(msgs)
-			return
-		}
-
-		block := response.GetBlock(topic, partition)
-		if block == nil {
-			bp.parent.returnErrors(msgs, ErrIncompleteResponse)
-			return
-		}
-
-		switch block.Err {
-		// Success
-		case ErrNoError:
-			if bp.parent.conf.Version.IsAtLeast(V0_10_0_0) && !block.Timestamp.IsZero() {
-				for _, msg := range msgs {
-					msg.Timestamp = block.Timestamp
-				}
-			}
-			for i, msg := range msgs {
-				msg.Offset = block.Offset + int64(i)
-			}
-			bp.parent.returnSuccesses(msgs)
-		// Retriable errors
-		case ErrInvalidMessage, ErrUnknownTopicOrPartition, ErrLeaderNotAvailable, ErrNotLeaderForPartition,
-			ErrRequestTimedOut, ErrNotEnoughReplicas, ErrNotEnoughReplicasAfterAppend:
-			Logger.Printf("producer/broker/%d state change to [retrying] on %s/%d because %v\n",
-				bp.broker.ID(), topic, partition, block.Err)
-			bp.currentRetries[topic][partition] = block.Err
-			bp.parent.retryMessages(msgs, block.Err)
-			bp.parent.retryMessages(bp.buffer.dropPartition(topic, partition), block.Err)
-		// Other non-retriable errors
-		default:
-			bp.parent.returnErrors(msgs, block.Err)
-		}
-	})
-}
-
-func (bp *brokerProducer) handleError(sent *produceSet, err error) {
-	switch err.(type) {
-	case PacketEncodingError:
-		sent.eachPartition(func(topic string, partition int32, msgs []*ProducerMessage) {
-			bp.parent.returnErrors(msgs, err)
-		})
-	default:
-		Logger.Printf("producer/broker/%d state change to [closing] because %s\n", bp.broker.ID(), err)
-		bp.parent.abandonBrokerConnection(bp.broker)
-		_ = bp.broker.Close()
-		bp.closing = err
-		sent.eachPartition(func(topic string, partition int32, msgs []*ProducerMessage) {
-			bp.parent.retryMessages(msgs, err)
-		})
-		bp.buffer.eachPartition(func(topic string, partition int32, msgs []*ProducerMessage) {
-			bp.parent.retryMessages(msgs, err)
-		})
-		bp.rollOver()
-	}
-}
-
-// singleton
-// effectively a "bridge" between the flushers and the dispatcher in order to avoid deadlock
-// based on https://godoc.org/github.com/eapache/channels#InfiniteChannel
-func (p *asyncProducer) retryHandler() {
-	var msg *ProducerMessage
-	buf := queue.New()
-
-	for {
-		if buf.Length() == 0 {
-			msg = <-p.retries
-		} else {
-			select {
-			case msg = <-p.retries:
-			case p.input <- buf.Peek().(*ProducerMessage):
-				buf.Remove()
-				continue
-			}
-		}
-
-		if msg == nil {
-			return
-		}
-
-		buf.Add(msg)
-	}
-}
-
-// utility functions
-
-func (p *asyncProducer) shutdown() {
-	Logger.Println("Producer shutting down.")
-	p.inFlight.Add(1)
-	p.input <- &ProducerMessage{flags: shutdown}
-
-	p.inFlight.Wait()
-
-	if p.ownClient {
-		err := p.client.Close()
-		if err != nil {
-			Logger.Println("producer/shutdown failed to close the embedded client:", err)
-		}
-	}
-
-	close(p.input)
-	close(p.retries)
-	close(p.errors)
-	close(p.successes)
-}
-
-func (p *asyncProducer) returnError(msg *ProducerMessage, err error) {
-	msg.clear()
-	pErr := &ProducerError{Msg: msg, Err: err}
-	if p.conf.Producer.Return.Errors {
-		p.errors <- pErr
-	} else {
-		Logger.Println(pErr)
-	}
-	p.inFlight.Done()
-}
-
-func (p *asyncProducer) returnErrors(batch []*ProducerMessage, err error) {
-	for _, msg := range batch {
-		p.returnError(msg, err)
-	}
-}
-
-func (p *asyncProducer) returnSuccesses(batch []*ProducerMessage) {
-	for _, msg := range batch {
-		if p.conf.Producer.Return.Successes {
-			msg.clear()
-			p.successes <- msg
-		}
-		p.inFlight.Done()
-	}
-}
-
-func (p *asyncProducer) retryMessage(msg *ProducerMessage, err error) {
-	if msg.retries >= p.conf.Producer.Retry.Max {
-		p.returnError(msg, err)
-	} else {
-		msg.retries++
-		p.retries <- msg
-	}
-}
-
-func (p *asyncProducer) retryMessages(batch []*ProducerMessage, err error) {
-	for _, msg := range batch {
-		p.retryMessage(msg, err)
-	}
-}
-
-func (p *asyncProducer) getBrokerProducer(broker *Broker) chan<- *ProducerMessage {
-	p.brokerLock.Lock()
-	defer p.brokerLock.Unlock()
-
-	bp := p.brokers[broker]
-
-	if bp == nil {
-		bp = p.newBrokerProducer(broker)
-		p.brokers[broker] = bp
-		p.brokerRefs[bp] = 0
-	}
-
-	p.brokerRefs[bp]++
-
-	return bp
-}
-
-func (p *asyncProducer) unrefBrokerProducer(broker *Broker, bp chan<- *ProducerMessage) {
-	p.brokerLock.Lock()
-	defer p.brokerLock.Unlock()
-
-	p.brokerRefs[bp]--
-	if p.brokerRefs[bp] == 0 {
-		close(bp)
-		delete(p.brokerRefs, bp)
-
-		if p.brokers[broker] == bp {
-			delete(p.brokers, broker)
-		}
-	}
-}
-
-func (p *asyncProducer) abandonBrokerConnection(broker *Broker) {
-	p.brokerLock.Lock()
-	defer p.brokerLock.Unlock()
-
-	delete(p.brokers, broker)
-}

vendor/github.com/Shopify/sarama/async_producer_test.go

@@ -1,841 +0,0 @@
-package sarama
-
-import (
-	"errors"
-	"log"
-	"os"
-	"os/signal"
-	"sync"
-	"testing"
-	"time"
-)
-
-const TestMessage = "ABC THE MESSAGE"
-
-func closeProducer(t *testing.T, p AsyncProducer) {
-	var wg sync.WaitGroup
-	p.AsyncClose()
-
-	wg.Add(2)
-	go func() {
-		for range p.Successes() {
-			t.Error("Unexpected message on Successes()")
-		}
-		wg.Done()
-	}()
-	go func() {
-		for msg := range p.Errors() {
-			t.Error(msg.Err)
-		}
-		wg.Done()
-	}()
-	wg.Wait()
-}
-
-func expectResults(t *testing.T, p AsyncProducer, successes, errors int) {
-	expect := successes + errors
-	for expect > 0 {
-		select {
-		case msg := <-p.Errors():
-			if msg.Msg.flags != 0 {
-				t.Error("Message had flags set")
-			}
-			errors--
-			expect--
-			if errors < 0 {
-				t.Error(msg.Err)
-			}
-		case msg := <-p.Successes():
-			if msg.flags != 0 {
-				t.Error("Message had flags set")
-			}
-			successes--
-			expect--
-			if successes < 0 {
-				t.Error("Too many successes")
-			}
-		}
-	}
-	if successes != 0 || errors != 0 {
-		t.Error("Unexpected successes", successes, "or errors", errors)
-	}
-}
-
-type testPartitioner chan *int32
-
-func (p testPartitioner) Partition(msg *ProducerMessage, numPartitions int32) (int32, error) {
-	part := <-p
-	if part == nil {
-		return 0, errors.New("BOOM")
-	}
-
-	return *part, nil
-}
-
-func (p testPartitioner) RequiresConsistency() bool {
-	return true
-}
-
-func (p testPartitioner) feed(partition int32) {
-	p <- &partition
-}
-
-type flakyEncoder bool
-
-func (f flakyEncoder) Length() int {
-	return len(TestMessage)
-}
-
-func (f flakyEncoder) Encode() ([]byte, error) {
-	if !bool(f) {
-		return nil, errors.New("flaky encoding error")
-	}
-	return []byte(TestMessage), nil
-}
-
-func TestAsyncProducer(t *testing.T) {
-	seedBroker := NewMockBroker(t, 1)
-	leader := NewMockBroker(t, 2)
-
-	metadataResponse := new(MetadataResponse)
-	metadataResponse.AddBroker(leader.Addr(), leader.BrokerID())
-	metadataResponse.AddTopicPartition("my_topic", 0, leader.BrokerID(), nil, nil, ErrNoError)
-	seedBroker.Returns(metadataResponse)
-
-	prodSuccess := new(ProduceResponse)
-	prodSuccess.AddTopicPartition("my_topic", 0, ErrNoError)
-	leader.Returns(prodSuccess)
-
-	config := NewConfig()
-	config.Producer.Flush.Messages = 10
-	config.Producer.Return.Successes = true
-	producer, err := NewAsyncProducer([]string{seedBroker.Addr()}, config)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	for i := 0; i < 10; i++ {
-		producer.Input() <- &ProducerMessage{Topic: "my_topic", Key: nil, Value: StringEncoder(TestMessage), Metadata: i}
-	}
-	for i := 0; i < 10; i++ {
-		select {
-		case msg := <-producer.Errors():
-			t.Error(msg.Err)
-			if msg.Msg.flags != 0 {
-				t.Error("Message had flags set")
-			}
-		case msg := <-producer.Successes():
-			if msg.flags != 0 {
-				t.Error("Message had flags set")
-			}
-			if msg.Metadata.(int) != i {
-				t.Error("Message metadata did not match")
-			}
-		}
-	}
-
-	closeProducer(t, producer)
-	leader.Close()
-	seedBroker.Close()
-}
-
-func TestAsyncProducerMultipleFlushes(t *testing.T) {
-	seedBroker := NewMockBroker(t, 1)
-	leader := NewMockBroker(t, 2)
-
-	metadataResponse := new(MetadataResponse)
-	metadataResponse.AddBroker(leader.Addr(), leader.BrokerID())
-	metadataResponse.AddTopicPartition("my_topic", 0, leader.BrokerID(), nil, nil, ErrNoError)
-	seedBroker.Returns(metadataResponse)
-
-	prodSuccess := new(ProduceResponse)
-	prodSuccess.AddTopicPartition("my_topic", 0, ErrNoError)
-	leader.Returns(prodSuccess)
-	leader.Returns(prodSuccess)
-	leader.Returns(prodSuccess)
-
-	config := NewConfig()
-	config.Producer.Flush.Messages = 5
-	config.Producer.Return.Successes = true
-	producer, err := NewAsyncProducer([]string{seedBroker.Addr()}, config)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	for flush := 0; flush < 3; flush++ {
-		for i := 0; i < 5; i++ {
-			producer.Input() <- &ProducerMessage{Topic: "my_topic", Key: nil, Value: StringEncoder(TestMessage)}
-		}
-		expectResults(t, producer, 5, 0)
-	}
-
-	closeProducer(t, producer)
-	leader.Close()
-	seedBroker.Close()
-}
-
-func TestAsyncProducerMultipleBrokers(t *testing.T) {
-	seedBroker := NewMockBroker(t, 1)
-	leader0 := NewMockBroker(t, 2)
-	leader1 := NewMockBroker(t, 3)
-
-	metadataResponse := new(MetadataResponse)
-	metadataResponse.AddBroker(leader0.Addr(), leader0.BrokerID())
-	metadataResponse.AddBroker(leader1.Addr(), leader1.BrokerID())
-	metadataResponse.AddTopicPartition("my_topic", 0, leader0.BrokerID(), nil, nil, ErrNoError)
-	metadataResponse.AddTopicPartition("my_topic", 1, leader1.BrokerID(), nil, nil, ErrNoError)
-	seedBroker.Returns(metadataResponse)
-
-	prodResponse0 := new(ProduceResponse)
-	prodResponse0.AddTopicPartition("my_topic", 0, ErrNoError)
-	leader0.Returns(prodResponse0)
-
-	prodResponse1 := new(ProduceResponse)
-	prodResponse1.AddTopicPartition("my_topic", 1, ErrNoError)
-	leader1.Returns(prodResponse1)
-
-	config := NewConfig()
-	config.Producer.Flush.Messages = 5
-	config.Producer.Return.Successes = true
-	config.Producer.Partitioner = NewRoundRobinPartitioner
-	producer, err := NewAsyncProducer([]string{seedBroker.Addr()}, config)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	for i := 0; i < 10; i++ {
-		producer.Input() <- &ProducerMessage{Topic: "my_topic", Key: nil, Value: StringEncoder(TestMessage)}
-	}
-	expectResults(t, producer, 10, 0)
-
-	closeProducer(t, producer)
-	leader1.Close()
-	leader0.Close()
-	seedBroker.Close()
-}
-
-func TestAsyncProducerCustomPartitioner(t *testing.T) {
-	seedBroker := NewMockBroker(t, 1)
-	leader := NewMockBroker(t, 2)
-
-	metadataResponse := new(MetadataResponse)
-	metadataResponse.AddBroker(leader.Addr(), leader.BrokerID())
-	metadataResponse.AddTopicPartition("my_topic", 0, leader.BrokerID(), nil, nil, ErrNoError)
-	seedBroker.Returns(metadataResponse)
-
-	prodResponse := new(ProduceResponse)
-	prodResponse.AddTopicPartition("my_topic", 0, ErrNoError)
-	leader.Returns(prodResponse)
-
-	config := NewConfig()
-	config.Producer.Flush.Messages = 2
-	config.Producer.Return.Successes = true
-	config.Producer.Partitioner = func(topic string) Partitioner {
-		p := make(testPartitioner)
-		go func() {
-			p.feed(0)
-			p <- nil
-			p <- nil
-			p <- nil
-			p.feed(0)
-		}()
-		return p
-	}
-	producer, err := NewAsyncProducer([]string{seedBroker.Addr()}, config)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	for i := 0; i < 5; i++ {
-		producer.Input() <- &ProducerMessage{Topic: "my_topic", Key: nil, Value: StringEncoder(TestMessage)}
-	}
-	expectResults(t, producer, 2, 3)
-
-	closeProducer(t, producer)
-	leader.Close()
-	seedBroker.Close()
-}
-
-func TestAsyncProducerFailureRetry(t *testing.T) {
-	seedBroker := NewMockBroker(t, 1)
-	leader1 := NewMockBroker(t, 2)
-	leader2 := NewMockBroker(t, 3)
-
-	metadataLeader1 := new(MetadataResponse)
-	metadataLeader1.AddBroker(leader1.Addr(), leader1.BrokerID())
-	metadataLeader1.AddTopicPartition("my_topic", 0, leader1.BrokerID(), nil, nil, ErrNoError)
-	seedBroker.Returns(metadataLeader1)
-
-	config := NewConfig()
-	config.Producer.Flush.Messages = 10
-	config.Producer.Return.Successes = true
-	config.Producer.Retry.Backoff = 0
-	producer, err := NewAsyncProducer([]string{seedBroker.Addr()}, config)
-	if err != nil {
-		t.Fatal(err)
-	}
-	seedBroker.Close()
-
-	for i := 0; i < 10; i++ {
-		producer.Input() <- &ProducerMessage{Topic: "my_topic", Key: nil, Value: StringEncoder(TestMessage)}
-	}
-	prodNotLeader := new(ProduceResponse)
-	prodNotLeader.AddTopicPartition("my_topic", 0, ErrNotLeaderForPartition)
-	leader1.Returns(prodNotLeader)
-
-	metadataLeader2 := new(MetadataResponse)
-	metadataLeader2.AddBroker(leader2.Addr(), leader2.BrokerID())
-	metadataLeader2.AddTopicPartition("my_topic", 0, leader2.BrokerID(), nil, nil, ErrNoError)
-	leader1.Returns(metadataLeader2)
-
-	prodSuccess := new(ProduceResponse)
-	prodSuccess.AddTopicPartition("my_topic", 0, ErrNoError)
-	leader2.Returns(prodSuccess)
-	expectResults(t, producer, 10, 0)
-	leader1.Close()
-
-	for i := 0; i < 10; i++ {
-		producer.Input() <- &ProducerMessage{Topic: "my_topic", Key: nil, Value: StringEncoder(TestMessage)}
-	}
-	leader2.Returns(prodSuccess)
-	expectResults(t, producer, 10, 0)
-
-	leader2.Close()
-	closeProducer(t, producer)
-}
-
-func TestAsyncProducerEncoderFailures(t *testing.T) {
-	seedBroker := NewMockBroker(t, 1)
-	leader := NewMockBroker(t, 2)
-
-	metadataResponse := new(MetadataResponse)
-	metadataResponse.AddBroker(leader.Addr(), leader.BrokerID())
-	metadataResponse.AddTopicPartition("my_topic", 0, leader.BrokerID(), nil, nil, ErrNoError)
-	seedBroker.Returns(metadataResponse)
-
-	prodSuccess := new(ProduceResponse)
-	prodSuccess.AddTopicPartition("my_topic", 0, ErrNoError)
-	leader.Returns(prodSuccess)
-	leader.Returns(prodSuccess)
-	leader.Returns(prodSuccess)
-
-	config := NewConfig()
-	config.Producer.Flush.Messages = 1
-	config.Producer.Return.Successes = true
-	config.Producer.Partitioner = NewManualPartitioner
-	producer, err := NewAsyncProducer([]string{seedBroker.Addr()}, config)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	for flush := 0; flush < 3; flush++ {
-		producer.Input() <- &ProducerMessage{Topic: "my_topic", Key: flakyEncoder(true), Value: flakyEncoder(false)}
-		producer.Input() <- &ProducerMessage{Topic: "my_topic", Key: flakyEncoder(false), Value: flakyEncoder(true)}
-		producer.Input() <- &ProducerMessage{Topic: "my_topic", Key: flakyEncoder(true), Value: flakyEncoder(true)}
-		expectResults(t, producer, 1, 2)
-	}
-
-	closeProducer(t, producer)
-	leader.Close()
-	seedBroker.Close()
-}
-
-// If a Kafka broker becomes unavailable and then returns back in service, then
-// producer reconnects to it and continues sending messages.
-func TestAsyncProducerBrokerBounce(t *testing.T) {
-	// Given
-	seedBroker := NewMockBroker(t, 1)
-	leader := NewMockBroker(t, 2)
-	leaderAddr := leader.Addr()
-
-	metadataResponse := new(MetadataResponse)
-	metadataResponse.AddBroker(leaderAddr, leader.BrokerID())
-	metadataResponse.AddTopicPartition("my_topic", 0, leader.BrokerID(), nil, nil, ErrNoError)
-	seedBroker.Returns(metadataResponse)
-
-	prodSuccess := new(ProduceResponse)
-	prodSuccess.AddTopicPartition("my_topic", 0, ErrNoError)
-
-	config := NewConfig()
-	config.Producer.Flush.Messages = 1
-	config.Producer.Return.Successes = true
-	config.Producer.Retry.Backoff = 0
-	producer, err := NewAsyncProducer([]string{seedBroker.Addr()}, config)
-	if err != nil {
-		t.Fatal(err)
-	}
-	producer.Input() <- &ProducerMessage{Topic: "my_topic", Key: nil, Value: StringEncoder(TestMessage)}
-	leader.Returns(prodSuccess)
-	expectResults(t, producer, 1, 0)
-
-	// When: a broker connection gets reset by a broker (network glitch, restart, you name it).
-	leader.Close()                               // producer should get EOF
-	leader = NewMockBrokerAddr(t, 2, leaderAddr) // start it up again right away for giggles
-	seedBroker.Returns(metadataResponse)         // tell it to go to broker 2 again
-
-	// Then: a produced message goes through the new broker connection.
-	producer.Input() <- &ProducerMessage{Topic: "my_topic", Key: nil, Value: StringEncoder(TestMessage)}
-	leader.Returns(prodSuccess)
-	expectResults(t, producer, 1, 0)
-
-	closeProducer(t, producer)
-	seedBroker.Close()
-	leader.Close()
-}
-
-func TestAsyncProducerBrokerBounceWithStaleMetadata(t *testing.T) {
-	seedBroker := NewMockBroker(t, 1)
-	leader1 := NewMockBroker(t, 2)
-	leader2 := NewMockBroker(t, 3)
-
-	metadataLeader1 := new(MetadataResponse)
-	metadataLeader1.AddBroker(leader1.Addr(), leader1.BrokerID())
-	metadataLeader1.AddTopicPartition("my_topic", 0, leader1.BrokerID(), nil, nil, ErrNoError)
-	seedBroker.Returns(metadataLeader1)
-
-	config := NewConfig()
-	config.Producer.Flush.Messages = 10
-	config.Producer.Return.Successes = true
-	config.Producer.Retry.Max = 3
-	config.Producer.Retry.Backoff = 0
-	producer, err := NewAsyncProducer([]string{seedBroker.Addr()}, config)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	for i := 0; i < 10; i++ {
-		producer.Input() <- &ProducerMessage{Topic: "my_topic", Key: nil, Value: StringEncoder(TestMessage)}
-	}
-	leader1.Close()                     // producer should get EOF
-	seedBroker.Returns(metadataLeader1) // tell it to go to leader1 again even though it's still down
-	seedBroker.Returns(metadataLeader1) // tell it to go to leader1 again even though it's still down
-
-	// ok fine, tell it to go to leader2 finally
-	metadataLeader2 := new(MetadataResponse)
-	metadataLeader2.AddBroker(leader2.Addr(), leader2.BrokerID())
-	metadataLeader2.AddTopicPartition("my_topic", 0, leader2.BrokerID(), nil, nil, ErrNoError)
-	seedBroker.Returns(metadataLeader2)
-
-	prodSuccess := new(ProduceResponse)
-	prodSuccess.AddTopicPartition("my_topic", 0, ErrNoError)
-	leader2.Returns(prodSuccess)
-	expectResults(t, producer, 10, 0)
-	seedBroker.Close()
-	leader2.Close()
-
-	closeProducer(t, producer)
-}
-
-func TestAsyncProducerMultipleRetries(t *testing.T) {
-	seedBroker := NewMockBroker(t, 1)
-	leader1 := NewMockBroker(t, 2)
-	leader2 := NewMockBroker(t, 3)
-
-	metadataLeader1 := new(MetadataResponse)
-	metadataLeader1.AddBroker(leader1.Addr(), leader1.BrokerID())
-	metadataLeader1.AddTopicPartition("my_topic", 0, leader1.BrokerID(), nil, nil, ErrNoError)
-	seedBroker.Returns(metadataLeader1)
-
-	config := NewConfig()
-	config.Producer.Flush.Messages = 10
-	config.Producer.Return.Successes = true
-	config.Producer.Retry.Max = 4
-	config.Producer.Retry.Backoff = 0
-	producer, err := NewAsyncProducer([]string{seedBroker.Addr()}, config)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	for i := 0; i < 10; i++ {
-		producer.Input() <- &ProducerMessage{Topic: "my_topic", Key: nil, Value: StringEncoder(TestMessage)}
-	}
-	prodNotLeader := new(ProduceResponse)
-	prodNotLeader.AddTopicPartition("my_topic", 0, ErrNotLeaderForPartition)
-	leader1.Returns(prodNotLeader)
-
-	metadataLeader2 := new(MetadataResponse)
-	metadataLeader2.AddBroker(leader2.Addr(), leader2.BrokerID())
-	metadataLeader2.AddTopicPartition("my_topic", 0, leader2.BrokerID(), nil, nil, ErrNoError)
-	seedBroker.Returns(metadataLeader2)
-	leader2.Returns(prodNotLeader)
-	seedBroker.Returns(metadataLeader1)
-	leader1.Returns(prodNotLeader)
-	seedBroker.Returns(metadataLeader1)
-	leader1.Returns(prodNotLeader)
-	seedBroker.Returns(metadataLeader2)
-
-	prodSuccess := new(ProduceResponse)
-	prodSuccess.AddTopicPartition("my_topic", 0, ErrNoError)
-	leader2.Returns(prodSuccess)
-	expectResults(t, producer, 10, 0)
-
-	for i := 0; i < 10; i++ {
-		producer.Input() <- &ProducerMessage{Topic: "my_topic", Key: nil, Value: StringEncoder(TestMessage)}
-	}
-	leader2.Returns(prodSuccess)
-	expectResults(t, producer, 10, 0)
-
-	seedBroker.Close()
-	leader1.Close()
-	leader2.Close()
-	closeProducer(t, producer)
-}
-
-func TestAsyncProducerOutOfRetries(t *testing.T) {
-	t.Skip("Enable once bug #294 is fixed.")
-
-	seedBroker := NewMockBroker(t, 1)
-	leader := NewMockBroker(t, 2)
-
-	metadataResponse := new(MetadataResponse)
-	metadataResponse.AddBroker(leader.Addr(), leader.BrokerID())
-	metadataResponse.AddTopicPartition("my_topic", 0, leader.BrokerID(), nil, nil, ErrNoError)
-	seedBroker.Returns(metadataResponse)
-
-	config := NewConfig()
-	config.Producer.Flush.Messages = 10
-	config.Producer.Return.Successes = true
-	config.Producer.Retry.Backoff = 0
-	config.Producer.Retry.Max = 0
-	producer, err := NewAsyncProducer([]string{seedBroker.Addr()}, config)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	for i := 0; i < 10; i++ {
-		producer.Input() <- &ProducerMessage{Topic: "my_topic", Key: nil, Value: StringEncoder(TestMessage)}
-	}
-
-	prodNotLeader := new(ProduceResponse)
-	prodNotLeader.AddTopicPartition("my_topic", 0, ErrNotLeaderForPartition)
-	leader.Returns(prodNotLeader)
-
-	for i := 0; i < 10; i++ {
-		select {
-		case msg := <-producer.Errors():
-			if msg.Err != ErrNotLeaderForPartition {
-				t.Error(msg.Err)
-			}
-		case <-producer.Successes():
-			t.Error("Unexpected success")
-		}
-	}
-
-	seedBroker.Returns(metadataResponse)
-
-	for i := 0; i < 10; i++ {
-		producer.Input() <- &ProducerMessage{Topic: "my_topic", Key: nil, Value: StringEncoder(TestMessage)}
-	}
-
-	prodSuccess := new(ProduceResponse)
-	prodSuccess.AddTopicPartition("my_topic", 0, ErrNoError)
-	leader.Returns(prodSuccess)
-
-	expectResults(t, producer, 10, 0)
-
-	leader.Close()
-	seedBroker.Close()
-	safeClose(t, producer)
-}
-
-func TestAsyncProducerRetryWithReferenceOpen(t *testing.T) {
-	seedBroker := NewMockBroker(t, 1)
-	leader := NewMockBroker(t, 2)
-	leaderAddr := leader.Addr()
-
-	metadataResponse := new(MetadataResponse)
-	metadataResponse.AddBroker(leaderAddr, leader.BrokerID())
-	metadataResponse.AddTopicPartition("my_topic", 0, leader.BrokerID(), nil, nil, ErrNoError)
-	metadataResponse.AddTopicPartition("my_topic", 1, leader.BrokerID(), nil, nil, ErrNoError)
-	seedBroker.Returns(metadataResponse)
-
-	config := NewConfig()
-	config.Producer.Return.Successes = true
-	config.Producer.Retry.Backoff = 0
-	config.Producer.Retry.Max = 1
-	config.Producer.Partitioner = NewRoundRobinPartitioner
-	producer, err := NewAsyncProducer([]string{seedBroker.Addr()}, config)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// prime partition 0
-	producer.Input() <- &ProducerMessage{Topic: "my_topic", Key: nil, Value: StringEncoder(TestMessage)}
-	prodSuccess := new(ProduceResponse)
-	prodSuccess.AddTopicPartition("my_topic", 0, ErrNoError)
-	leader.Returns(prodSuccess)
-	expectResults(t, producer, 1, 0)
-
-	// prime partition 1
-	producer.Input() <- &ProducerMessage{Topic: "my_topic", Key: nil, Value: StringEncoder(TestMessage)}
-	prodSuccess = new(ProduceResponse)
-	prodSuccess.AddTopicPartition("my_topic", 1, ErrNoError)
-	leader.Returns(prodSuccess)
-	expectResults(t, producer, 1, 0)
-
-	// reboot the broker (the producer will get EOF on its existing connection)
-	leader.Close()
-	leader = NewMockBrokerAddr(t, 2, leaderAddr)
-
-	// send another message on partition 0 to trigger the EOF and retry
-	producer.Input() <- &ProducerMessage{Topic: "my_topic", Key: nil, Value: StringEncoder(TestMessage)}
-
-	// tell partition 0 to go to that broker again
-	seedBroker.Returns(metadataResponse)
-
-	// succeed this time
-	prodSuccess = new(ProduceResponse)
-	prodSuccess.AddTopicPartition("my_topic", 0, ErrNoError)
-	leader.Returns(prodSuccess)
-	expectResults(t, producer, 1, 0)
-
-	// shutdown
-	closeProducer(t, producer)
-	seedBroker.Close()
-	leader.Close()
-}
-
-func TestAsyncProducerFlusherRetryCondition(t *testing.T) {
-	seedBroker := NewMockBroker(t, 1)
-	leader := NewMockBroker(t, 2)
-
-	metadataResponse := new(MetadataResponse)
-	metadataResponse.AddBroker(leader.Addr(), leader.BrokerID())
-	metadataResponse.AddTopicPartition("my_topic", 0, leader.BrokerID(), nil, nil, ErrNoError)
-	metadataResponse.AddTopicPartition("my_topic", 1, leader.BrokerID(), nil, nil, ErrNoError)
-	seedBroker.Returns(metadataResponse)
-
-	config := NewConfig()
-	config.Producer.Flush.Messages = 5
-	config.Producer.Return.Successes = true
-	config.Producer.Retry.Backoff = 0
-	config.Producer.Retry.Max = 1
-	config.Producer.Partitioner = NewManualPartitioner
-	producer, err := NewAsyncProducer([]string{seedBroker.Addr()}, config)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// prime partitions
-	for p := int32(0); p < 2; p++ {
-		for i := 0; i < 5; i++ {
-			producer.Input() <- &ProducerMessage{Topic: "my_topic", Key: nil, Value: StringEncoder(TestMessage), Partition: p}
-		}
-		prodSuccess := new(ProduceResponse)
-		prodSuccess.AddTopicPartition("my_topic", p, ErrNoError)
-		leader.Returns(prodSuccess)
-		expectResults(t, producer, 5, 0)
-	}
-
-	// send more messages on partition 0
-	for i := 0; i < 5; i++ {
-		producer.Input() <- &ProducerMessage{Topic: "my_topic", Key: nil, Value: StringEncoder(TestMessage), Partition: 0}
-	}
-	prodNotLeader := new(ProduceResponse)
-	prodNotLeader.AddTopicPartition("my_topic", 0, ErrNotLeaderForPartition)
-	leader.Returns(prodNotLeader)
-
-	time.Sleep(50 * time.Millisecond)
-
-	leader.SetHandlerByMap(map[string]MockResponse{
-		"ProduceRequest": NewMockProduceResponse(t).
-			SetError("my_topic", 0, ErrNoError),
-	})
-
-	// tell partition 0 to go to that broker again
-	seedBroker.Returns(metadataResponse)
-
-	// succeed this time
-	expectResults(t, producer, 5, 0)
-
-	// put five more through
-	for i := 0; i < 5; i++ {
-		producer.Input() <- &ProducerMessage{Topic: "my_topic", Key: nil, Value: StringEncoder(TestMessage), Partition: 0}
-	}
-	expectResults(t, producer, 5, 0)
-
-	// shutdown
-	closeProducer(t, producer)
-	seedBroker.Close()
-	leader.Close()
-}
-
-func TestAsyncProducerRetryShutdown(t *testing.T) {
-	seedBroker := NewMockBroker(t, 1)
-	leader := NewMockBroker(t, 2)
-
-	metadataLeader := new(MetadataResponse)
-	metadataLeader.AddBroker(leader.Addr(), leader.BrokerID())
-	metadataLeader.AddTopicPartition("my_topic", 0, leader.BrokerID(), nil, nil, ErrNoError)
-	seedBroker.Returns(metadataLeader)
-
-	config := NewConfig()
-	config.Producer.Flush.Messages = 10
-	config.Producer.Return.Successes = true
-	config.Producer.Retry.Backoff = 0
-	producer, err := NewAsyncProducer([]string{seedBroker.Addr()}, config)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	for i := 0; i < 10; i++ {
-		producer.Input() <- &ProducerMessage{Topic: "my_topic", Key: nil, Value: StringEncoder(TestMessage)}
-	}
-	producer.AsyncClose()
-	time.Sleep(5 * time.Millisecond) // let the shutdown goroutine kick in
-
-	producer.Input() <- &ProducerMessage{Topic: "FOO"}
-	if err := <-producer.Errors(); err.Err != ErrShuttingDown {
-		t.Error(err)
-	}
-
-	prodNotLeader := new(ProduceResponse)
-	prodNotLeader.AddTopicPartition("my_topic", 0, ErrNotLeaderForPartition)
-	leader.Returns(prodNotLeader)
-
-	seedBroker.Returns(metadataLeader)
-
-	prodSuccess := new(ProduceResponse)
-	prodSuccess.AddTopicPartition("my_topic", 0, ErrNoError)
-	leader.Returns(prodSuccess)
-	expectResults(t, producer, 10, 0)
-
-	seedBroker.Close()
-	leader.Close()
-
-	// wait for the async-closed producer to shut down fully
-	for err := range producer.Errors() {
-		t.Error(err)
-	}
-}
-
-func TestAsyncProducerNoReturns(t *testing.T) {
-	seedBroker := NewMockBroker(t, 1)
-	leader := NewMockBroker(t, 2)
-
-	metadataLeader := new(MetadataResponse)
-	metadataLeader.AddBroker(leader.Addr(), leader.BrokerID())
-	metadataLeader.AddTopicPartition("my_topic", 0, leader.BrokerID(), nil, nil, ErrNoError)
-	seedBroker.Returns(metadataLeader)
-
-	config := NewConfig()
-	config.Producer.Flush.Messages = 10
-	config.Producer.Return.Successes = false
-	config.Producer.Return.Errors = false
-	config.Producer.Retry.Backoff = 0
-	producer, err := NewAsyncProducer([]string{seedBroker.Addr()}, config)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	for i := 0; i < 10; i++ {
-		producer.Input() <- &ProducerMessage{Topic: "my_topic", Key: nil, Value: StringEncoder(TestMessage)}
-	}
-
-	wait := make(chan bool)
-	go func() {
-		if err := producer.Close(); err != nil {
-			t.Error(err)
-		}
-		close(wait)
-	}()
-
-	prodSuccess := new(ProduceResponse)
-	prodSuccess.AddTopicPartition("my_topic", 0, ErrNoError)
-	leader.Returns(prodSuccess)
-
-	<-wait
-	seedBroker.Close()
-	leader.Close()
-}
-
-// This example shows how to use the producer while simultaneously
-// reading the Errors channel to know about any failures.
-func ExampleAsyncProducer_select() {
-	producer, err := NewAsyncProducer([]string{"localhost:9092"}, nil)
-	if err != nil {
-		panic(err)
-	}
-
-	defer func() {
-		if err := producer.Close(); err != nil {
-			log.Fatalln(err)
-		}
-	}()
-
-	// Trap SIGINT to trigger a shutdown.
-	signals := make(chan os.Signal, 1)
-	signal.Notify(signals, os.Interrupt)
-
-	var enqueued, errors int
-ProducerLoop:
-	for {
-		select {
-		case producer.Input() <- &ProducerMessage{Topic: "my_topic", Key: nil, Value: StringEncoder("testing 123")}:
-			enqueued++
-		case err := <-producer.Errors():
-			log.Println("Failed to produce message", err)
-			errors++
-		case <-signals:
-			break ProducerLoop
-		}
-	}
-
-	log.Printf("Enqueued: %d; errors: %d\n", enqueued, errors)
-}
-
-// This example shows how to use the producer with separate goroutines
-// reading from the Successes and Errors channels. Note that in order
-// for the Successes channel to be populated, you have to set
-// config.Producer.Return.Successes to true.
-func ExampleAsyncProducer_goroutines() {
-	config := NewConfig()
-	config.Producer.Return.Successes = true
-	producer, err := NewAsyncProducer([]string{"localhost:9092"}, config)
-	if err != nil {
-		panic(err)
-	}
-
-	// Trap SIGINT to trigger a graceful shutdown.
-	signals := make(chan os.Signal, 1)
-	signal.Notify(signals, os.Interrupt)
-
-	var (
-		wg                          sync.WaitGroup
-		enqueued, successes, errors int
-	)
-
-	wg.Add(1)
-	go func() {
-		defer wg.Done()
-		for range producer.Successes() {
-			successes++
-		}
-	}()
-
-	wg.Add(1)
-	go func() {
-		defer wg.Done()
-		for err := range producer.Errors() {
-			log.Println(err)
-			errors++
-		}
-	}()
-
-ProducerLoop:
-	for {
-		message := &ProducerMessage{Topic: "my_topic", Value: StringEncoder("testing 123")}
-		select {
-		case producer.Input() <- message:
-			enqueued++
-
-		case <-signals:
-			producer.AsyncClose() // Trigger a shutdown of the producer.
-			break ProducerLoop
-		}
-	}
-
-	wg.Wait()
-
-	log.Printf("Successfully produced: %d; errors: %d\n", successes, errors)
-}

vendor/github.com/Shopify/sarama/broker.go

@@ -1,692 +0,0 @@
-package sarama
-
-import (
-	"crypto/tls"
-	"encoding/binary"
-	"fmt"
-	"io"
-	"net"
-	"strconv"
-	"sync"
-	"sync/atomic"
-	"time"
-
-	"github.com/rcrowley/go-metrics"
-)
-
-// Broker represents a single Kafka broker connection. All operations on this object are entirely concurrency-safe.
-type Broker struct {
-	id   int32
-	addr string
-
-	conf          *Config
-	correlationID int32
-	conn          net.Conn
-	connErr       error
-	lock          sync.Mutex
-	opened        int32
-
-	responses chan responsePromise
-	done      chan bool
-
-	incomingByteRate       metrics.Meter
-	requestRate            metrics.Meter
-	requestSize            metrics.Histogram
-	requestLatency         metrics.Histogram
-	outgoingByteRate       metrics.Meter
-	responseRate           metrics.Meter
-	responseSize           metrics.Histogram
-	brokerIncomingByteRate metrics.Meter
-	brokerRequestRate      metrics.Meter
-	brokerRequestSize      metrics.Histogram
-	brokerRequestLatency   metrics.Histogram
-	brokerOutgoingByteRate metrics.Meter
-	brokerResponseRate     metrics.Meter
-	brokerResponseSize     metrics.Histogram
-}
-
-type responsePromise struct {
-	requestTime   time.Time
-	correlationID int32
-	packets       chan []byte
-	errors        chan error
-}
-
-// NewBroker creates and returns a Broker targeting the given host:port address.
-// This does not attempt to actually connect, you have to call Open() for that.
-func NewBroker(addr string) *Broker {
-	return &Broker{id: -1, addr: addr}
-}
-
-// Open tries to connect to the Broker if it is not already connected or connecting, but does not block
-// waiting for the connection to complete. This means that any subsequent operations on the broker will
-// block waiting for the connection to succeed or fail. To get the effect of a fully synchronous Open call,
-// follow it by a call to Connected(). The only errors Open will return directly are ConfigurationError or
-// AlreadyConnected. If conf is nil, the result of NewConfig() is used.
-func (b *Broker) Open(conf *Config) error {
-	if !atomic.CompareAndSwapInt32(&b.opened, 0, 1) {
-		return ErrAlreadyConnected
-	}
-
-	if conf == nil {
-		conf = NewConfig()
-	}
-
-	err := conf.Validate()
-	if err != nil {
-		return err
-	}
-
-	b.lock.Lock()
-
-	go withRecover(func() {
-		defer b.lock.Unlock()
-
-		dialer := net.Dialer{
-			Timeout:   conf.Net.DialTimeout,
-			KeepAlive: conf.Net.KeepAlive,
-		}
-
-		if conf.Net.TLS.Enable {
-			b.conn, b.connErr = tls.DialWithDialer(&dialer, "tcp", b.addr, conf.Net.TLS.Config)
-		} else {
-			b.conn, b.connErr = dialer.Dial("tcp", b.addr)
-		}
-		if b.connErr != nil {
-			Logger.Printf("Failed to connect to broker %s: %s\n", b.addr, b.connErr)
-			b.conn = nil
-			atomic.StoreInt32(&b.opened, 0)
-			return
-		}
-		b.conn = newBufConn(b.conn)
-
-		b.conf = conf
-
-		// Create or reuse the global metrics shared between brokers
-		b.incomingByteRate = metrics.GetOrRegisterMeter("incoming-byte-rate", conf.MetricRegistry)
-		b.requestRate = metrics.GetOrRegisterMeter("request-rate", conf.MetricRegistry)
-		b.requestSize = getOrRegisterHistogram("request-size", conf.MetricRegistry)
-		b.requestLatency = getOrRegisterHistogram("request-latency-in-ms", conf.MetricRegistry)
-		b.outgoingByteRate = metrics.GetOrRegisterMeter("outgoing-byte-rate", conf.MetricRegistry)
-		b.responseRate = metrics.GetOrRegisterMeter("response-rate", conf.MetricRegistry)
-		b.responseSize = getOrRegisterHistogram("response-size", conf.MetricRegistry)
-		// Do not gather metrics for seeded broker (only used during bootstrap) because they share
-		// the same id (-1) and are already exposed through the global metrics above
-		if b.id >= 0 {
-			b.brokerIncomingByteRate = getOrRegisterBrokerMeter("incoming-byte-rate", b, conf.MetricRegistry)
-			b.brokerRequestRate = getOrRegisterBrokerMeter("request-rate", b, conf.MetricRegistry)
-			b.brokerRequestSize = getOrRegisterBrokerHistogram("request-size", b, conf.MetricRegistry)
-			b.brokerRequestLatency = getOrRegisterBrokerHistogram("request-latency-in-ms", b, conf.MetricRegistry)
-			b.brokerOutgoingByteRate = getOrRegisterBrokerMeter("outgoing-byte-rate", b, conf.MetricRegistry)
-			b.brokerResponseRate = getOrRegisterBrokerMeter("response-rate", b, conf.MetricRegistry)
-			b.brokerResponseSize = getOrRegisterBrokerHistogram("response-size", b, conf.MetricRegistry)
-		}
-
-		if conf.Net.SASL.Enable {
-			b.connErr = b.sendAndReceiveSASLPlainAuth()
-			if b.connErr != nil {
-				err = b.conn.Close()
-				if err == nil {
-					Logger.Printf("Closed connection to broker %s\n", b.addr)
-				} else {
-					Logger.Printf("Error while closing connection to broker %s: %s\n", b.addr, err)
-				}
-				b.conn = nil
-				atomic.StoreInt32(&b.opened, 0)
-				return
-			}
-		}
-
-		b.done = make(chan bool)
-		b.responses = make(chan responsePromise, b.conf.Net.MaxOpenRequests-1)
-
-		if b.id >= 0 {
-			Logger.Printf("Connected to broker at %s (registered as #%d)\n", b.addr, b.id)
-		} else {
-			Logger.Printf("Connected to broker at %s (unregistered)\n", b.addr)
-		}
-		go withRecover(b.responseReceiver)
-	})
-
-	return nil
-}
-
-// Connected returns true if the broker is connected and false otherwise. If the broker is not
-// connected but it had tried to connect, the error from that connection attempt is also returned.
-func (b *Broker) Connected() (bool, error) {
-	b.lock.Lock()
-	defer b.lock.Unlock()
-
-	return b.conn != nil, b.connErr
-}
-
-func (b *Broker) Close() error {
-	b.lock.Lock()
-	defer b.lock.Unlock()
-
-	if b.conn == nil {
-		return ErrNotConnected
-	}
-
-	close(b.responses)
-	<-b.done
-
-	err := b.conn.Close()
-
-	b.conn = nil
-	b.connErr = nil
-	b.done = nil
-	b.responses = nil
-
-	if b.id >= 0 {
-		b.conf.MetricRegistry.Unregister(getMetricNameForBroker("incoming-byte-rate", b))
-		b.conf.MetricRegistry.Unregister(getMetricNameForBroker("request-rate", b))
-		b.conf.MetricRegistry.Unregister(getMetricNameForBroker("outgoing-byte-rate", b))
-		b.conf.MetricRegistry.Unregister(getMetricNameForBroker("response-rate", b))
-	}
-
-	if err == nil {
-		Logger.Printf("Closed connection to broker %s\n", b.addr)
-	} else {
-		Logger.Printf("Error while closing connection to broker %s: %s\n", b.addr, err)
-	}
-
-	atomic.StoreInt32(&b.opened, 0)
-
-	return err
-}
-
-// ID returns the broker ID retrieved from Kafka's metadata, or -1 if that is not known.
-func (b *Broker) ID() int32 {
-	return b.id
-}
-
-// Addr returns the broker address as either retrieved from Kafka's metadata or passed to NewBroker.
-func (b *Broker) Addr() string {
-	return b.addr
-}
-
-func (b *Broker) GetMetadata(request *MetadataRequest) (*MetadataResponse, error) {
-	response := new(MetadataResponse)
-
-	err := b.sendAndReceive(request, response)
-
-	if err != nil {
-		return nil, err
-	}
-
-	return response, nil
-}
-
-func (b *Broker) GetConsumerMetadata(request *ConsumerMetadataRequest) (*ConsumerMetadataResponse, error) {
-	response := new(ConsumerMetadataResponse)
-
-	err := b.sendAndReceive(request, response)
-
-	if err != nil {
-		return nil, err
-	}
-
-	return response, nil
-}
-
-func (b *Broker) GetAvailableOffsets(request *OffsetRequest) (*OffsetResponse, error) {
-	response := new(OffsetResponse)
-
-	err := b.sendAndReceive(request, response)
-
-	if err != nil {
-		return nil, err
-	}
-
-	return response, nil
-}
-
-func (b *Broker) Produce(request *ProduceRequest) (*ProduceResponse, error) {
-	var response *ProduceResponse
-	var err error
-
-	if request.RequiredAcks == NoResponse {
-		err = b.sendAndReceive(request, nil)
-	} else {
-		response = new(ProduceResponse)
-		err = b.sendAndReceive(request, response)
-	}
-
-	if err != nil {
-		return nil, err
-	}
-
-	return response, nil
-}
-
-func (b *Broker) Fetch(request *FetchRequest) (*FetchResponse, error) {
-	response := new(FetchResponse)
-
-	err := b.sendAndReceive(request, response)
-
-	if err != nil {
-		return nil, err
-	}
-
-	return response, nil
-}
-
-func (b *Broker) CommitOffset(request *OffsetCommitRequest) (*OffsetCommitResponse, error) {
-	response := new(OffsetCommitResponse)
-
-	err := b.sendAndReceive(request, response)
-
-	if err != nil {
-		return nil, err
-	}
-
-	return response, nil
-}
-
-func (b *Broker) FetchOffset(request *OffsetFetchRequest) (*OffsetFetchResponse, error) {
-	response := new(OffsetFetchResponse)
-
-	err := b.sendAndReceive(request, response)
-
-	if err != nil {
-		return nil, err
-	}
-
-	return response, nil
-}
-
-func (b *Broker) JoinGroup(request *JoinGroupRequest) (*JoinGroupResponse, error) {
-	response := new(JoinGroupResponse)
-
-	err := b.sendAndReceive(request, response)
-	if err != nil {
-		return nil, err
-	}
-
-	return response, nil
-}
-
-func (b *Broker) SyncGroup(request *SyncGroupRequest) (*SyncGroupResponse, error) {
-	response := new(SyncGroupResponse)
-
-	err := b.sendAndReceive(request, response)
-	if err != nil {
-		return nil, err
-	}
-
-	return response, nil
-}
-
-func (b *Broker) LeaveGroup(request *LeaveGroupRequest) (*LeaveGroupResponse, error) {
-	response := new(LeaveGroupResponse)
-
-	err := b.sendAndReceive(request, response)
-	if err != nil {
-		return nil, err
-	}
-
-	return response, nil
-}
-
-func (b *Broker) Heartbeat(request *HeartbeatRequest) (*HeartbeatResponse, error) {
-	response := new(HeartbeatResponse)
-
-	err := b.sendAndReceive(request, response)
-	if err != nil {
-		return nil, err
-	}
-
-	return response, nil
-}
-
-func (b *Broker) ListGroups(request *ListGroupsRequest) (*ListGroupsResponse, error) {
-	response := new(ListGroupsResponse)
-
-	err := b.sendAndReceive(request, response)
-	if err != nil {
-		return nil, err
-	}
-
-	return response, nil
-}
-
-func (b *Broker) DescribeGroups(request *DescribeGroupsRequest) (*DescribeGroupsResponse, error) {
-	response := new(DescribeGroupsResponse)
-
-	err := b.sendAndReceive(request, response)
-	if err != nil {
-		return nil, err
-	}
-
-	return response, nil
-}
-
-func (b *Broker) ApiVersions(request *ApiVersionsRequest) (*ApiVersionsResponse, error) {
-	response := new(ApiVersionsResponse)
-
-	err := b.sendAndReceive(request, response)
-	if err != nil {
-		return nil, err
-	}
-
-	return response, nil
-}
-
-func (b *Broker) send(rb protocolBody, promiseResponse bool) (*responsePromise, error) {
-	b.lock.Lock()
-	defer b.lock.Unlock()
-
-	if b.conn == nil {
-		if b.connErr != nil {
-			return nil, b.connErr
-		}
-		return nil, ErrNotConnected
-	}
-
-	if !b.conf.Version.IsAtLeast(rb.requiredVersion()) {
-		return nil, ErrUnsupportedVersion
-	}
-
-	req := &request{correlationID: b.correlationID, clientID: b.conf.ClientID, body: rb}
-	buf, err := encode(req, b.conf.MetricRegistry)
-	if err != nil {
-		return nil, err
-	}
-
-	err = b.conn.SetWriteDeadline(time.Now().Add(b.conf.Net.WriteTimeout))
-	if err != nil {
-		return nil, err
-	}
-
-	requestTime := time.Now()
-	bytes, err := b.conn.Write(buf)
-	b.updateOutgoingCommunicationMetrics(bytes)
-	if err != nil {
-		return nil, err
-	}
-	b.correlationID++
-
-	if !promiseResponse {
-		// Record request latency without the response
-		b.updateRequestLatencyMetrics(time.Since(requestTime))
-		return nil, nil
-	}
-
-	promise := responsePromise{requestTime, req.correlationID, make(chan []byte), make(chan error)}
-	b.responses <- promise
-
-	return &promise, nil
-}
-
-func (b *Broker) sendAndReceive(req protocolBody, res versionedDecoder) error {
-	promise, err := b.send(req, res != nil)
-
-	if err != nil {
-		return err
-	}
-
-	if promise == nil {
-		return nil
-	}
-
-	select {
-	case buf := <-promise.packets:
-		return versionedDecode(buf, res, req.version())
-	case err = <-promise.errors:
-		return err
-	}
-}
-
-func (b *Broker) decode(pd packetDecoder) (err error) {
-	b.id, err = pd.getInt32()
-	if err != nil {
-		return err
-	}
-
-	host, err := pd.getString()
-	if err != nil {
-		return err
-	}
-
-	port, err := pd.getInt32()
-	if err != nil {
-		return err
-	}
-
-	b.addr = net.JoinHostPort(host, fmt.Sprint(port))
-	if _, _, err := net.SplitHostPort(b.addr); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func (b *Broker) encode(pe packetEncoder) (err error) {
-
-	host, portstr, err := net.SplitHostPort(b.addr)
-	if err != nil {
-		return err
-	}
-	port, err := strconv.Atoi(portstr)
-	if err != nil {
-		return err
-	}
-
-	pe.putInt32(b.id)
-
-	err = pe.putString(host)
-	if err != nil {
-		return err
-	}
-
-	pe.putInt32(int32(port))
-
-	return nil
-}
-
-func (b *Broker) responseReceiver() {
-	var dead error
-	header := make([]byte, 8)
-	for response := range b.responses {
-		if dead != nil {
-			response.errors <- dead
-			continue
-		}
-
-		err := b.conn.SetReadDeadline(time.Now().Add(b.conf.Net.ReadTimeout))
-		if err != nil {
-			dead = err
-			response.errors <- err
-			continue
-		}
-
-		bytesReadHeader, err := io.ReadFull(b.conn, header)
-		requestLatency := time.Since(response.requestTime)
-		if err != nil {
-			b.updateIncomingCommunicationMetrics(bytesReadHeader, requestLatency)
-			dead = err
-			response.errors <- err
-			continue
-		}
-
-		decodedHeader := responseHeader{}
-		err = decode(header, &decodedHeader)
-		if err != nil {
-			b.updateIncomingCommunicationMetrics(bytesReadHeader, requestLatency)
-			dead = err
-			response.errors <- err
-			continue
-		}
-		if decodedHeader.correlationID != response.correlationID {
-			b.updateIncomingCommunicationMetrics(bytesReadHeader, requestLatency)
-			// TODO if decoded ID < cur ID, discard until we catch up
-			// TODO if decoded ID > cur ID, save it so when cur ID catches up we have a response
-			dead = PacketDecodingError{fmt.Sprintf("correlation ID didn't match, wanted %d, got %d", response.correlationID, decodedHeader.correlationID)}
-			response.errors <- dead
-			continue
-		}
-
-		buf := make([]byte, decodedHeader.length-4)
-		bytesReadBody, err := io.ReadFull(b.conn, buf)
-		b.updateIncomingCommunicationMetrics(bytesReadHeader+bytesReadBody, requestLatency)
-		if err != nil {
-			dead = err
-			response.errors <- err
-			continue
-		}
-
-		response.packets <- buf
-	}
-	close(b.done)
-}
-
-func (b *Broker) sendAndReceiveSASLPlainHandshake() error {
-	rb := &SaslHandshakeRequest{"PLAIN"}
-	req := &request{correlationID: b.correlationID, clientID: b.conf.ClientID, body: rb}
-	buf, err := encode(req, b.conf.MetricRegistry)
-	if err != nil {
-		return err
-	}
-
-	err = b.conn.SetWriteDeadline(time.Now().Add(b.conf.Net.WriteTimeout))
-	if err != nil {
-		return err
-	}
-
-	requestTime := time.Now()
-	bytes, err := b.conn.Write(buf)
-	b.updateOutgoingCommunicationMetrics(bytes)
-	if err != nil {
-		Logger.Printf("Failed to send SASL handshake %s: %s\n", b.addr, err.Error())
-		return err
-	}
-	b.correlationID++
-	//wait for the response
-	header := make([]byte, 8) // response header
-	_, err = io.ReadFull(b.conn, header)
-	if err != nil {
-		Logger.Printf("Failed to read SASL handshake header : %s\n", err.Error())
-		return err
-	}
-	length := binary.BigEndian.Uint32(header[:4])
-	payload := make([]byte, length-4)
-	n, err := io.ReadFull(b.conn, payload)
-	if err != nil {
-		Logger.Printf("Failed to read SASL handshake payload : %s\n", err.Error())
-		return err
-	}
-	b.updateIncomingCommunicationMetrics(n+8, time.Since(requestTime))
-	res := &SaslHandshakeResponse{}
-	err = versionedDecode(payload, res, 0)
-	if err != nil {
-		Logger.Printf("Failed to parse SASL handshake : %s\n", err.Error())
-		return err
-	}
-	if res.Err != ErrNoError {
-		Logger.Printf("Invalid SASL Mechanism : %s\n", res.Err.Error())
-		return res.Err
-	}
-	Logger.Print("Successful SASL handshake")
-	return nil
-}
-
-// Kafka 0.10.0 plans to support SASL Plain and Kerberos as per PR #812 (KIP-43)/(JIRA KAFKA-3149)
-// Some hosted kafka services such as IBM Message Hub already offer SASL/PLAIN auth with Kafka 0.9
-//
-// In SASL Plain, Kafka expects the auth header to be in the following format
-// Message format (from https://tools.ietf.org/html/rfc4616):
-//
-//   message   = [authzid] UTF8NUL authcid UTF8NUL passwd
-//   authcid   = 1*SAFE ; MUST accept up to 255 octets
-//   authzid   = 1*SAFE ; MUST accept up to 255 octets
-//   passwd    = 1*SAFE ; MUST accept up to 255 octets
-//   UTF8NUL   = %x00 ; UTF-8 encoded NUL character
-//
-//   SAFE      = UTF1 / UTF2 / UTF3 / UTF4
-//                  ;; any UTF-8 encoded Unicode character except NUL
-//
-// When credentials are valid, Kafka returns a 4 byte array of null characters.
-// When credentials are invalid, Kafka closes the connection. This does not seem to be the ideal way
-// of responding to bad credentials but thats how its being done today.
-func (b *Broker) sendAndReceiveSASLPlainAuth() error {
-	if b.conf.Net.SASL.Handshake {
-		handshakeErr := b.sendAndReceiveSASLPlainHandshake()
-		if handshakeErr != nil {
-			Logger.Printf("Error while performing SASL handshake %s\n", b.addr)
-			return handshakeErr
-		}
-	}
-	length := 1 + len(b.conf.Net.SASL.User) + 1 + len(b.conf.Net.SASL.Password)
-	authBytes := make([]byte, length+4) //4 byte length header + auth data
-	binary.BigEndian.PutUint32(authBytes, uint32(length))
-	copy(authBytes[4:], []byte("\x00"+b.conf.Net.SASL.User+"\x00"+b.conf.Net.SASL.Password))
-
-	err := b.conn.SetWriteDeadline(time.Now().Add(b.conf.Net.WriteTimeout))
-	if err != nil {
-		Logger.Printf("Failed to set write deadline when doing SASL auth with broker %s: %s\n", b.addr, err.Error())
-		return err
-	}
-
-	requestTime := time.Now()
-	bytesWritten, err := b.conn.Write(authBytes)
-	b.updateOutgoingCommunicationMetrics(bytesWritten)
-	if err != nil {
-		Logger.Printf("Failed to write SASL auth header to broker %s: %s\n", b.addr, err.Error())
-		return err
-	}
-
-	header := make([]byte, 4)
-	n, err := io.ReadFull(b.conn, header)
-	b.updateIncomingCommunicationMetrics(n, time.Since(requestTime))
-	// If the credentials are valid, we would get a 4 byte response filled with null characters.
-	// Otherwise, the broker closes the connection and we get an EOF
-	if err != nil {
-		Logger.Printf("Failed to read response while authenticating with SASL to broker %s: %s\n", b.addr, err.Error())
-		return err
-	}
-
-	Logger.Printf("SASL authentication successful with broker %s:%v - %v\n", b.addr, n, header)
-	return nil
-}
-
-func (b *Broker) updateIncomingCommunicationMetrics(bytes int, requestLatency time.Duration) {
-	b.updateRequestLatencyMetrics(requestLatency)
-	b.responseRate.Mark(1)
-	if b.brokerResponseRate != nil {
-		b.brokerResponseRate.Mark(1)
-	}
-	responseSize := int64(bytes)
-	b.incomingByteRate.Mark(responseSize)
-	if b.brokerIncomingByteRate != nil {
-		b.brokerIncomingByteRate.Mark(responseSize)
-	}
-	b.responseSize.Update(responseSize)
-	if b.brokerResponseSize != nil {
-		b.brokerResponseSize.Update(responseSize)
-	}
-}
-
-func (b *Broker) updateRequestLatencyMetrics(requestLatency time.Duration) {
-	requestLatencyInMs := int64(requestLatency / time.Millisecond)
-	b.requestLatency.Update(requestLatencyInMs)
-	if b.brokerRequestLatency != nil {
-		b.brokerRequestLatency.Update(requestLatencyInMs)
-	}
-}
-
-func (b *Broker) updateOutgoingCommunicationMetrics(bytes int) {
-	b.requestRate.Mark(1)
-	if b.brokerRequestRate != nil {
-		b.brokerRequestRate.Mark(1)
-	}
-	requestSize := int64(bytes)
-	b.outgoingByteRate.Mark(requestSize)
-	if b.brokerOutgoingByteRate != nil {
-		b.brokerOutgoingByteRate.Mark(requestSize)
-	}
-	b.requestSize.Update(requestSize)
-	if b.brokerRequestSize != nil {
-		b.brokerRequestSize.Update(requestSize)
-	}
-}

vendor/github.com/Shopify/sarama/broker_test.go

@@ -1,328 +0,0 @@
-package sarama
-
-import (
-	"fmt"
-	"testing"
-	"time"
-)
-
-func ExampleBroker() {
-	broker := NewBroker("localhost:9092")
-	err := broker.Open(nil)
-	if err != nil {
-		panic(err)
-	}
-
-	request := MetadataRequest{Topics: []string{"myTopic"}}
-	response, err := broker.GetMetadata(&request)
-	if err != nil {
-		_ = broker.Close()
-		panic(err)
-	}
-
-	fmt.Println("There are", len(response.Topics), "topics active in the cluster.")
-
-	if err = broker.Close(); err != nil {
-		panic(err)
-	}
-}
-
-type mockEncoder struct {
-	bytes []byte
-}
-
-func (m mockEncoder) encode(pe packetEncoder) error {
-	return pe.putRawBytes(m.bytes)
-}
-
-type brokerMetrics struct {
-	bytesRead    int
-	bytesWritten int
-}
-
-func TestBrokerAccessors(t *testing.T) {
-	broker := NewBroker("abc:123")
-
-	if broker.ID() != -1 {
-		t.Error("New broker didn't have an ID of -1.")
-	}
-
-	if broker.Addr() != "abc:123" {
-		t.Error("New broker didn't have the correct address")
-	}
-
-	broker.id = 34
-	if broker.ID() != 34 {
-		t.Error("Manually setting broker ID did not take effect.")
-	}
-}
-
-func TestSimpleBrokerCommunication(t *testing.T) {
-	for _, tt := range brokerTestTable {
-		Logger.Printf("Testing broker communication for %s", tt.name)
-		mb := NewMockBroker(t, 0)
-		mb.Returns(&mockEncoder{tt.response})
-		pendingNotify := make(chan brokerMetrics)
-		// Register a callback to be notified about successful requests
-		mb.SetNotifier(func(bytesRead, bytesWritten int) {
-			pendingNotify <- brokerMetrics{bytesRead, bytesWritten}
-		})
-		broker := NewBroker(mb.Addr())
-		// Set the broker id in order to validate local broker metrics
-		broker.id = 0
-		conf := NewConfig()
-		conf.Version = V0_10_0_0
-		err := broker.Open(conf)
-		if err != nil {
-			t.Fatal(err)
-		}
-		tt.runner(t, broker)
-		// Wait up to 500 ms for the remote broker to process the request and
-		// notify us about the metrics
-		timeout := 500 * time.Millisecond
-		select {
-		case mockBrokerMetrics := <-pendingNotify:
-			validateBrokerMetrics(t, broker, mockBrokerMetrics)
-		case <-time.After(timeout):
-			t.Errorf("No request received for: %s after waiting for %v", tt.name, timeout)
-		}
-		mb.Close()
-		err = broker.Close()
-		if err != nil {
-			t.Error(err)
-		}
-	}
-
-}
-
-// We're not testing encoding/decoding here, so most of the requests/responses will be empty for simplicity's sake
-var brokerTestTable = []struct {
-	name     string
-	response []byte
-	runner   func(*testing.T, *Broker)
-}{
-	{"MetadataRequest",
-		[]byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00},
-		func(t *testing.T, broker *Broker) {
-			request := MetadataRequest{}
-			response, err := broker.GetMetadata(&request)
-			if err != nil {
-				t.Error(err)
-			}
-			if response == nil {
-				t.Error("Metadata request got no response!")
-			}
-		}},
-
-	{"ConsumerMetadataRequest",
-		[]byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 't', 0x00, 0x00, 0x00, 0x00},
-		func(t *testing.T, broker *Broker) {
-			request := ConsumerMetadataRequest{}
-			response, err := broker.GetConsumerMetadata(&request)
-			if err != nil {
-				t.Error(err)
-			}
-			if response == nil {
-				t.Error("Consumer Metadata request got no response!")
-			}
-		}},
-
-	{"ProduceRequest (NoResponse)",
-		[]byte{},
-		func(t *testing.T, broker *Broker) {
-			request := ProduceRequest{}
-			request.RequiredAcks = NoResponse
-			response, err := broker.Produce(&request)
-			if err != nil {
-				t.Error(err)
-			}
-			if response != nil {
-				t.Error("Produce request with NoResponse got a response!")
-			}
-		}},
-
-	{"ProduceRequest (WaitForLocal)",
-		[]byte{0x00, 0x00, 0x00, 0x00},
-		func(t *testing.T, broker *Broker) {
-			request := ProduceRequest{}
-			request.RequiredAcks = WaitForLocal
-			response, err := broker.Produce(&request)
-			if err != nil {
-				t.Error(err)
-			}
-			if response == nil {
-				t.Error("Produce request without NoResponse got no response!")
-			}
-		}},
-
-	{"FetchRequest",
-		[]byte{0x00, 0x00, 0x00, 0x00},
-		func(t *testing.T, broker *Broker) {
-			request := FetchRequest{}
-			response, err := broker.Fetch(&request)
-			if err != nil {
-				t.Error(err)
-			}
-			if response == nil {
-				t.Error("Fetch request got no response!")
-			}
-		}},
-
-	{"OffsetFetchRequest",
-		[]byte{0x00, 0x00, 0x00, 0x00},
-		func(t *testing.T, broker *Broker) {
-			request := OffsetFetchRequest{}
-			response, err := broker.FetchOffset(&request)
-			if err != nil {
-				t.Error(err)
-			}
-			if response == nil {
-				t.Error("OffsetFetch request got no response!")
-			}
-		}},
-
-	{"OffsetCommitRequest",
-		[]byte{0x00, 0x00, 0x00, 0x00},
-		func(t *testing.T, broker *Broker) {
-			request := OffsetCommitRequest{}
-			response, err := broker.CommitOffset(&request)
-			if err != nil {
-				t.Error(err)
-			}
-			if response == nil {
-				t.Error("OffsetCommit request got no response!")
-			}
-		}},
-
-	{"OffsetRequest",
-		[]byte{0x00, 0x00, 0x00, 0x00},
-		func(t *testing.T, broker *Broker) {
-			request := OffsetRequest{}
-			response, err := broker.GetAvailableOffsets(&request)
-			if err != nil {
-				t.Error(err)
-			}
-			if response == nil {
-				t.Error("Offset request got no response!")
-			}
-		}},
-
-	{"JoinGroupRequest",
-		[]byte{0x00, 0x17, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00},
-		func(t *testing.T, broker *Broker) {
-			request := JoinGroupRequest{}
-			response, err := broker.JoinGroup(&request)
-			if err != nil {
-				t.Error(err)
-			}
-			if response == nil {
-				t.Error("JoinGroup request got no response!")
-			}
-		}},
-
-	{"SyncGroupRequest",
-		[]byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00},
-		func(t *testing.T, broker *Broker) {
-			request := SyncGroupRequest{}
-			response, err := broker.SyncGroup(&request)
-			if err != nil {
-				t.Error(err)
-			}
-			if response == nil {
-				t.Error("SyncGroup request got no response!")
-			}
-		}},
-
-	{"LeaveGroupRequest",
-		[]byte{0x00, 0x00},
-		func(t *testing.T, broker *Broker) {
-			request := LeaveGroupRequest{}
-			response, err := broker.LeaveGroup(&request)
-			if err != nil {
-				t.Error(err)
-			}
-			if response == nil {
-				t.Error("LeaveGroup request got no response!")
-			}
-		}},
-
-	{"HeartbeatRequest",
-		[]byte{0x00, 0x00},
-		func(t *testing.T, broker *Broker) {
-			request := HeartbeatRequest{}
-			response, err := broker.Heartbeat(&request)
-			if err != nil {
-				t.Error(err)
-			}
-			if response == nil {
-				t.Error("Heartbeat request got no response!")
-			}
-		}},
-
-	{"ListGroupsRequest",
-		[]byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00},
-		func(t *testing.T, broker *Broker) {
-			request := ListGroupsRequest{}
-			response, err := broker.ListGroups(&request)
-			if err != nil {
-				t.Error(err)
-			}
-			if response == nil {
-				t.Error("ListGroups request got no response!")
-			}
-		}},
-
-	{"DescribeGroupsRequest",
-		[]byte{0x00, 0x00, 0x00, 0x00},
-		func(t *testing.T, broker *Broker) {
-			request := DescribeGroupsRequest{}
-			response, err := broker.DescribeGroups(&request)
-			if err != nil {
-				t.Error(err)
-			}
-			if response == nil {
-				t.Error("DescribeGroups request got no response!")
-			}
-		}},
-
-	{"ApiVersionsRequest",
-		[]byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00},
-		func(t *testing.T, broker *Broker) {
-			request := ApiVersionsRequest{}
-			response, err := broker.ApiVersions(&request)
-			if err != nil {
-				t.Error(err)
-			}
-			if response == nil {
-				t.Error("ApiVersions request got no response!")
-			}
-		}},
-}
-
-func validateBrokerMetrics(t *testing.T, broker *Broker, mockBrokerMetrics brokerMetrics) {
-	metricValidators := newMetricValidators()
-	mockBrokerBytesRead := mockBrokerMetrics.bytesRead
-	mockBrokerBytesWritten := mockBrokerMetrics.bytesWritten
-
-	// Check that the number of bytes sent corresponds to what the mock broker received
-	metricValidators.registerForAllBrokers(broker, countMeterValidator("incoming-byte-rate", mockBrokerBytesWritten))
-	if mockBrokerBytesWritten == 0 {
-		// This a ProduceRequest with NoResponse
-		metricValidators.registerForAllBrokers(broker, countMeterValidator("response-rate", 0))
-		metricValidators.registerForAllBrokers(broker, countHistogramValidator("response-size", 0))
-		metricValidators.registerForAllBrokers(broker, minMaxHistogramValidator("response-size", 0, 0))
-	} else {
-		metricValidators.registerForAllBrokers(broker, countMeterValidator("response-rate", 1))
-		metricValidators.registerForAllBrokers(broker, countHistogramValidator("response-size", 1))
-		metricValidators.registerForAllBrokers(broker, minMaxHistogramValidator("response-size", mockBrokerBytesWritten, mockBrokerBytesWritten))
-	}
-
-	// Check that the number of bytes received corresponds to what the mock broker sent
-	metricValidators.registerForAllBrokers(broker, countMeterValidator("outgoing-byte-rate", mockBrokerBytesRead))
-	metricValidators.registerForAllBrokers(broker, countMeterValidator("request-rate", 1))
-	metricValidators.registerForAllBrokers(broker, countHistogramValidator("request-size", 1))
-	metricValidators.registerForAllBrokers(broker, minMaxHistogramValidator("request-size", mockBrokerBytesRead, mockBrokerBytesRead))
-
-	// Run the validators
-	metricValidators.run(t, broker.conf.MetricRegistry)
-}

vendor/github.com/Shopify/sarama/client.go

@@ -1,794 +0,0 @@
-package sarama
-
-import (
-	"math/rand"
-	"sort"
-	"sync"
-	"time"
-)
-
-// Client is a generic Kafka client. It manages connections to one or more Kafka brokers.
-// You MUST call Close() on a client to avoid leaks, it will not be garbage-collected
-// automatically when it passes out of scope. It is safe to share a client amongst many
-// users, however Kafka will process requests from a single client strictly in serial,
-// so it is generally more efficient to use the default one client per producer/consumer.
-type Client interface {
-	// Config returns the Config struct of the client. This struct should not be
-	// altered after it has been created.
-	Config() *Config
-
-	// Brokers returns the current set of active brokers as retrieved from cluster metadata.
-	Brokers() []*Broker
-
-	// Topics returns the set of available topics as retrieved from cluster metadata.
-	Topics() ([]string, error)
-
-	// Partitions returns the sorted list of all partition IDs for the given topic.
-	Partitions(topic string) ([]int32, error)
-
-	// WritablePartitions returns the sorted list of all writable partition IDs for
-	// the given topic, where "writable" means "having a valid leader accepting
-	// writes".
-	WritablePartitions(topic string) ([]int32, error)
-
-	// Leader returns the broker object that is the leader of the current
-	// topic/partition, as determined by querying the cluster metadata.
-	Leader(topic string, partitionID int32) (*Broker, error)
-
-	// Replicas returns the set of all replica IDs for the given partition.
-	Replicas(topic string, partitionID int32) ([]int32, error)
-
-	// InSyncReplicas returns the set of all in-sync replica IDs for the given
-	// partition. In-sync replicas are replicas which are fully caught up with
-	// the partition leader.
-	InSyncReplicas(topic string, partitionID int32) ([]int32, error)
-
-	// RefreshMetadata takes a list of topics and queries the cluster to refresh the
-	// available metadata for those topics. If no topics are provided, it will refresh
-	// metadata for all topics.
-	RefreshMetadata(topics ...string) error
-
-	// GetOffset queries the cluster to get the most recent available offset at the
-	// given time (in milliseconds) on the topic/partition combination.
-	// Time should be OffsetOldest for the earliest available offset,
-	// OffsetNewest for the offset of the message that will be produced next, or a time.
-	GetOffset(topic string, partitionID int32, time int64) (int64, error)
-
-	// Coordinator returns the coordinating broker for a consumer group. It will
-	// return a locally cached value if it's available. You can call
-	// RefreshCoordinator to update the cached value. This function only works on
-	// Kafka 0.8.2 and higher.
-	Coordinator(consumerGroup string) (*Broker, error)
-
-	// RefreshCoordinator retrieves the coordinator for a consumer group and stores it
-	// in local cache. This function only works on Kafka 0.8.2 and higher.
-	RefreshCoordinator(consumerGroup string) error
-
-	// Close shuts down all broker connections managed by this client. It is required
-	// to call this function before a client object passes out of scope, as it will
-	// otherwise leak memory. You must close any Producers or Consumers using a client
-	// before you close the client.
-	Close() error
-
-	// Closed returns true if the client has already had Close called on it
-	Closed() bool
-}
-
-const (
-	// OffsetNewest stands for the log head offset, i.e. the offset that will be
-	// assigned to the next message that will be produced to the partition. You
-	// can send this to a client's GetOffset method to get this offset, or when
-	// calling ConsumePartition to start consuming new messages.
-	OffsetNewest int64 = -1
-	// OffsetOldest stands for the oldest offset available on the broker for a
-	// partition. You can send this to a client's GetOffset method to get this
-	// offset, or when calling ConsumePartition to start consuming from the
-	// oldest offset that is still available on the broker.
-	OffsetOldest int64 = -2
-)
-
-type client struct {
-	conf           *Config
-	closer, closed chan none // for shutting down background metadata updater
-
-	// the broker addresses given to us through the constructor are not guaranteed to be returned in
-	// the cluster metadata (I *think* it only returns brokers who are currently leading partitions?)
-	// so we store them separately
-	seedBrokers []*Broker
-	deadSeeds   []*Broker
-
-	brokers      map[int32]*Broker                       // maps broker ids to brokers
-	metadata     map[string]map[int32]*PartitionMetadata // maps topics to partition ids to metadata
-	coordinators map[string]int32                        // Maps consumer group names to coordinating broker IDs
-
-	// If the number of partitions is large, we can get some churn calling cachedPartitions,
-	// so the result is cached.  It is important to update this value whenever metadata is changed
-	cachedPartitionsResults map[string][maxPartitionIndex][]int32
-
-	lock sync.RWMutex // protects access to the maps that hold cluster state.
-}
-
-// NewClient creates a new Client. It connects to one of the given broker addresses
-// and uses that broker to automatically fetch metadata on the rest of the kafka cluster. If metadata cannot
-// be retrieved from any of the given broker addresses, the client is not created.
-func NewClient(addrs []string, conf *Config) (Client, error) {
-	Logger.Println("Initializing new client")
-
-	if conf == nil {
-		conf = NewConfig()
-	}
-
-	if err := conf.Validate(); err != nil {
-		return nil, err
-	}
-
-	if len(addrs) < 1 {
-		return nil, ConfigurationError("You must provide at least one broker address")
-	}
-
-	client := &client{
-		conf:                    conf,
-		closer:                  make(chan none),
-		closed:                  make(chan none),
-		brokers:                 make(map[int32]*Broker),
-		metadata:                make(map[string]map[int32]*PartitionMetadata),
-		cachedPartitionsResults: make(map[string][maxPartitionIndex][]int32),
-		coordinators:            make(map[string]int32),
-	}
-
-	random := rand.New(rand.NewSource(time.Now().UnixNano()))
-	for _, index := range random.Perm(len(addrs)) {
-		client.seedBrokers = append(client.seedBrokers, NewBroker(addrs[index]))
-	}
-
-	if conf.Metadata.Full {
-		// do an initial fetch of all cluster metadata by specifying an empty list of topics
-		err := client.RefreshMetadata()
-		switch err {
-		case nil:
-			break
-		case ErrLeaderNotAvailable, ErrReplicaNotAvailable, ErrTopicAuthorizationFailed, ErrClusterAuthorizationFailed:
-			// indicates that maybe part of the cluster is down, but is not fatal to creating the client
-			Logger.Println(err)
-		default:
-			close(client.closed) // we haven't started the background updater yet, so we have to do this manually
-			_ = client.Close()
-			return nil, err
-		}
-	}
-	go withRecover(client.backgroundMetadataUpdater)
-
-	Logger.Println("Successfully initialized new client")
-
-	return client, nil
-}
-
-func (client *client) Config() *Config {
-	return client.conf
-}
-
-func (client *client) Brokers() []*Broker {
-	client.lock.RLock()
-	defer client.lock.RUnlock()
-	brokers := make([]*Broker, 0)
-	for _, broker := range client.brokers {
-		brokers = append(brokers, broker)
-	}
-	return brokers
-}
-
-func (client *client) Close() error {
-	if client.Closed() {
-		// Chances are this is being called from a defer() and the error will go unobserved
-		// so we go ahead and log the event in this case.
-		Logger.Printf("Close() called on already closed client")
-		return ErrClosedClient
-	}
-
-	// shutdown and wait for the background thread before we take the lock, to avoid races
-	close(client.closer)
-	<-client.closed
-
-	client.lock.Lock()
-	defer client.lock.Unlock()
-	Logger.Println("Closing Client")
-
-	for _, broker := range client.brokers {
-		safeAsyncClose(broker)
-	}
-
-	for _, broker := range client.seedBrokers {
-		safeAsyncClose(broker)
-	}
-
-	client.brokers = nil
-	client.metadata = nil
-
-	return nil
-}
-
-func (client *client) Closed() bool {
-	return client.brokers == nil
-}
-
-func (client *client) Topics() ([]string, error) {
-	if client.Closed() {
-		return nil, ErrClosedClient
-	}
-
-	client.lock.RLock()
-	defer client.lock.RUnlock()
-
-	ret := make([]string, 0, len(client.metadata))
-	for topic := range client.metadata {
-		ret = append(ret, topic)
-	}
-
-	return ret, nil
-}
-
-func (client *client) Partitions(topic string) ([]int32, error) {
-	if client.Closed() {
-		return nil, ErrClosedClient
-	}
-
-	partitions := client.cachedPartitions(topic, allPartitions)
-
-	if len(partitions) == 0 {
-		err := client.RefreshMetadata(topic)
-		if err != nil {
-			return nil, err
-		}
-		partitions = client.cachedPartitions(topic, allPartitions)
-	}
-
-	if partitions == nil {
-		return nil, ErrUnknownTopicOrPartition
-	}
-
-	return partitions, nil
-}
-
-func (client *client) WritablePartitions(topic string) ([]int32, error) {
-	if client.Closed() {
-		return nil, ErrClosedClient
-	}
-
-	partitions := client.cachedPartitions(topic, writablePartitions)
-
-	// len==0 catches when it's nil (no such topic) and the odd case when every single
-	// partition is undergoing leader election simultaneously. Callers have to be able to handle
-	// this function returning an empty slice (which is a valid return value) but catching it
-	// here the first time (note we *don't* catch it below where we return ErrUnknownTopicOrPartition) triggers
-	// a metadata refresh as a nicety so callers can just try again and don't have to manually
-	// trigger a refresh (otherwise they'd just keep getting a stale cached copy).
-	if len(partitions) == 0 {
-		err := client.RefreshMetadata(topic)
-		if err != nil {
-			return nil, err
-		}
-		partitions = client.cachedPartitions(topic, writablePartitions)
-	}
-
-	if partitions == nil {
-		return nil, ErrUnknownTopicOrPartition
-	}
-
-	return partitions, nil
-}
-
-func (client *client) Replicas(topic string, partitionID int32) ([]int32, error) {
-	if client.Closed() {
-		return nil, ErrClosedClient
-	}
-
-	metadata := client.cachedMetadata(topic, partitionID)
-
-	if metadata == nil {
-		err := client.RefreshMetadata(topic)
-		if err != nil {
-			return nil, err
-		}
-		metadata = client.cachedMetadata(topic, partitionID)
-	}
-
-	if metadata == nil {
-		return nil, ErrUnknownTopicOrPartition
-	}
-
-	if metadata.Err == ErrReplicaNotAvailable {
-		return dupInt32Slice(metadata.Replicas), metadata.Err
-	}
-	return dupInt32Slice(metadata.Replicas), nil
-}
-
-func (client *client) InSyncReplicas(topic string, partitionID int32) ([]int32, error) {
-	if client.Closed() {
-		return nil, ErrClosedClient
-	}
-
-	metadata := client.cachedMetadata(topic, partitionID)
-
-	if metadata == nil {
-		err := client.RefreshMetadata(topic)
-		if err != nil {
-			return nil, err
-		}
-		metadata = client.cachedMetadata(topic, partitionID)
-	}
-
-	if metadata == nil {
-		return nil, ErrUnknownTopicOrPartition
-	}
-
-	if metadata.Err == ErrReplicaNotAvailable {
-		return dupInt32Slice(metadata.Isr), metadata.Err
-	}
-	return dupInt32Slice(metadata.Isr), nil
-}
-
-func (client *client) Leader(topic string, partitionID int32) (*Broker, error) {
-	if client.Closed() {
-		return nil, ErrClosedClient
-	}
-
-	leader, err := client.cachedLeader(topic, partitionID)
-
-	if leader == nil {
-		err = client.RefreshMetadata(topic)
-		if err != nil {
-			return nil, err
-		}
-		leader, err = client.cachedLeader(topic, partitionID)
-	}
-
-	return leader, err
-}
-
-func (client *client) RefreshMetadata(topics ...string) error {
-	if client.Closed() {
-		return ErrClosedClient
-	}
-
-	// Prior to 0.8.2, Kafka will throw exceptions on an empty topic and not return a proper
-	// error. This handles the case by returning an error instead of sending it
-	// off to Kafka. See: https://github.com/Shopify/sarama/pull/38#issuecomment-26362310
-	for _, topic := range topics {
-		if len(topic) == 0 {
-			return ErrInvalidTopic // this is the error that 0.8.2 and later correctly return
-		}
-	}
-
-	return client.tryRefreshMetadata(topics, client.conf.Metadata.Retry.Max)
-}
-
-func (client *client) GetOffset(topic string, partitionID int32, time int64) (int64, error) {
-	if client.Closed() {
-		return -1, ErrClosedClient
-	}
-
-	offset, err := client.getOffset(topic, partitionID, time)
-
-	if err != nil {
-		if err := client.RefreshMetadata(topic); err != nil {
-			return -1, err
-		}
-		return client.getOffset(topic, partitionID, time)
-	}
-
-	return offset, err
-}
-
-func (client *client) Coordinator(consumerGroup string) (*Broker, error) {
-	if client.Closed() {
-		return nil, ErrClosedClient
-	}
-
-	coordinator := client.cachedCoordinator(consumerGroup)
-
-	if coordinator == nil {
-		if err := client.RefreshCoordinator(consumerGroup); err != nil {
-			return nil, err
-		}
-		coordinator = client.cachedCoordinator(consumerGroup)
-	}
-
-	if coordinator == nil {
-		return nil, ErrConsumerCoordinatorNotAvailable
-	}
-
-	_ = coordinator.Open(client.conf)
-	return coordinator, nil
-}
-
-func (client *client) RefreshCoordinator(consumerGroup string) error {
-	if client.Closed() {
-		return ErrClosedClient
-	}
-
-	response, err := client.getConsumerMetadata(consumerGroup, client.conf.Metadata.Retry.Max)
-	if err != nil {
-		return err
-	}
-
-	client.lock.Lock()
-	defer client.lock.Unlock()
-	client.registerBroker(response.Coordinator)
-	client.coordinators[consumerGroup] = response.Coordinator.ID()
-	return nil
-}
-
-// private broker management helpers
-
-// registerBroker makes sure a broker received by a Metadata or Coordinator request is registered
-// in the brokers map. It returns the broker that is registered, which may be the provided broker,
-// or a previously registered Broker instance. You must hold the write lock before calling this function.
-func (client *client) registerBroker(broker *Broker) {
-	if client.brokers[broker.ID()] == nil {
-		client.brokers[broker.ID()] = broker
-		Logger.Printf("client/brokers registered new broker #%d at %s", broker.ID(), broker.Addr())
-	} else if broker.Addr() != client.brokers[broker.ID()].Addr() {
-		safeAsyncClose(client.brokers[broker.ID()])
-		client.brokers[broker.ID()] = broker
-		Logger.Printf("client/brokers replaced registered broker #%d with %s", broker.ID(), broker.Addr())
-	}
-}
-
-// deregisterBroker removes a broker from the seedsBroker list, and if it's
-// not the seedbroker, removes it from brokers map completely.
-func (client *client) deregisterBroker(broker *Broker) {
-	client.lock.Lock()
-	defer client.lock.Unlock()
-
-	if len(client.seedBrokers) > 0 && broker == client.seedBrokers[0] {
-		client.deadSeeds = append(client.deadSeeds, broker)
-		client.seedBrokers = client.seedBrokers[1:]
-	} else {
-		// we do this so that our loop in `tryRefreshMetadata` doesn't go on forever,
-		// but we really shouldn't have to; once that loop is made better this case can be
-		// removed, and the function generally can be renamed from `deregisterBroker` to
-		// `nextSeedBroker` or something
-		Logger.Printf("client/brokers deregistered broker #%d at %s", broker.ID(), broker.Addr())
-		delete(client.brokers, broker.ID())
-	}
-}
-
-func (client *client) resurrectDeadBrokers() {
-	client.lock.Lock()
-	defer client.lock.Unlock()
-
-	Logger.Printf("client/brokers resurrecting %d dead seed brokers", len(client.deadSeeds))
-	client.seedBrokers = append(client.seedBrokers, client.deadSeeds...)
-	client.deadSeeds = nil
-}
-
-func (client *client) any() *Broker {
-	client.lock.RLock()
-	defer client.lock.RUnlock()
-
-	if len(client.seedBrokers) > 0 {
-		_ = client.seedBrokers[0].Open(client.conf)
-		return client.seedBrokers[0]
-	}
-
-	// not guaranteed to be random *or* deterministic
-	for _, broker := range client.brokers {
-		_ = broker.Open(client.conf)
-		return broker
-	}
-
-	return nil
-}
-
-// private caching/lazy metadata helpers
-
-type partitionType int
-
-const (
-	allPartitions partitionType = iota
-	writablePartitions
-	// If you add any more types, update the partition cache in update()
-
-	// Ensure this is the last partition type value
-	maxPartitionIndex
-)
-
-func (client *client) cachedMetadata(topic string, partitionID int32) *PartitionMetadata {
-	client.lock.RLock()
-	defer client.lock.RUnlock()
-
-	partitions := client.metadata[topic]
-	if partitions != nil {
-		return partitions[partitionID]
-	}
-
-	return nil
-}
-
-func (client *client) cachedPartitions(topic string, partitionSet partitionType) []int32 {
-	client.lock.RLock()
-	defer client.lock.RUnlock()
-
-	partitions, exists := client.cachedPartitionsResults[topic]
-
-	if !exists {
-		return nil
-	}
-	return partitions[partitionSet]
-}
-
-func (client *client) setPartitionCache(topic string, partitionSet partitionType) []int32 {
-	partitions := client.metadata[topic]
-
-	if partitions == nil {
-		return nil
-	}
-
-	ret := make([]int32, 0, len(partitions))
-	for _, partition := range partitions {
-		if partitionSet == writablePartitions && partition.Err == ErrLeaderNotAvailable {
-			continue
-		}
-		ret = append(ret, partition.ID)
-	}
-
-	sort.Sort(int32Slice(ret))
-	return ret
-}
-
-func (client *client) cachedLeader(topic string, partitionID int32) (*Broker, error) {
-	client.lock.RLock()
-	defer client.lock.RUnlock()
-
-	partitions := client.metadata[topic]
-	if partitions != nil {
-		metadata, ok := partitions[partitionID]
-		if ok {
-			if metadata.Err == ErrLeaderNotAvailable {
-				return nil, ErrLeaderNotAvailable
-			}
-			b := client.brokers[metadata.Leader]
-			if b == nil {
-				return nil, ErrLeaderNotAvailable
-			}
-			_ = b.Open(client.conf)
-			return b, nil
-		}
-	}
-
-	return nil, ErrUnknownTopicOrPartition
-}
-
-func (client *client) getOffset(topic string, partitionID int32, time int64) (int64, error) {
-	broker, err := client.Leader(topic, partitionID)
-	if err != nil {
-		return -1, err
-	}
-
-	request := &OffsetRequest{}
-	if client.conf.Version.IsAtLeast(V0_10_1_0) {
-		request.Version = 1
-	}
-	request.AddBlock(topic, partitionID, time, 1)
-
-	response, err := broker.GetAvailableOffsets(request)
-	if err != nil {
-		_ = broker.Close()
-		return -1, err
-	}
-
-	block := response.GetBlock(topic, partitionID)
-	if block == nil {
-		_ = broker.Close()
-		return -1, ErrIncompleteResponse
-	}
-	if block.Err != ErrNoError {
-		return -1, block.Err
-	}
-	if len(block.Offsets) != 1 {
-		return -1, ErrOffsetOutOfRange
-	}
-
-	return block.Offsets[0], nil
-}
-
-// core metadata update logic
-
-func (client *client) backgroundMetadataUpdater() {
-	defer close(client.closed)
-
-	if client.conf.Metadata.RefreshFrequency == time.Duration(0) {
-		return
-	}
-
-	ticker := time.NewTicker(client.conf.Metadata.RefreshFrequency)
-	defer ticker.Stop()
-
-	for {
-		select {
-		case <-ticker.C:
-			topics := []string{}
-			if !client.conf.Metadata.Full {
-				if specificTopics, err := client.Topics(); err != nil {
-					Logger.Println("Client background metadata topic load:", err)
-					break
-				} else if len(specificTopics) == 0 {
-					Logger.Println("Client background metadata update: no specific topics to update")
-					break
-				} else {
-					topics = specificTopics
-				}
-			}
-
-			if err := client.RefreshMetadata(topics...); err != nil {
-				Logger.Println("Client background metadata update:", err)
-			}
-		case <-client.closer:
-			return
-		}
-	}
-}
-
-func (client *client) tryRefreshMetadata(topics []string, attemptsRemaining int) error {
-	retry := func(err error) error {
-		if attemptsRemaining > 0 {
-			Logger.Printf("client/metadata retrying after %dms... (%d attempts remaining)\n", client.conf.Metadata.Retry.Backoff/time.Millisecond, attemptsRemaining)
-			time.Sleep(client.conf.Metadata.Retry.Backoff)
-			return client.tryRefreshMetadata(topics, attemptsRemaining-1)
-		}
-		return err
-	}
-
-	for broker := client.any(); broker != nil; broker = client.any() {
-		if len(topics) > 0 {
-			Logger.Printf("client/metadata fetching metadata for %v from broker %s\n", topics, broker.addr)
-		} else {
-			Logger.Printf("client/metadata fetching metadata for all topics from broker %s\n", broker.addr)
-		}
-		response, err := broker.GetMetadata(&MetadataRequest{Topics: topics})
-
-		switch err.(type) {
-		case nil:
-			// valid response, use it
-			shouldRetry, err := client.updateMetadata(response)
-			if shouldRetry {
-				Logger.Println("client/metadata found some partitions to be leaderless")
-				return retry(err) // note: err can be nil
-			}
-			return err
-
-		case PacketEncodingError:
-			// didn't even send, return the error
-			return err
-		default:
-			// some other error, remove that broker and try again
-			Logger.Println("client/metadata got error from broker while fetching metadata:", err)
-			_ = broker.Close()
-			client.deregisterBroker(broker)
-		}
-	}
-
-	Logger.Println("client/metadata no available broker to send metadata request to")
-	client.resurrectDeadBrokers()
-	return retry(ErrOutOfBrokers)
-}
-
-// if no fatal error, returns a list of topics that need retrying due to ErrLeaderNotAvailable
-func (client *client) updateMetadata(data *MetadataResponse) (retry bool, err error) {
-	client.lock.Lock()
-	defer client.lock.Unlock()
-
-	// For all the brokers we received:
-	// - if it is a new ID, save it
-	// - if it is an existing ID, but the address we have is stale, discard the old one and save it
-	// - otherwise ignore it, replacing our existing one would just bounce the connection
-	for _, broker := range data.Brokers {
-		client.registerBroker(broker)
-	}
-
-	for _, topic := range data.Topics {
-		delete(client.metadata, topic.Name)
-		delete(client.cachedPartitionsResults, topic.Name)
-
-		switch topic.Err {
-		case ErrNoError:
-			break
-		case ErrInvalidTopic, ErrTopicAuthorizationFailed: // don't retry, don't store partial results
-			err = topic.Err
-			continue
-		case ErrUnknownTopicOrPartition: // retry, do not store partial partition results
-			err = topic.Err
-			retry = true
-			continue
-		case ErrLeaderNotAvailable: // retry, but store partial partition results
-			retry = true
-			break
-		default: // don't retry, don't store partial results
-			Logger.Printf("Unexpected topic-level metadata error: %s", topic.Err)
-			err = topic.Err
-			continue
-		}
-
-		client.metadata[topic.Name] = make(map[int32]*PartitionMetadata, len(topic.Partitions))
-		for _, partition := range topic.Partitions {
-			client.metadata[topic.Name][partition.ID] = partition
-			if partition.Err == ErrLeaderNotAvailable {
-				retry = true
-			}
-		}
-
-		var partitionCache [maxPartitionIndex][]int32
-		partitionCache[allPartitions] = client.setPartitionCache(topic.Name, allPartitions)
-		partitionCache[writablePartitions] = client.setPartitionCache(topic.Name, writablePartitions)
-		client.cachedPartitionsResults[topic.Name] = partitionCache
-	}
-
-	return
-}
-
-func (client *client) cachedCoordinator(consumerGroup string) *Broker {
-	client.lock.RLock()
-	defer client.lock.RUnlock()
-	if coordinatorID, ok := client.coordinators[consumerGroup]; ok {
-		return client.brokers[coordinatorID]
-	}
-	return nil
-}
-
-func (client *client) getConsumerMetadata(consumerGroup string, attemptsRemaining int) (*ConsumerMetadataResponse, error) {
-	retry := func(err error) (*ConsumerMetadataResponse, error) {
-		if attemptsRemaining > 0 {
-			Logger.Printf("client/coordinator retrying after %dms... (%d attempts remaining)\n", client.conf.Metadata.Retry.Backoff/time.Millisecond, attemptsRemaining)
-			time.Sleep(client.conf.Metadata.Retry.Backoff)
-			return client.getConsumerMetadata(consumerGroup, attemptsRemaining-1)
-		}
-		return nil, err
-	}
-
-	for broker := client.any(); broker != nil; broker = client.any() {
-		Logger.Printf("client/coordinator requesting coordinator for consumergroup %s from %s\n", consumerGroup, broker.Addr())
-
-		request := new(ConsumerMetadataRequest)
-		request.ConsumerGroup = consumerGroup
-
-		response, err := broker.GetConsumerMetadata(request)
-
-		if err != nil {
-			Logger.Printf("client/coordinator request to broker %s failed: %s\n", broker.Addr(), err)
-
-			switch err.(type) {
-			case PacketEncodingError:
-				return nil, err
-			default:
-				_ = broker.Close()
-				client.deregisterBroker(broker)
-				continue
-			}
-		}
-
-		switch response.Err {
-		case ErrNoError:
-			Logger.Printf("client/coordinator coordinator for consumergroup %s is #%d (%s)\n", consumerGroup, response.Coordinator.ID(), response.Coordinator.Addr())
-			return response, nil
-
-		case ErrConsumerCoordinatorNotAvailable:
-			Logger.Printf("client/coordinator coordinator for consumer group %s is not available\n", consumerGroup)
-
-			// This is very ugly, but this scenario will only happen once per cluster.
-			// The __consumer_offsets topic only has to be created one time.
-			// The number of partitions not configurable, but partition 0 should always exist.
-			if _, err := client.Leader("__consumer_offsets", 0); err != nil {
-				Logger.Printf("client/coordinator the __consumer_offsets topic is not initialized completely yet. Waiting 2 seconds...\n")
-				time.Sleep(2 * time.Second)
-			}
-
-			return retry(ErrConsumerCoordinatorNotAvailable)
-		default:
-			return nil, response.Err
-		}
-	}
-
-	Logger.Println("client/coordinator no available broker to send consumer metadata request to")
-	client.resurrectDeadBrokers()
-	return retry(ErrOutOfBrokers)
-}

vendor/github.com/Shopify/sarama/client_test.go

@@ -1,619 +0,0 @@
-package sarama
-
-import (
-	"io"
-	"sync"
-	"testing"
-	"time"
-)
-
-func safeClose(t testing.TB, c io.Closer) {
-	err := c.Close()
-	if err != nil {
-		t.Error(err)
-	}
-}
-
-func TestSimpleClient(t *testing.T) {
-	seedBroker := NewMockBroker(t, 1)
-
-	seedBroker.Returns(new(MetadataResponse))
-
-	client, err := NewClient([]string{seedBroker.Addr()}, nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	seedBroker.Close()
-	safeClose(t, client)
-}
-
-func TestCachedPartitions(t *testing.T) {
-	seedBroker := NewMockBroker(t, 1)
-
-	replicas := []int32{3, 1, 5}
-	isr := []int32{5, 1}
-
-	metadataResponse := new(MetadataResponse)
-	metadataResponse.AddBroker("localhost:12345", 2)
-	metadataResponse.AddTopicPartition("my_topic", 0, 2, replicas, isr, ErrNoError)
-	metadataResponse.AddTopicPartition("my_topic", 1, 2, replicas, isr, ErrLeaderNotAvailable)
-	seedBroker.Returns(metadataResponse)
-
-	config := NewConfig()
-	config.Metadata.Retry.Max = 0
-	c, err := NewClient([]string{seedBroker.Addr()}, config)
-	if err != nil {
-		t.Fatal(err)
-	}
-	client := c.(*client)
-
-	// Verify they aren't cached the same
-	allP := client.cachedPartitionsResults["my_topic"][allPartitions]
-	writeP := client.cachedPartitionsResults["my_topic"][writablePartitions]
-	if len(allP) == len(writeP) {
-		t.Fatal("Invalid lengths!")
-	}
-
-	tmp := client.cachedPartitionsResults["my_topic"]
-	// Verify we actually use the cache at all!
-	tmp[allPartitions] = []int32{1, 2, 3, 4}
-	client.cachedPartitionsResults["my_topic"] = tmp
-	if 4 != len(client.cachedPartitions("my_topic", allPartitions)) {
-		t.Fatal("Not using the cache!")
-	}
-
-	seedBroker.Close()
-	safeClose(t, client)
-}
-
-func TestClientDoesntCachePartitionsForTopicsWithErrors(t *testing.T) {
-	seedBroker := NewMockBroker(t, 1)
-
-	replicas := []int32{seedBroker.BrokerID()}
-
-	metadataResponse := new(MetadataResponse)
-	metadataResponse.AddBroker(seedBroker.Addr(), seedBroker.BrokerID())
-	metadataResponse.AddTopicPartition("my_topic", 1, replicas[0], replicas, replicas, ErrNoError)
-	metadataResponse.AddTopicPartition("my_topic", 2, replicas[0], replicas, replicas, ErrNoError)
-	seedBroker.Returns(metadataResponse)
-
-	config := NewConfig()
-	config.Metadata.Retry.Max = 0
-	client, err := NewClient([]string{seedBroker.Addr()}, config)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	metadataResponse = new(MetadataResponse)
-	metadataResponse.AddTopic("unknown", ErrUnknownTopicOrPartition)
-	seedBroker.Returns(metadataResponse)
-
-	partitions, err := client.Partitions("unknown")
-
-	if err != ErrUnknownTopicOrPartition {
-		t.Error("Expected ErrUnknownTopicOrPartition, found", err)
-	}
-	if partitions != nil {
-		t.Errorf("Should return nil as partition list, found %v", partitions)
-	}
-
-	// Should still use the cache of a known topic
-	partitions, err = client.Partitions("my_topic")
-	if err != nil {
-		t.Errorf("Expected no error, found %v", err)
-	}
-
-	metadataResponse = new(MetadataResponse)
-	metadataResponse.AddTopic("unknown", ErrUnknownTopicOrPartition)
-	seedBroker.Returns(metadataResponse)
-
-	// Should not use cache for unknown topic
-	partitions, err = client.Partitions("unknown")
-	if err != ErrUnknownTopicOrPartition {
-		t.Error("Expected ErrUnknownTopicOrPartition, found", err)
-	}
-	if partitions != nil {
-		t.Errorf("Should return nil as partition list, found %v", partitions)
-	}
-
-	seedBroker.Close()
-	safeClose(t, client)
-}
-
-func TestClientSeedBrokers(t *testing.T) {
-	seedBroker := NewMockBroker(t, 1)
-
-	metadataResponse := new(MetadataResponse)
-	metadataResponse.AddBroker("localhost:12345", 2)
-	seedBroker.Returns(metadataResponse)
-
-	client, err := NewClient([]string{seedBroker.Addr()}, nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	seedBroker.Close()
-	safeClose(t, client)
-}
-
-func TestClientMetadata(t *testing.T) {
-	seedBroker := NewMockBroker(t, 1)
-	leader := NewMockBroker(t, 5)
-
-	replicas := []int32{3, 1, 5}
-	isr := []int32{5, 1}
-
-	metadataResponse := new(MetadataResponse)
-	metadataResponse.AddBroker(leader.Addr(), leader.BrokerID())
-	metadataResponse.AddTopicPartition("my_topic", 0, leader.BrokerID(), replicas, isr, ErrNoError)
-	metadataResponse.AddTopicPartition("my_topic", 1, leader.BrokerID(), replicas, isr, ErrLeaderNotAvailable)
-	seedBroker.Returns(metadataResponse)
-
-	config := NewConfig()
-	config.Metadata.Retry.Max = 0
-	client, err := NewClient([]string{seedBroker.Addr()}, config)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	topics, err := client.Topics()
-	if err != nil {
-		t.Error(err)
-	} else if len(topics) != 1 || topics[0] != "my_topic" {
-		t.Error("Client returned incorrect topics:", topics)
-	}
-
-	parts, err := client.Partitions("my_topic")
-	if err != nil {
-		t.Error(err)
-	} else if len(parts) != 2 || parts[0] != 0 || parts[1] != 1 {
-		t.Error("Client returned incorrect partitions for my_topic:", parts)
-	}
-
-	parts, err = client.WritablePartitions("my_topic")
-	if err != nil {
-		t.Error(err)
-	} else if len(parts) != 1 || parts[0] != 0 {
-		t.Error("Client returned incorrect writable partitions for my_topic:", parts)
-	}
-
-	tst, err := client.Leader("my_topic", 0)
-	if err != nil {
-		t.Error(err)
-	} else if tst.ID() != 5 {
-		t.Error("Leader for my_topic had incorrect ID.")
-	}
-
-	replicas, err = client.Replicas("my_topic", 0)
-	if err != nil {
-		t.Error(err)
-	} else if replicas[0] != 3 {
-		t.Error("Incorrect (or sorted) replica")
-	} else if replicas[1] != 1 {
-		t.Error("Incorrect (or sorted) replica")
-	} else if replicas[2] != 5 {
-		t.Error("Incorrect (or sorted) replica")
-	}
-
-	isr, err = client.InSyncReplicas("my_topic", 0)
-	if err != nil {
-		t.Error(err)
-	} else if len(isr) != 2 {
-		t.Error("Client returned incorrect ISRs for partition:", isr)
-	} else if isr[0] != 5 {
-		t.Error("Incorrect (or sorted) ISR:", isr)
-	} else if isr[1] != 1 {
-		t.Error("Incorrect (or sorted) ISR:", isr)
-	}
-
-	leader.Close()
-	seedBroker.Close()
-	safeClose(t, client)
-}
-
-func TestClientGetOffset(t *testing.T) {
-	seedBroker := NewMockBroker(t, 1)
-	leader := NewMockBroker(t, 2)
-	leaderAddr := leader.Addr()
-
-	metadata := new(MetadataResponse)
-	metadata.AddTopicPartition("foo", 0, leader.BrokerID(), nil, nil, ErrNoError)
-	metadata.AddBroker(leaderAddr, leader.BrokerID())
-	seedBroker.Returns(metadata)
-
-	client, err := NewClient([]string{seedBroker.Addr()}, nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	offsetResponse := new(OffsetResponse)
-	offsetResponse.AddTopicPartition("foo", 0, 123)
-	leader.Returns(offsetResponse)
-
-	offset, err := client.GetOffset("foo", 0, OffsetNewest)
-	if err != nil {
-		t.Error(err)
-	}
-	if offset != 123 {
-		t.Error("Unexpected offset, got ", offset)
-	}
-
-	leader.Close()
-	seedBroker.Returns(metadata)
-
-	leader = NewMockBrokerAddr(t, 2, leaderAddr)
-	offsetResponse = new(OffsetResponse)
-	offsetResponse.AddTopicPartition("foo", 0, 456)
-	leader.Returns(offsetResponse)
-
-	offset, err = client.GetOffset("foo", 0, OffsetNewest)
-	if err != nil {
-		t.Error(err)
-	}
-	if offset != 456 {
-		t.Error("Unexpected offset, got ", offset)
-	}
-
-	seedBroker.Close()
-	leader.Close()
-	safeClose(t, client)
-}
-
-func TestClientReceivingUnknownTopic(t *testing.T) {
-	seedBroker := NewMockBroker(t, 1)
-
-	metadataResponse1 := new(MetadataResponse)
-	seedBroker.Returns(metadataResponse1)
-
-	config := NewConfig()
-	config.Metadata.Retry.Max = 1
-	config.Metadata.Retry.Backoff = 0
-	client, err := NewClient([]string{seedBroker.Addr()}, config)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	metadataUnknownTopic := new(MetadataResponse)
-	metadataUnknownTopic.AddTopic("new_topic", ErrUnknownTopicOrPartition)
-	seedBroker.Returns(metadataUnknownTopic)
-	seedBroker.Returns(metadataUnknownTopic)
-
-	if err := client.RefreshMetadata("new_topic"); err != ErrUnknownTopicOrPartition {
-		t.Error("ErrUnknownTopicOrPartition expected, got", err)
-	}
-
-	// If we are asking for the leader of a partition of the non-existing topic.
-	// we will request metadata again.
-	seedBroker.Returns(metadataUnknownTopic)
-	seedBroker.Returns(metadataUnknownTopic)
-
-	if _, err = client.Leader("new_topic", 1); err != ErrUnknownTopicOrPartition {
-		t.Error("Expected ErrUnknownTopicOrPartition, got", err)
-	}
-
-	safeClose(t, client)
-	seedBroker.Close()
-}
-
-func TestClientReceivingPartialMetadata(t *testing.T) {
-	seedBroker := NewMockBroker(t, 1)
-	leader := NewMockBroker(t, 5)
-
-	metadataResponse1 := new(MetadataResponse)
-	metadataResponse1.AddBroker(leader.Addr(), leader.BrokerID())
-	seedBroker.Returns(metadataResponse1)
-
-	config := NewConfig()
-	config.Metadata.Retry.Max = 0
-	client, err := NewClient([]string{seedBroker.Addr()}, config)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	replicas := []int32{leader.BrokerID(), seedBroker.BrokerID()}
-
-	metadataPartial := new(MetadataResponse)
-	metadataPartial.AddTopic("new_topic", ErrLeaderNotAvailable)
-	metadataPartial.AddTopicPartition("new_topic", 0, leader.BrokerID(), replicas, replicas, ErrNoError)
-	metadataPartial.AddTopicPartition("new_topic", 1, -1, replicas, []int32{}, ErrLeaderNotAvailable)
-	seedBroker.Returns(metadataPartial)
-
-	if err := client.RefreshMetadata("new_topic"); err != nil {
-		t.Error("ErrLeaderNotAvailable should not make RefreshMetadata respond with an error")
-	}
-
-	// Even though the metadata was incomplete, we should be able to get the leader of a partition
-	// for which we did get a useful response, without doing additional requests.
-
-	partition0Leader, err := client.Leader("new_topic", 0)
-	if err != nil {
-		t.Error(err)
-	} else if partition0Leader.Addr() != leader.Addr() {
-		t.Error("Unexpected leader returned", partition0Leader.Addr())
-	}
-
-	// If we are asking for the leader of a partition that didn't have a leader before,
-	// we will do another metadata request.
-
-	seedBroker.Returns(metadataPartial)
-
-	// Still no leader for the partition, so asking for it should return an error.
-	_, err = client.Leader("new_topic", 1)
-	if err != ErrLeaderNotAvailable {
-		t.Error("Expected ErrLeaderNotAvailable, got", err)
-	}
-
-	safeClose(t, client)
-	seedBroker.Close()
-	leader.Close()
-}
-
-func TestClientRefreshBehaviour(t *testing.T) {
-	seedBroker := NewMockBroker(t, 1)
-	leader := NewMockBroker(t, 5)
-
-	metadataResponse1 := new(MetadataResponse)
-	metadataResponse1.AddBroker(leader.Addr(), leader.BrokerID())
-	seedBroker.Returns(metadataResponse1)
-
-	metadataResponse2 := new(MetadataResponse)
-	metadataResponse2.AddTopicPartition("my_topic", 0xb, leader.BrokerID(), nil, nil, ErrNoError)
-	seedBroker.Returns(metadataResponse2)
-
-	client, err := NewClient([]string{seedBroker.Addr()}, nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	parts, err := client.Partitions("my_topic")
-	if err != nil {
-		t.Error(err)
-	} else if len(parts) != 1 || parts[0] != 0xb {
-		t.Error("Client returned incorrect partitions for my_topic:", parts)
-	}
-
-	tst, err := client.Leader("my_topic", 0xb)
-	if err != nil {
-		t.Error(err)
-	} else if tst.ID() != 5 {
-		t.Error("Leader for my_topic had incorrect ID.")
-	}
-
-	leader.Close()
-	seedBroker.Close()
-	safeClose(t, client)
-}
-
-func TestClientResurrectDeadSeeds(t *testing.T) {
-	initialSeed := NewMockBroker(t, 0)
-	emptyMetadata := new(MetadataResponse)
-	initialSeed.Returns(emptyMetadata)
-
-	conf := NewConfig()
-	conf.Metadata.Retry.Backoff = 0
-	conf.Metadata.RefreshFrequency = 0
-	c, err := NewClient([]string{initialSeed.Addr()}, conf)
-	if err != nil {
-		t.Fatal(err)
-	}
-	initialSeed.Close()
-
-	client := c.(*client)
-
-	seed1 := NewMockBroker(t, 1)
-	seed2 := NewMockBroker(t, 2)
-	seed3 := NewMockBroker(t, 3)
-	addr1 := seed1.Addr()
-	addr2 := seed2.Addr()
-	addr3 := seed3.Addr()
-
-	// Overwrite the seed brokers with a fixed ordering to make this test deterministic.
-	safeClose(t, client.seedBrokers[0])
-	client.seedBrokers = []*Broker{NewBroker(addr1), NewBroker(addr2), NewBroker(addr3)}
-	client.deadSeeds = []*Broker{}
-
-	wg := sync.WaitGroup{}
-	wg.Add(1)
-	go func() {
-		if err := client.RefreshMetadata(); err != nil {
-			t.Error(err)
-		}
-		wg.Done()
-	}()
-	seed1.Close()
-	seed2.Close()
-
-	seed1 = NewMockBrokerAddr(t, 1, addr1)
-	seed2 = NewMockBrokerAddr(t, 2, addr2)
-
-	seed3.Close()
-
-	seed1.Close()
-	seed2.Returns(emptyMetadata)
-
-	wg.Wait()
-
-	if len(client.seedBrokers) != 2 {
-		t.Error("incorrect number of live seeds")
-	}
-	if len(client.deadSeeds) != 1 {
-		t.Error("incorrect number of dead seeds")
-	}
-
-	safeClose(t, c)
-}
-
-func TestClientCoordinatorWithConsumerOffsetsTopic(t *testing.T) {
-	seedBroker := NewMockBroker(t, 1)
-	staleCoordinator := NewMockBroker(t, 2)
-	freshCoordinator := NewMockBroker(t, 3)
-
-	replicas := []int32{staleCoordinator.BrokerID(), freshCoordinator.BrokerID()}
-	metadataResponse1 := new(MetadataResponse)
-	metadataResponse1.AddBroker(staleCoordinator.Addr(), staleCoordinator.BrokerID())
-	metadataResponse1.AddBroker(freshCoordinator.Addr(), freshCoordinator.BrokerID())
-	metadataResponse1.AddTopicPartition("__consumer_offsets", 0, replicas[0], replicas, replicas, ErrNoError)
-	seedBroker.Returns(metadataResponse1)
-
-	client, err := NewClient([]string{seedBroker.Addr()}, nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	coordinatorResponse1 := new(ConsumerMetadataResponse)
-	coordinatorResponse1.Err = ErrConsumerCoordinatorNotAvailable
-	seedBroker.Returns(coordinatorResponse1)
-
-	coordinatorResponse2 := new(ConsumerMetadataResponse)
-	coordinatorResponse2.CoordinatorID = staleCoordinator.BrokerID()
-	coordinatorResponse2.CoordinatorHost = "127.0.0.1"
-	coordinatorResponse2.CoordinatorPort = staleCoordinator.Port()
-
-	seedBroker.Returns(coordinatorResponse2)
-
-	broker, err := client.Coordinator("my_group")
-	if err != nil {
-		t.Error(err)
-	}
-
-	if staleCoordinator.Addr() != broker.Addr() {
-		t.Errorf("Expected coordinator to have address %s, found %s", staleCoordinator.Addr(), broker.Addr())
-	}
-
-	if staleCoordinator.BrokerID() != broker.ID() {
-		t.Errorf("Expected coordinator to have ID %d, found %d", staleCoordinator.BrokerID(), broker.ID())
-	}
-
-	// Grab the cached value
-	broker2, err := client.Coordinator("my_group")
-	if err != nil {
-		t.Error(err)
-	}
-
-	if broker2.Addr() != broker.Addr() {
-		t.Errorf("Expected the coordinator to be the same, but found %s vs. %s", broker2.Addr(), broker.Addr())
-	}
-
-	coordinatorResponse3 := new(ConsumerMetadataResponse)
-	coordinatorResponse3.CoordinatorID = freshCoordinator.BrokerID()
-	coordinatorResponse3.CoordinatorHost = "127.0.0.1"
-	coordinatorResponse3.CoordinatorPort = freshCoordinator.Port()
-
-	seedBroker.Returns(coordinatorResponse3)
-
-	// Refresh the locally cahced value because it's stale
-	if err := client.RefreshCoordinator("my_group"); err != nil {
-		t.Error(err)
-	}
-
-	// Grab the fresh value
-	broker3, err := client.Coordinator("my_group")
-	if err != nil {
-		t.Error(err)
-	}
-
-	if broker3.Addr() != freshCoordinator.Addr() {
-		t.Errorf("Expected the freshCoordinator to be returned, but found %s.", broker3.Addr())
-	}
-
-	freshCoordinator.Close()
-	staleCoordinator.Close()
-	seedBroker.Close()
-	safeClose(t, client)
-}
-
-func TestClientCoordinatorWithoutConsumerOffsetsTopic(t *testing.T) {
-	seedBroker := NewMockBroker(t, 1)
-	coordinator := NewMockBroker(t, 2)
-
-	metadataResponse1 := new(MetadataResponse)
-	seedBroker.Returns(metadataResponse1)
-
-	config := NewConfig()
-	config.Metadata.Retry.Max = 1
-	config.Metadata.Retry.Backoff = 0
-	client, err := NewClient([]string{seedBroker.Addr()}, config)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	coordinatorResponse1 := new(ConsumerMetadataResponse)
-	coordinatorResponse1.Err = ErrConsumerCoordinatorNotAvailable
-	seedBroker.Returns(coordinatorResponse1)
-
-	metadataResponse2 := new(MetadataResponse)
-	metadataResponse2.AddTopic("__consumer_offsets", ErrUnknownTopicOrPartition)
-	seedBroker.Returns(metadataResponse2)
-
-	replicas := []int32{coordinator.BrokerID()}
-	metadataResponse3 := new(MetadataResponse)
-	metadataResponse3.AddTopicPartition("__consumer_offsets", 0, replicas[0], replicas, replicas, ErrNoError)
-	seedBroker.Returns(metadataResponse3)
-
-	coordinatorResponse2 := new(ConsumerMetadataResponse)
-	coordinatorResponse2.CoordinatorID = coordinator.BrokerID()
-	coordinatorResponse2.CoordinatorHost = "127.0.0.1"
-	coordinatorResponse2.CoordinatorPort = coordinator.Port()
-
-	seedBroker.Returns(coordinatorResponse2)
-
-	broker, err := client.Coordinator("my_group")
-	if err != nil {
-		t.Error(err)
-	}
-
-	if coordinator.Addr() != broker.Addr() {
-		t.Errorf("Expected coordinator to have address %s, found %s", coordinator.Addr(), broker.Addr())
-	}
-
-	if coordinator.BrokerID() != broker.ID() {
-		t.Errorf("Expected coordinator to have ID %d, found %d", coordinator.BrokerID(), broker.ID())
-	}
-
-	coordinator.Close()
-	seedBroker.Close()
-	safeClose(t, client)
-}
-
-func TestClientAutorefreshShutdownRace(t *testing.T) {
-	seedBroker := NewMockBroker(t, 1)
-
-	metadataResponse := new(MetadataResponse)
-	seedBroker.Returns(metadataResponse)
-
-	conf := NewConfig()
-	conf.Metadata.RefreshFrequency = 100 * time.Millisecond
-	client, err := NewClient([]string{seedBroker.Addr()}, conf)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// Wait for the background refresh to kick in
-	time.Sleep(110 * time.Millisecond)
-
-	done := make(chan none)
-	go func() {
-		// Close the client
-		if err := client.Close(); err != nil {
-			t.Fatal(err)
-		}
-		close(done)
-	}()
-
-	// Wait for the Close to kick in
-	time.Sleep(10 * time.Millisecond)
-
-	// Then return some metadata to the still-running background thread
-	leader := NewMockBroker(t, 2)
-	metadataResponse.AddBroker(leader.Addr(), leader.BrokerID())
-	metadataResponse.AddTopicPartition("foo", 0, leader.BrokerID(), []int32{2}, []int32{2}, ErrNoError)
-	seedBroker.Returns(metadataResponse)
-
-	<-done
-
-	seedBroker.Close()
-
-	// give the update time to happen so we get a panic if it's still running (which it shouldn't)
-	time.Sleep(10 * time.Millisecond)
-}

vendor/github.com/Shopify/sarama/config.go

@@ -1,442 +0,0 @@
-package sarama
-
-import (
-	"crypto/tls"
-	"regexp"
-	"time"
-
-	"github.com/rcrowley/go-metrics"
-)
-
-const defaultClientID = "sarama"
-
-var validID = regexp.MustCompile(`\A[A-Za-z0-9._-]+\z`)
-
-// Config is used to pass multiple configuration options to Sarama's constructors.
-type Config struct {
-	// Net is the namespace for network-level properties used by the Broker, and
-	// shared by the Client/Producer/Consumer.
-	Net struct {
-		// How many outstanding requests a connection is allowed to have before
-		// sending on it blocks (default 5).
-		MaxOpenRequests int
-
-		// All three of the below configurations are similar to the
-		// `socket.timeout.ms` setting in JVM kafka. All of them default
-		// to 30 seconds.
-		DialTimeout  time.Duration // How long to wait for the initial connection.
-		ReadTimeout  time.Duration // How long to wait for a response.
-		WriteTimeout time.Duration // How long to wait for a transmit.
-
-		TLS struct {
-			// Whether or not to use TLS when connecting to the broker
-			// (defaults to false).
-			Enable bool
-			// The TLS configuration to use for secure connections if
-			// enabled (defaults to nil).
-			Config *tls.Config
-		}
-
-		// SASL based authentication with broker. While there are multiple SASL authentication methods
-		// the current implementation is limited to plaintext (SASL/PLAIN) authentication
-		SASL struct {
-			// Whether or not to use SASL authentication when connecting to the broker
-			// (defaults to false).
-			Enable bool
-			// Whether or not to send the Kafka SASL handshake first if enabled
-			// (defaults to true). You should only set this to false if you're using
-			// a non-Kafka SASL proxy.
-			Handshake bool
-			//username and password for SASL/PLAIN authentication
-			User     string
-			Password string
-		}
-
-		// KeepAlive specifies the keep-alive period for an active network connection.
-		// If zero, keep-alives are disabled. (default is 0: disabled).
-		KeepAlive time.Duration
-	}
-
-	// Metadata is the namespace for metadata management properties used by the
-	// Client, and shared by the Producer/Consumer.
-	Metadata struct {
-		Retry struct {
-			// The total number of times to retry a metadata request when the
-			// cluster is in the middle of a leader election (default 3).
-			Max int
-			// How long to wait for leader election to occur before retrying
-			// (default 250ms). Similar to the JVM's `retry.backoff.ms`.
-			Backoff time.Duration
-		}
-		// How frequently to refresh the cluster metadata in the background.
-		// Defaults to 10 minutes. Set to 0 to disable. Similar to
-		// `topic.metadata.refresh.interval.ms` in the JVM version.
-		RefreshFrequency time.Duration
-
-		// Whether to maintain a full set of metadata for all topics, or just
-		// the minimal set that has been necessary so far. The full set is simpler
-		// and usually more convenient, but can take up a substantial amount of
-		// memory if you have many topics and partitions. Defaults to true.
-		Full bool
-	}
-
-	// Producer is the namespace for configuration related to producing messages,
-	// used by the Producer.
-	Producer struct {
-		// The maximum permitted size of a message (defaults to 1000000). Should be
-		// set equal to or smaller than the broker's `message.max.bytes`.
-		MaxMessageBytes int
-		// The level of acknowledgement reliability needed from the broker (defaults
-		// to WaitForLocal). Equivalent to the `request.required.acks` setting of the
-		// JVM producer.
-		RequiredAcks RequiredAcks
-		// The maximum duration the broker will wait the receipt of the number of
-		// RequiredAcks (defaults to 10 seconds). This is only relevant when
-		// RequiredAcks is set to WaitForAll or a number > 1. Only supports
-		// millisecond resolution, nanoseconds will be truncated. Equivalent to
-		// the JVM producer's `request.timeout.ms` setting.
-		Timeout time.Duration
-		// The type of compression to use on messages (defaults to no compression).
-		// Similar to `compression.codec` setting of the JVM producer.
-		Compression CompressionCodec
-		// Generates partitioners for choosing the partition to send messages to
-		// (defaults to hashing the message key). Similar to the `partitioner.class`
-		// setting for the JVM producer.
-		Partitioner PartitionerConstructor
-
-		// Return specifies what channels will be populated. If they are set to true,
-		// you must read from the respective channels to prevent deadlock. If,
-		// however, this config is used to create a `SyncProducer`, both must be set
-		// to true and you shall not read from the channels since the producer does
-		// this internally.
-		Return struct {
-			// If enabled, successfully delivered messages will be returned on the
-			// Successes channel (default disabled).
-			Successes bool
-
-			// If enabled, messages that failed to deliver will be returned on the
-			// Errors channel, including error (default enabled).
-			Errors bool
-		}
-
-		// The following config options control how often messages are batched up and
-		// sent to the broker. By default, messages are sent as fast as possible, and
-		// all messages received while the current batch is in-flight are placed
-		// into the subsequent batch.
-		Flush struct {
-			// The best-effort number of bytes needed to trigger a flush. Use the
-			// global sarama.MaxRequestSize to set a hard upper limit.
-			Bytes int
-			// The best-effort number of messages needed to trigger a flush. Use
-			// `MaxMessages` to set a hard upper limit.
-			Messages int
-			// The best-effort frequency of flushes. Equivalent to
-			// `queue.buffering.max.ms` setting of JVM producer.
-			Frequency time.Duration
-			// The maximum number of messages the producer will send in a single
-			// broker request. Defaults to 0 for unlimited. Similar to
-			// `queue.buffering.max.messages` in the JVM producer.
-			MaxMessages int
-		}
-
-		Retry struct {
-			// The total number of times to retry sending a message (default 3).
-			// Similar to the `message.send.max.retries` setting of the JVM producer.
-			Max int
-			// How long to wait for the cluster to settle between retries
-			// (default 100ms). Similar to the `retry.backoff.ms` setting of the
-			// JVM producer.
-			Backoff time.Duration
-		}
-	}
-
-	// Consumer is the namespace for configuration related to consuming messages,
-	// used by the Consumer.
-	//
-	// Note that Sarama's Consumer type does not currently support automatic
-	// consumer-group rebalancing and offset tracking.  For Zookeeper-based
-	// tracking (Kafka 0.8.2 and earlier), the https://github.com/wvanbergen/kafka
-	// library builds on Sarama to add this support. For Kafka-based tracking
-	// (Kafka 0.9 and later), the https://github.com/bsm/sarama-cluster library
-	// builds on Sarama to add this support.
-	Consumer struct {
-		Retry struct {
-			// How long to wait after a failing to read from a partition before
-			// trying again (default 2s).
-			Backoff time.Duration
-		}
-
-		// Fetch is the namespace for controlling how many bytes are retrieved by any
-		// given request.
-		Fetch struct {
-			// The minimum number of message bytes to fetch in a request - the broker
-			// will wait until at least this many are available. The default is 1,
-			// as 0 causes the consumer to spin when no messages are available.
-			// Equivalent to the JVM's `fetch.min.bytes`.
-			Min int32
-			// The default number of message bytes to fetch from the broker in each
-			// request (default 32768). This should be larger than the majority of
-			// your messages, or else the consumer will spend a lot of time
-			// negotiating sizes and not actually consuming. Similar to the JVM's
-			// `fetch.message.max.bytes`.
-			Default int32
-			// The maximum number of message bytes to fetch from the broker in a
-			// single request. Messages larger than this will return
-			// ErrMessageTooLarge and will not be consumable, so you must be sure
-			// this is at least as large as your largest message. Defaults to 0
-			// (no limit). Similar to the JVM's `fetch.message.max.bytes`. The
-			// global `sarama.MaxResponseSize` still applies.
-			Max int32
-		}
-		// The maximum amount of time the broker will wait for Consumer.Fetch.Min
-		// bytes to become available before it returns fewer than that anyways. The
-		// default is 250ms, since 0 causes the consumer to spin when no events are
-		// available. 100-500ms is a reasonable range for most cases. Kafka only
-		// supports precision up to milliseconds; nanoseconds will be truncated.
-		// Equivalent to the JVM's `fetch.wait.max.ms`.
-		MaxWaitTime time.Duration
-
-		// The maximum amount of time the consumer expects a message takes to
-		// process for the user. If writing to the Messages channel takes longer
-		// than this, that partition will stop fetching more messages until it
-		// can proceed again.
-		// Note that, since the Messages channel is buffered, the actual grace time is
-		// (MaxProcessingTime * ChanneBufferSize). Defaults to 100ms.
-		// If a message is not written to the Messages channel between two ticks
-		// of the expiryTicker then a timeout is detected.
-		// Using a ticker instead of a timer to detect timeouts should typically
-		// result in many fewer calls to Timer functions which may result in a
-		// significant performance improvement if many messages are being sent
-		// and timeouts are infrequent.
-		// The disadvantage of using a ticker instead of a timer is that
-		// timeouts will be less accurate. That is, the effective timeout could
-		// be between `MaxProcessingTime` and `2 * MaxProcessingTime`. For
-		// example, if `MaxProcessingTime` is 100ms then a delay of 180ms
-		// between two messages being sent may not be recognized as a timeout.
-		MaxProcessingTime time.Duration
-
-		// Return specifies what channels will be populated. If they are set to true,
-		// you must read from them to prevent deadlock.
-		Return struct {
-			// If enabled, any errors that occurred while consuming are returned on
-			// the Errors channel (default disabled).
-			Errors bool
-		}
-
-		// Offsets specifies configuration for how and when to commit consumed
-		// offsets. This currently requires the manual use of an OffsetManager
-		// but will eventually be automated.
-		Offsets struct {
-			// How frequently to commit updated offsets. Defaults to 1s.
-			CommitInterval time.Duration
-
-			// The initial offset to use if no offset was previously committed.
-			// Should be OffsetNewest or OffsetOldest. Defaults to OffsetNewest.
-			Initial int64
-
-			// The retention duration for committed offsets. If zero, disabled
-			// (in which case the `offsets.retention.minutes` option on the
-			// broker will be used).  Kafka only supports precision up to
-			// milliseconds; nanoseconds will be truncated. Requires Kafka
-			// broker version 0.9.0 or later.
-			// (default is 0: disabled).
-			Retention time.Duration
-		}
-	}
-
-	// A user-provided string sent with every request to the brokers for logging,
-	// debugging, and auditing purposes. Defaults to "sarama", but you should
-	// probably set it to something specific to your application.
-	ClientID string
-	// The number of events to buffer in internal and external channels. This
-	// permits the producer and consumer to continue processing some messages
-	// in the background while user code is working, greatly improving throughput.
-	// Defaults to 256.
-	ChannelBufferSize int
-	// The version of Kafka that Sarama will assume it is running against.
-	// Defaults to the oldest supported stable version. Since Kafka provides
-	// backwards-compatibility, setting it to a version older than you have
-	// will not break anything, although it may prevent you from using the
-	// latest features. Setting it to a version greater than you are actually
-	// running may lead to random breakage.
-	Version KafkaVersion
-	// The registry to define metrics into.
-	// Defaults to a local registry.
-	// If you want to disable metrics gathering, set "metrics.UseNilMetrics" to "true"
-	// prior to starting Sarama.
-	// See Examples on how to use the metrics registry
-	MetricRegistry metrics.Registry
-}
-
-// NewConfig returns a new configuration instance with sane defaults.
-func NewConfig() *Config {
-	c := &Config{}
-
-	c.Net.MaxOpenRequests = 5
-	c.Net.DialTimeout = 30 * time.Second
-	c.Net.ReadTimeout = 30 * time.Second
-	c.Net.WriteTimeout = 30 * time.Second
-	c.Net.SASL.Handshake = true
-
-	c.Metadata.Retry.Max = 3
-	c.Metadata.Retry.Backoff = 250 * time.Millisecond
-	c.Metadata.RefreshFrequency = 10 * time.Minute
-	c.Metadata.Full = true
-
-	c.Producer.MaxMessageBytes = 1000000
-	c.Producer.RequiredAcks = WaitForLocal
-	c.Producer.Timeout = 10 * time.Second
-	c.Producer.Partitioner = NewHashPartitioner
-	c.Producer.Retry.Max = 3
-	c.Producer.Retry.Backoff = 100 * time.Millisecond
-	c.Producer.Return.Errors = true
-
-	c.Consumer.Fetch.Min = 1
-	c.Consumer.Fetch.Default = 32768
-	c.Consumer.Retry.Backoff = 2 * time.Second
-	c.Consumer.MaxWaitTime = 250 * time.Millisecond
-	c.Consumer.MaxProcessingTime = 100 * time.Millisecond
-	c.Consumer.Return.Errors = false
-	c.Consumer.Offsets.CommitInterval = 1 * time.Second
-	c.Consumer.Offsets.Initial = OffsetNewest
-
-	c.ClientID = defaultClientID
-	c.ChannelBufferSize = 256
-	c.Version = minVersion
-	c.MetricRegistry = metrics.NewRegistry()
-
-	return c
-}
-
-// Validate checks a Config instance. It will return a
-// ConfigurationError if the specified values don't make sense.
-func (c *Config) Validate() error {
-	// some configuration values should be warned on but not fail completely, do those first
-	if c.Net.TLS.Enable == false && c.Net.TLS.Config != nil {
-		Logger.Println("Net.TLS is disabled but a non-nil configuration was provided.")
-	}
-	if c.Net.SASL.Enable == false {
-		if c.Net.SASL.User != "" {
-			Logger.Println("Net.SASL is disabled but a non-empty username was provided.")
-		}
-		if c.Net.SASL.Password != "" {
-			Logger.Println("Net.SASL is disabled but a non-empty password was provided.")
-		}
-	}
-	if c.Producer.RequiredAcks > 1 {
-		Logger.Println("Producer.RequiredAcks > 1 is deprecated and will raise an exception with kafka >= 0.8.2.0.")
-	}
-	if c.Producer.MaxMessageBytes >= int(MaxRequestSize) {
-		Logger.Println("Producer.MaxMessageBytes must be smaller than MaxRequestSize; it will be ignored.")
-	}
-	if c.Producer.Flush.Bytes >= int(MaxRequestSize) {
-		Logger.Println("Producer.Flush.Bytes must be smaller than MaxRequestSize; it will be ignored.")
-	}
-	if (c.Producer.Flush.Bytes > 0 || c.Producer.Flush.Messages > 0) && c.Producer.Flush.Frequency == 0 {
-		Logger.Println("Producer.Flush: Bytes or Messages are set, but Frequency is not; messages may not get flushed.")
-	}
-	if c.Producer.Timeout%time.Millisecond != 0 {
-		Logger.Println("Producer.Timeout only supports millisecond resolution; nanoseconds will be truncated.")
-	}
-	if c.Consumer.MaxWaitTime < 100*time.Millisecond {
-		Logger.Println("Consumer.MaxWaitTime is very low, which can cause high CPU and network usage. See documentation for details.")
-	}
-	if c.Consumer.MaxWaitTime%time.Millisecond != 0 {
-		Logger.Println("Consumer.MaxWaitTime only supports millisecond precision; nanoseconds will be truncated.")
-	}
-	if c.Consumer.Offsets.Retention%time.Millisecond != 0 {
-		Logger.Println("Consumer.Offsets.Retention only supports millisecond precision; nanoseconds will be truncated.")
-	}
-	if c.ClientID == defaultClientID {
-		Logger.Println("ClientID is the default of 'sarama', you should consider setting it to something application-specific.")
-	}
-
-	// validate Net values
-	switch {
-	case c.Net.MaxOpenRequests <= 0:
-		return ConfigurationError("Net.MaxOpenRequests must be > 0")
-	case c.Net.DialTimeout <= 0:
-		return ConfigurationError("Net.DialTimeout must be > 0")
-	case c.Net.ReadTimeout <= 0:
-		return ConfigurationError("Net.ReadTimeout must be > 0")
-	case c.Net.WriteTimeout <= 0:
-		return ConfigurationError("Net.WriteTimeout must be > 0")
-	case c.Net.KeepAlive < 0:
-		return ConfigurationError("Net.KeepAlive must be >= 0")
-	case c.Net.SASL.Enable == true && c.Net.SASL.User == "":
-		return ConfigurationError("Net.SASL.User must not be empty when SASL is enabled")
-	case c.Net.SASL.Enable == true && c.Net.SASL.Password == "":
-		return ConfigurationError("Net.SASL.Password must not be empty when SASL is enabled")
-	}
-
-	// validate the Metadata values
-	switch {
-	case c.Metadata.Retry.Max < 0:
-		return ConfigurationError("Metadata.Retry.Max must be >= 0")
-	case c.Metadata.Retry.Backoff < 0:
-		return ConfigurationError("Metadata.Retry.Backoff must be >= 0")
-	case c.Metadata.RefreshFrequency < 0:
-		return ConfigurationError("Metadata.RefreshFrequency must be >= 0")
-	}
-
-	// validate the Producer values
-	switch {
-	case c.Producer.MaxMessageBytes <= 0:
-		return ConfigurationError("Producer.MaxMessageBytes must be > 0")
-	case c.Producer.RequiredAcks < -1:
-		return ConfigurationError("Producer.RequiredAcks must be >= -1")
-	case c.Producer.Timeout <= 0:
-		return ConfigurationError("Producer.Timeout must be > 0")
-	case c.Producer.Partitioner == nil:
-		return ConfigurationError("Producer.Partitioner must not be nil")
-	case c.Producer.Flush.Bytes < 0:
-		return ConfigurationError("Producer.Flush.Bytes must be >= 0")
-	case c.Producer.Flush.Messages < 0:
-		return ConfigurationError("Producer.Flush.Messages must be >= 0")
-	case c.Producer.Flush.Frequency < 0:
-		return ConfigurationError("Producer.Flush.Frequency must be >= 0")
-	case c.Producer.Flush.MaxMessages < 0:
-		return ConfigurationError("Producer.Flush.MaxMessages must be >= 0")
-	case c.Producer.Flush.MaxMessages > 0 && c.Producer.Flush.MaxMessages < c.Producer.Flush.Messages:
-		return ConfigurationError("Producer.Flush.MaxMessages must be >= Producer.Flush.Messages when set")
-	case c.Producer.Retry.Max < 0:
-		return ConfigurationError("Producer.Retry.Max must be >= 0")
-	case c.Producer.Retry.Backoff < 0:
-		return ConfigurationError("Producer.Retry.Backoff must be >= 0")
-	}
-
-	if c.Producer.Compression == CompressionLZ4 && !c.Version.IsAtLeast(V0_10_0_0) {
-		return ConfigurationError("lz4 compression requires Version >= V0_10_0_0")
-	}
-
-	// validate the Consumer values
-	switch {
-	case c.Consumer.Fetch.Min <= 0:
-		return ConfigurationError("Consumer.Fetch.Min must be > 0")
-	case c.Consumer.Fetch.Default <= 0:
-		return ConfigurationError("Consumer.Fetch.Default must be > 0")
-	case c.Consumer.Fetch.Max < 0:
-		return ConfigurationError("Consumer.Fetch.Max must be >= 0")
-	case c.Consumer.MaxWaitTime < 1*time.Millisecond:
-		return ConfigurationError("Consumer.MaxWaitTime must be >= 1ms")
-	case c.Consumer.MaxProcessingTime <= 0:
-		return ConfigurationError("Consumer.MaxProcessingTime must be > 0")
-	case c.Consumer.Retry.Backoff < 0:
-		return ConfigurationError("Consumer.Retry.Backoff must be >= 0")
-	case c.Consumer.Offsets.CommitInterval <= 0:
-		return ConfigurationError("Consumer.Offsets.CommitInterval must be > 0")
-	case c.Consumer.Offsets.Initial != OffsetOldest && c.Consumer.Offsets.Initial != OffsetNewest:
-		return ConfigurationError("Consumer.Offsets.Initial must be OffsetOldest or OffsetNewest")
-
-	}
-
-	// validate misc shared values
-	switch {
-	case c.ChannelBufferSize < 0:
-		return ConfigurationError("ChannelBufferSize must be >= 0")
-	case !validID.MatchString(c.ClientID):
-		return ConfigurationError("ClientID is invalid")
-	}
-
-	return nil
-}

vendor/github.com/Shopify/sarama/config_test.go

@@ -1,233 +0,0 @@
-package sarama
-
-import (
-	"os"
-	"testing"
-
-	"github.com/rcrowley/go-metrics"
-)
-
-func TestDefaultConfigValidates(t *testing.T) {
-	config := NewConfig()
-	if err := config.Validate(); err != nil {
-		t.Error(err)
-	}
-	if config.MetricRegistry == nil {
-		t.Error("Expected non nil metrics.MetricRegistry, got nil")
-	}
-}
-
-func TestInvalidClientIDConfigValidates(t *testing.T) {
-	config := NewConfig()
-	config.ClientID = "foo:bar"
-	if err := config.Validate(); string(err.(ConfigurationError)) != "ClientID is invalid" {
-		t.Error("Expected invalid ClientID, got ", err)
-	}
-}
-
-func TestEmptyClientIDConfigValidates(t *testing.T) {
-	config := NewConfig()
-	config.ClientID = ""
-	if err := config.Validate(); string(err.(ConfigurationError)) != "ClientID is invalid" {
-		t.Error("Expected invalid ClientID, got ", err)
-	}
-}
-
-func TestNetConfigValidates(t *testing.T) {
-	tests := []struct {
-		name string
-		cfg  func(*Config) // resorting to using a function as a param because of internal composite structs
-		err  string
-	}{
-		{
-			"OpenRequests",
-			func(cfg *Config) {
-				cfg.Net.MaxOpenRequests = 0
-			},
-			"Net.MaxOpenRequests must be > 0"},
-		{"DialTimeout",
-			func(cfg *Config) {
-				cfg.Net.DialTimeout = 0
-			},
-			"Net.DialTimeout must be > 0"},
-		{"ReadTimeout",
-			func(cfg *Config) {
-				cfg.Net.ReadTimeout = 0
-			},
-			"Net.ReadTimeout must be > 0"},
-		{"WriteTimeout",
-			func(cfg *Config) {
-				cfg.Net.WriteTimeout = 0
-			},
-			"Net.WriteTimeout must be > 0"},
-		{"KeepAlive",
-			func(cfg *Config) {
-				cfg.Net.KeepAlive = -1
-			},
-			"Net.KeepAlive must be >= 0"},
-		{"SASL.User",
-			func(cfg *Config) {
-				cfg.Net.SASL.Enable = true
-				cfg.Net.SASL.User = ""
-			},
-			"Net.SASL.User must not be empty when SASL is enabled"},
-		{"SASL.Password",
-			func(cfg *Config) {
-				cfg.Net.SASL.Enable = true
-				cfg.Net.SASL.User = "user"
-				cfg.Net.SASL.Password = ""
-			},
-			"Net.SASL.Password must not be empty when SASL is enabled"},
-	}
-
-	for i, test := range tests {
-		c := NewConfig()
-		test.cfg(c)
-		if err := c.Validate(); string(err.(ConfigurationError)) != test.err {
-			t.Errorf("[%d]:[%s] Expected %s, Got %s\n", i, test.name, test.err, err)
-		}
-	}
-}
-
-func TestMetadataConfigValidates(t *testing.T) {
-	tests := []struct {
-		name string
-		cfg  func(*Config) // resorting to using a function as a param because of internal composite structs
-		err  string
-	}{
-		{
-			"Retry.Max",
-			func(cfg *Config) {
-				cfg.Metadata.Retry.Max = -1
-			},
-			"Metadata.Retry.Max must be >= 0"},
-		{"Retry.Backoff",
-			func(cfg *Config) {
-				cfg.Metadata.Retry.Backoff = -1
-			},
-			"Metadata.Retry.Backoff must be >= 0"},
-		{"RefreshFrequency",
-			func(cfg *Config) {
-				cfg.Metadata.RefreshFrequency = -1
-			},
-			"Metadata.RefreshFrequency must be >= 0"},
-	}
-
-	for i, test := range tests {
-		c := NewConfig()
-		test.cfg(c)
-		if err := c.Validate(); string(err.(ConfigurationError)) != test.err {
-			t.Errorf("[%d]:[%s] Expected %s, Got %s\n", i, test.name, test.err, err)
-		}
-	}
-}
-
-func TestProducerConfigValidates(t *testing.T) {
-	tests := []struct {
-		name string
-		cfg  func(*Config) // resorting to using a function as a param because of internal composite structs
-		err  string
-	}{
-		{
-			"MaxMessageBytes",
-			func(cfg *Config) {
-				cfg.Producer.MaxMessageBytes = 0
-			},
-			"Producer.MaxMessageBytes must be > 0"},
-		{"RequiredAcks",
-			func(cfg *Config) {
-				cfg.Producer.RequiredAcks = -2
-			},
-			"Producer.RequiredAcks must be >= -1"},
-		{"Timeout",
-			func(cfg *Config) {
-				cfg.Producer.Timeout = 0
-			},
-			"Producer.Timeout must be > 0"},
-		{"Partitioner",
-			func(cfg *Config) {
-				cfg.Producer.Partitioner = nil
-			},
-			"Producer.Partitioner must not be nil"},
-		{"Flush.Bytes",
-			func(cfg *Config) {
-				cfg.Producer.Flush.Bytes = -1
-			},
-			"Producer.Flush.Bytes must be >= 0"},
-		{"Flush.Messages",
-			func(cfg *Config) {
-				cfg.Producer.Flush.Messages = -1
-			},
-			"Producer.Flush.Messages must be >= 0"},
-		{"Flush.Frequency",
-			func(cfg *Config) {
-				cfg.Producer.Flush.Frequency = -1
-			},
-			"Producer.Flush.Frequency must be >= 0"},
-		{"Flush.MaxMessages",
-			func(cfg *Config) {
-				cfg.Producer.Flush.MaxMessages = -1
-			},
-			"Producer.Flush.MaxMessages must be >= 0"},
-		{"Flush.MaxMessages with Producer.Flush.Messages",
-			func(cfg *Config) {
-				cfg.Producer.Flush.MaxMessages = 1
-				cfg.Producer.Flush.Messages = 2
-			},
-			"Producer.Flush.MaxMessages must be >= Producer.Flush.Messages when set"},
-		{"Flush.Retry.Max",
-			func(cfg *Config) {
-				cfg.Producer.Retry.Max = -1
-			},
-			"Producer.Retry.Max must be >= 0"},
-		{"Flush.Retry.Backoff",
-			func(cfg *Config) {
-				cfg.Producer.Retry.Backoff = -1
-			},
-			"Producer.Retry.Backoff must be >= 0"},
-	}
-
-	for i, test := range tests {
-		c := NewConfig()
-		test.cfg(c)
-		if err := c.Validate(); string(err.(ConfigurationError)) != test.err {
-			t.Errorf("[%d]:[%s] Expected %s, Got %s\n", i, test.name, test.err, err)
-		}
-	}
-}
-
-func TestLZ4ConfigValidation(t *testing.T) {
-	config := NewConfig()
-	config.Producer.Compression = CompressionLZ4
-	if err := config.Validate(); string(err.(ConfigurationError)) != "lz4 compression requires Version >= V0_10_0_0" {
-		t.Error("Expected invalid lz4/kakfa version error, got ", err)
-	}
-	config.Version = V0_10_0_0
-	if err := config.Validate(); err != nil {
-		t.Error("Expected lz4 to work, got ", err)
-	}
-}
-
-// This example shows how to integrate with an existing registry as well as publishing metrics
-// on the standard output
-func ExampleConfig_metrics() {
-	// Our application registry
-	appMetricRegistry := metrics.NewRegistry()
-	appGauge := metrics.GetOrRegisterGauge("m1", appMetricRegistry)
-	appGauge.Update(1)
-
-	config := NewConfig()
-	// Use a prefix registry instead of the default local one
-	config.MetricRegistry = metrics.NewPrefixedChildRegistry(appMetricRegistry, "sarama.")
-
-	// Simulate a metric created by sarama without starting a broker
-	saramaGauge := metrics.GetOrRegisterGauge("m2", config.MetricRegistry)
-	saramaGauge.Update(2)
-
-	metrics.WriteOnce(appMetricRegistry, os.Stdout)
-	// Output:
-	// gauge m1
-	//   value:               1
-	// gauge sarama.m2
-	//   value:               2
-}

vendor/github.com/Shopify/sarama/consumer.go

@@ -1,806 +0,0 @@
-package sarama
-
-import (
-	"errors"
-	"fmt"
-	"sync"
-	"sync/atomic"
-	"time"
-)
-
-// ConsumerMessage encapsulates a Kafka message returned by the consumer.
-type ConsumerMessage struct {
-	Key, Value     []byte
-	Topic          string
-	Partition      int32
-	Offset         int64
-	Timestamp      time.Time       // only set if kafka is version 0.10+, inner message timestamp
-	BlockTimestamp time.Time       // only set if kafka is version 0.10+, outer (compressed) block timestamp
-	Headers        []*RecordHeader // only set if kafka is version 0.11+
-}
-
-// ConsumerError is what is provided to the user when an error occurs.
-// It wraps an error and includes the topic and partition.
-type ConsumerError struct {
-	Topic     string
-	Partition int32
-	Err       error
-}
-
-func (ce ConsumerError) Error() string {
-	return fmt.Sprintf("kafka: error while consuming %s/%d: %s", ce.Topic, ce.Partition, ce.Err)
-}
-
-// ConsumerErrors is a type that wraps a batch of errors and implements the Error interface.
-// It can be returned from the PartitionConsumer's Close methods to avoid the need to manually drain errors
-// when stopping.
-type ConsumerErrors []*ConsumerError
-
-func (ce ConsumerErrors) Error() string {
-	return fmt.Sprintf("kafka: %d errors while consuming", len(ce))
-}
-
-// Consumer manages PartitionConsumers which process Kafka messages from brokers. You MUST call Close()
-// on a consumer to avoid leaks, it will not be garbage-collected automatically when it passes out of
-// scope.
-//
-// Sarama's Consumer type does not currently support automatic consumer-group rebalancing and offset tracking.
-// For Zookeeper-based tracking (Kafka 0.8.2 and earlier), the https://github.com/wvanbergen/kafka library
-// builds on Sarama to add this support. For Kafka-based tracking (Kafka 0.9 and later), the
-// https://github.com/bsm/sarama-cluster library builds on Sarama to add this support.
-type Consumer interface {
-
-	// Topics returns the set of available topics as retrieved from the cluster
-	// metadata. This method is the same as Client.Topics(), and is provided for
-	// convenience.
-	Topics() ([]string, error)
-
-	// Partitions returns the sorted list of all partition IDs for the given topic.
-	// This method is the same as Client.Partitions(), and is provided for convenience.
-	Partitions(topic string) ([]int32, error)
-
-	// ConsumePartition creates a PartitionConsumer on the given topic/partition with
-	// the given offset. It will return an error if this Consumer is already consuming
-	// on the given topic/partition. Offset can be a literal offset, or OffsetNewest
-	// or OffsetOldest
-	ConsumePartition(topic string, partition int32, offset int64) (PartitionConsumer, error)
-
-	// HighWaterMarks returns the current high water marks for each topic and partition.
-	// Consistency between partitions is not guaranteed since high water marks are updated separately.
-	HighWaterMarks() map[string]map[int32]int64
-
-	// Close shuts down the consumer. It must be called after all child
-	// PartitionConsumers have already been closed.
-	Close() error
-}
-
-type consumer struct {
-	client    Client
-	conf      *Config
-	ownClient bool
-
-	lock            sync.Mutex
-	children        map[string]map[int32]*partitionConsumer
-	brokerConsumers map[*Broker]*brokerConsumer
-}
-
-// NewConsumer creates a new consumer using the given broker addresses and configuration.
-func NewConsumer(addrs []string, config *Config) (Consumer, error) {
-	client, err := NewClient(addrs, config)
-	if err != nil {
-		return nil, err
-	}
-
-	c, err := NewConsumerFromClient(client)
-	if err != nil {
-		return nil, err
-	}
-	c.(*consumer).ownClient = true
-	return c, nil
-}
-
-// NewConsumerFromClient creates a new consumer using the given client. It is still
-// necessary to call Close() on the underlying client when shutting down this consumer.
-func NewConsumerFromClient(client Client) (Consumer, error) {
-	// Check that we are not dealing with a closed Client before processing any other arguments
-	if client.Closed() {
-		return nil, ErrClosedClient
-	}
-
-	c := &consumer{
-		client:          client,
-		conf:            client.Config(),
-		children:        make(map[string]map[int32]*partitionConsumer),
-		brokerConsumers: make(map[*Broker]*brokerConsumer),
-	}
-
-	return c, nil
-}
-
-func (c *consumer) Close() error {
-	if c.ownClient {
-		return c.client.Close()
-	}
-	return nil
-}
-
-func (c *consumer) Topics() ([]string, error) {
-	return c.client.Topics()
-}
-
-func (c *consumer) Partitions(topic string) ([]int32, error) {
-	return c.client.Partitions(topic)
-}
-
-func (c *consumer) ConsumePartition(topic string, partition int32, offset int64) (PartitionConsumer, error) {
-	child := &partitionConsumer{
-		consumer:  c,
-		conf:      c.conf,
-		topic:     topic,
-		partition: partition,
-		messages:  make(chan *ConsumerMessage, c.conf.ChannelBufferSize),
-		errors:    make(chan *ConsumerError, c.conf.ChannelBufferSize),
-		feeder:    make(chan *FetchResponse, 1),
-		trigger:   make(chan none, 1),
-		dying:     make(chan none),
-		fetchSize: c.conf.Consumer.Fetch.Default,
-	}
-
-	if err := child.chooseStartingOffset(offset); err != nil {
-		return nil, err
-	}
-
-	var leader *Broker
-	var err error
-	if leader, err = c.client.Leader(child.topic, child.partition); err != nil {
-		return nil, err
-	}
-
-	if err := c.addChild(child); err != nil {
-		return nil, err
-	}
-
-	go withRecover(child.dispatcher)
-	go withRecover(child.responseFeeder)
-
-	child.broker = c.refBrokerConsumer(leader)
-	child.broker.input <- child
-
-	return child, nil
-}
-
-func (c *consumer) HighWaterMarks() map[string]map[int32]int64 {
-	c.lock.Lock()
-	defer c.lock.Unlock()
-
-	hwms := make(map[string]map[int32]int64)
-	for topic, p := range c.children {
-		hwm := make(map[int32]int64, len(p))
-		for partition, pc := range p {
-			hwm[partition] = pc.HighWaterMarkOffset()
-		}
-		hwms[topic] = hwm
-	}
-
-	return hwms
-}
-
-func (c *consumer) addChild(child *partitionConsumer) error {
-	c.lock.Lock()
-	defer c.lock.Unlock()
-
-	topicChildren := c.children[child.topic]
-	if topicChildren == nil {
-		topicChildren = make(map[int32]*partitionConsumer)
-		c.children[child.topic] = topicChildren
-	}
-
-	if topicChildren[child.partition] != nil {
-		return ConfigurationError("That topic/partition is already being consumed")
-	}
-
-	topicChildren[child.partition] = child
-	return nil
-}
-
-func (c *consumer) removeChild(child *partitionConsumer) {
-	c.lock.Lock()
-	defer c.lock.Unlock()
-
-	delete(c.children[child.topic], child.partition)
-}
-
-func (c *consumer) refBrokerConsumer(broker *Broker) *brokerConsumer {
-	c.lock.Lock()
-	defer c.lock.Unlock()
-
-	bc := c.brokerConsumers[broker]
-	if bc == nil {
-		bc = c.newBrokerConsumer(broker)
-		c.brokerConsumers[broker] = bc
-	}
-
-	bc.refs++
-
-	return bc
-}
-
-func (c *consumer) unrefBrokerConsumer(brokerWorker *brokerConsumer) {
-	c.lock.Lock()
-	defer c.lock.Unlock()
-
-	brokerWorker.refs--
-
-	if brokerWorker.refs == 0 {
-		close(brokerWorker.input)
-		if c.brokerConsumers[brokerWorker.broker] == brokerWorker {
-			delete(c.brokerConsumers, brokerWorker.broker)
-		}
-	}
-}
-
-func (c *consumer) abandonBrokerConsumer(brokerWorker *brokerConsumer) {
-	c.lock.Lock()
-	defer c.lock.Unlock()
-
-	delete(c.brokerConsumers, brokerWorker.broker)
-}
-
-// PartitionConsumer
-
-// PartitionConsumer processes Kafka messages from a given topic and partition. You MUST call one of Close() or
-// AsyncClose() on a PartitionConsumer to avoid leaks; it will not be garbage-collected automatically when it passes out
-// of scope.
-//
-// The simplest way of using a PartitionConsumer is to loop over its Messages channel using a for/range
-// loop. The PartitionConsumer will only stop itself in one case: when the offset being consumed is reported
-// as out of range by the brokers. In this case you should decide what you want to do (try a different offset,
-// notify a human, etc) and handle it appropriately. For all other error cases, it will just keep retrying.
-// By default, it logs these errors to sarama.Logger; if you want to be notified directly of all errors, set
-// your config's Consumer.Return.Errors to true and read from the Errors channel, using a select statement
-// or a separate goroutine. Check out the Consumer examples to see implementations of these different approaches.
-//
-// To terminate such a for/range loop while the loop is executing, call AsyncClose. This will kick off the process of
-// consumer tear-down & return imediately. Continue to loop, servicing the Messages channel until the teardown process
-// AsyncClose initiated closes it (thus terminating the for/range loop). If you've already ceased reading Messages, call
-// Close; this will signal the PartitionConsumer's goroutines to begin shutting down (just like AsyncClose), but will
-// also drain the Messages channel, harvest all errors & return them once cleanup has completed.
-type PartitionConsumer interface {
-
-	// AsyncClose initiates a shutdown of the PartitionConsumer. This method will return immediately, after which you
-	// should continue to service the 'Messages' and 'Errors' channels until they are empty. It is required to call this
-	// function, or Close before a consumer object passes out of scope, as it will otherwise leak memory. You must call
-	// this before calling Close on the underlying client.
-	AsyncClose()
-
-	// Close stops the PartitionConsumer from fetching messages. It will initiate a shutdown just like AsyncClose, drain
-	// the Messages channel, harvest any errors & return them to the caller. Note that if you are continuing to service
-	// the Messages channel when this function is called, you will be competing with Close for messages; consider
-	// calling AsyncClose, instead. It is required to call this function (or AsyncClose) before a consumer object passes
-	// out of scope, as it will otherwise leak memory. You must call this before calling Close on the underlying client.
-	Close() error
-
-	// Messages returns the read channel for the messages that are returned by
-	// the broker.
-	Messages() <-chan *ConsumerMessage
-
-	// Errors returns a read channel of errors that occurred during consuming, if
-	// enabled. By default, errors are logged and not returned over this channel.
-	// If you want to implement any custom error handling, set your config's
-	// Consumer.Return.Errors setting to true, and read from this channel.
-	Errors() <-chan *ConsumerError
-
-	// HighWaterMarkOffset returns the high water mark offset of the partition,
-	// i.e. the offset that will be used for the next message that will be produced.
-	// You can use this to determine how far behind the processing is.
-	HighWaterMarkOffset() int64
-}
-
-type partitionConsumer struct {
-	highWaterMarkOffset int64 // must be at the top of the struct because https://golang.org/pkg/sync/atomic/#pkg-note-BUG
-	consumer            *consumer
-	conf                *Config
-	topic               string
-	partition           int32
-
-	broker   *brokerConsumer
-	messages chan *ConsumerMessage
-	errors   chan *ConsumerError
-	feeder   chan *FetchResponse
-
-	trigger, dying chan none
-	responseResult error
-
-	fetchSize int32
-	offset    int64
-}
-
-var errTimedOut = errors.New("timed out feeding messages to the user") // not user-facing
-
-func (child *partitionConsumer) sendError(err error) {
-	cErr := &ConsumerError{
-		Topic:     child.topic,
-		Partition: child.partition,
-		Err:       err,
-	}
-
-	if child.conf.Consumer.Return.Errors {
-		child.errors <- cErr
-	} else {
-		Logger.Println(cErr)
-	}
-}
-
-func (child *partitionConsumer) dispatcher() {
-	for range child.trigger {
-		select {
-		case <-child.dying:
-			close(child.trigger)
-		case <-time.After(child.conf.Consumer.Retry.Backoff):
-			if child.broker != nil {
-				child.consumer.unrefBrokerConsumer(child.broker)
-				child.broker = nil
-			}
-
-			Logger.Printf("consumer/%s/%d finding new broker\n", child.topic, child.partition)
-			if err := child.dispatch(); err != nil {
-				child.sendError(err)
-				child.trigger <- none{}
-			}
-		}
-	}
-
-	if child.broker != nil {
-		child.consumer.unrefBrokerConsumer(child.broker)
-	}
-	child.consumer.removeChild(child)
-	close(child.feeder)
-}
-
-func (child *partitionConsumer) dispatch() error {
-	if err := child.consumer.client.RefreshMetadata(child.topic); err != nil {
-		return err
-	}
-
-	var leader *Broker
-	var err error
-	if leader, err = child.consumer.client.Leader(child.topic, child.partition); err != nil {
-		return err
-	}
-
-	child.broker = child.consumer.refBrokerConsumer(leader)
-
-	child.broker.input <- child
-
-	return nil
-}
-
-func (child *partitionConsumer) chooseStartingOffset(offset int64) error {
-	newestOffset, err := child.consumer.client.GetOffset(child.topic, child.partition, OffsetNewest)
-	if err != nil {
-		return err
-	}
-	oldestOffset, err := child.consumer.client.GetOffset(child.topic, child.partition, OffsetOldest)
-	if err != nil {
-		return err
-	}
-
-	switch {
-	case offset == OffsetNewest:
-		child.offset = newestOffset
-	case offset == OffsetOldest:
-		child.offset = oldestOffset
-	case offset >= oldestOffset && offset <= newestOffset:
-		child.offset = offset
-	default:
-		return ErrOffsetOutOfRange
-	}
-
-	return nil
-}
-
-func (child *partitionConsumer) Messages() <-chan *ConsumerMessage {
-	return child.messages
-}
-
-func (child *partitionConsumer) Errors() <-chan *ConsumerError {
-	return child.errors
-}
-
-func (child *partitionConsumer) AsyncClose() {
-	// this triggers whatever broker owns this child to abandon it and close its trigger channel, which causes
-	// the dispatcher to exit its loop, which removes it from the consumer then closes its 'messages' and
-	// 'errors' channel (alternatively, if the child is already at the dispatcher for some reason, that will
-	// also just close itself)
-	close(child.dying)
-}
-
-func (child *partitionConsumer) Close() error {
-	child.AsyncClose()
-
-	go withRecover(func() {
-		for range child.messages {
-			// drain
-		}
-	})
-
-	var errors ConsumerErrors
-	for err := range child.errors {
-		errors = append(errors, err)
-	}
-
-	if len(errors) > 0 {
-		return errors
-	}
-	return nil
-}
-
-func (child *partitionConsumer) HighWaterMarkOffset() int64 {
-	return atomic.LoadInt64(&child.highWaterMarkOffset)
-}
-
-func (child *partitionConsumer) responseFeeder() {
-	var msgs []*ConsumerMessage
-	msgSent := false
-
-feederLoop:
-	for response := range child.feeder {
-		msgs, child.responseResult = child.parseResponse(response)
-		expiryTicker := time.NewTicker(child.conf.Consumer.MaxProcessingTime)
-
-		for i, msg := range msgs {
-		messageSelect:
-			select {
-			case child.messages <- msg:
-				msgSent = true
-			case <-expiryTicker.C:
-				if !msgSent {
-					child.responseResult = errTimedOut
-					child.broker.acks.Done()
-					for _, msg = range msgs[i:] {
-						child.messages <- msg
-					}
-					child.broker.input <- child
-					continue feederLoop
-				} else {
-					// current message has not been sent, return to select
-					// statement
-					msgSent = false
-					goto messageSelect
-				}
-			}
-		}
-
-		expiryTicker.Stop()
-		child.broker.acks.Done()
-	}
-
-	close(child.messages)
-	close(child.errors)
-}
-
-func (child *partitionConsumer) parseMessages(msgSet *MessageSet) ([]*ConsumerMessage, error) {
-	var messages []*ConsumerMessage
-	var incomplete bool
-	prelude := true
-
-	for _, msgBlock := range msgSet.Messages {
-		for _, msg := range msgBlock.Messages() {
-			offset := msg.Offset
-			if msg.Msg.Version >= 1 {
-				baseOffset := msgBlock.Offset - msgBlock.Messages()[len(msgBlock.Messages())-1].Offset
-				offset += baseOffset
-			}
-			if prelude && offset < child.offset {
-				continue
-			}
-			prelude = false
-
-			if offset >= child.offset {
-				messages = append(messages, &ConsumerMessage{
-					Topic:          child.topic,
-					Partition:      child.partition,
-					Key:            msg.Msg.Key,
-					Value:          msg.Msg.Value,
-					Offset:         offset,
-					Timestamp:      msg.Msg.Timestamp,
-					BlockTimestamp: msgBlock.Msg.Timestamp,
-				})
-				child.offset = offset + 1
-			} else {
-				incomplete = true
-			}
-		}
-	}
-
-	if incomplete || len(messages) == 0 {
-		return nil, ErrIncompleteResponse
-	}
-	return messages, nil
-}
-
-func (child *partitionConsumer) parseRecords(batch *RecordBatch) ([]*ConsumerMessage, error) {
-	var messages []*ConsumerMessage
-	var incomplete bool
-	prelude := true
-
-	for _, rec := range batch.Records {
-		offset := batch.FirstOffset + rec.OffsetDelta
-		if prelude && offset < child.offset {
-			continue
-		}
-		prelude = false
-
-		if offset >= child.offset {
-			messages = append(messages, &ConsumerMessage{
-				Topic:     child.topic,
-				Partition: child.partition,
-				Key:       rec.Key,
-				Value:     rec.Value,
-				Offset:    offset,
-				Timestamp: batch.FirstTimestamp.Add(rec.TimestampDelta),
-				Headers:   rec.Headers,
-			})
-			child.offset = offset + 1
-		} else {
-			incomplete = true
-		}
-	}
-
-	if incomplete || len(messages) == 0 {
-		return nil, ErrIncompleteResponse
-	}
-	return messages, nil
-}
-
-func (child *partitionConsumer) parseResponse(response *FetchResponse) ([]*ConsumerMessage, error) {
-	block := response.GetBlock(child.topic, child.partition)
-	if block == nil {
-		return nil, ErrIncompleteResponse
-	}
-
-	if block.Err != ErrNoError {
-		return nil, block.Err
-	}
-
-	nRecs, err := block.Records.numRecords()
-	if err != nil {
-		return nil, err
-	}
-	if nRecs == 0 {
-		partialTrailingMessage, err := block.Records.isPartial()
-		if err != nil {
-			return nil, err
-		}
-		// We got no messages. If we got a trailing one then we need to ask for more data.
-		// Otherwise we just poll again and wait for one to be produced...
-		if partialTrailingMessage {
-			if child.conf.Consumer.Fetch.Max > 0 && child.fetchSize == child.conf.Consumer.Fetch.Max {
-				// we can't ask for more data, we've hit the configured limit
-				child.sendError(ErrMessageTooLarge)
-				child.offset++ // skip this one so we can keep processing future messages
-			} else {
-				child.fetchSize *= 2
-				if child.conf.Consumer.Fetch.Max > 0 && child.fetchSize > child.conf.Consumer.Fetch.Max {
-					child.fetchSize = child.conf.Consumer.Fetch.Max
-				}
-			}
-		}
-
-		return nil, nil
-	}
-
-	// we got messages, reset our fetch size in case it was increased for a previous request
-	child.fetchSize = child.conf.Consumer.Fetch.Default
-	atomic.StoreInt64(&child.highWaterMarkOffset, block.HighWaterMarkOffset)
-
-	if control, err := block.Records.isControl(); err != nil || control {
-		return nil, err
-	}
-
-	if block.Records.recordsType == legacyRecords {
-		return child.parseMessages(block.Records.msgSet)
-	}
-	return child.parseRecords(block.Records.recordBatch)
-}
-
-// brokerConsumer
-
-type brokerConsumer struct {
-	consumer         *consumer
-	broker           *Broker
-	input            chan *partitionConsumer
-	newSubscriptions chan []*partitionConsumer
-	wait             chan none
-	subscriptions    map[*partitionConsumer]none
-	acks             sync.WaitGroup
-	refs             int
-}
-
-func (c *consumer) newBrokerConsumer(broker *Broker) *brokerConsumer {
-	bc := &brokerConsumer{
-		consumer:         c,
-		broker:           broker,
-		input:            make(chan *partitionConsumer),
-		newSubscriptions: make(chan []*partitionConsumer),
-		wait:             make(chan none),
-		subscriptions:    make(map[*partitionConsumer]none),
-		refs:             0,
-	}
-
-	go withRecover(bc.subscriptionManager)
-	go withRecover(bc.subscriptionConsumer)
-
-	return bc
-}
-
-func (bc *brokerConsumer) subscriptionManager() {
-	var buffer []*partitionConsumer
-
-	// The subscriptionManager constantly accepts new subscriptions on `input` (even when the main subscriptionConsumer
-	// goroutine is in the middle of a network request) and batches it up. The main worker goroutine picks
-	// up a batch of new subscriptions between every network request by reading from `newSubscriptions`, so we give
-	// it nil if no new subscriptions are available. We also write to `wait` only when new subscriptions is available,
-	// so the main goroutine can block waiting for work if it has none.
-	for {
-		if len(buffer) > 0 {
-			select {
-			case event, ok := <-bc.input:
-				if !ok {
-					goto done
-				}
-				buffer = append(buffer, event)
-			case bc.newSubscriptions <- buffer:
-				buffer = nil
-			case bc.wait <- none{}:
-			}
-		} else {
-			select {
-			case event, ok := <-bc.input:
-				if !ok {
-					goto done
-				}
-				buffer = append(buffer, event)
-			case bc.newSubscriptions <- nil:
-			}
-		}
-	}
-
-done:
-	close(bc.wait)
-	if len(buffer) > 0 {
-		bc.newSubscriptions <- buffer
-	}
-	close(bc.newSubscriptions)
-}
-
-func (bc *brokerConsumer) subscriptionConsumer() {
-	<-bc.wait // wait for our first piece of work
-
-	// the subscriptionConsumer ensures we will get nil right away if no new subscriptions is available
-	for newSubscriptions := range bc.newSubscriptions {
-		bc.updateSubscriptions(newSubscriptions)
-
-		if len(bc.subscriptions) == 0 {
-			// We're about to be shut down or we're about to receive more subscriptions.
-			// Either way, the signal just hasn't propagated to our goroutine yet.
-			<-bc.wait
-			continue
-		}
-
-		response, err := bc.fetchNewMessages()
-
-		if err != nil {
-			Logger.Printf("consumer/broker/%d disconnecting due to error processing FetchRequest: %s\n", bc.broker.ID(), err)
-			bc.abort(err)
-			return
-		}
-
-		bc.acks.Add(len(bc.subscriptions))
-		for child := range bc.subscriptions {
-			child.feeder <- response
-		}
-		bc.acks.Wait()
-		bc.handleResponses()
-	}
-}
-
-func (bc *brokerConsumer) updateSubscriptions(newSubscriptions []*partitionConsumer) {
-	for _, child := range newSubscriptions {
-		bc.subscriptions[child] = none{}
-		Logger.Printf("consumer/broker/%d added subscription to %s/%d\n", bc.broker.ID(), child.topic, child.partition)
-	}
-
-	for child := range bc.subscriptions {
-		select {
-		case <-child.dying:
-			Logger.Printf("consumer/broker/%d closed dead subscription to %s/%d\n", bc.broker.ID(), child.topic, child.partition)
-			close(child.trigger)
-			delete(bc.subscriptions, child)
-		default:
-			break
-		}
-	}
-}
-
-func (bc *brokerConsumer) handleResponses() {
-	// handles the response codes left for us by our subscriptions, and abandons ones that have been closed
-	for child := range bc.subscriptions {
-		result := child.responseResult
-		child.responseResult = nil
-
-		switch result {
-		case nil:
-			break
-		case errTimedOut:
-			Logger.Printf("consumer/broker/%d abandoned subscription to %s/%d because consuming was taking too long\n",
-				bc.broker.ID(), child.topic, child.partition)
-			delete(bc.subscriptions, child)
-		case ErrOffsetOutOfRange:
-			// there's no point in retrying this it will just fail the same way again
-			// shut it down and force the user to choose what to do
-			child.sendError(result)
-			Logger.Printf("consumer/%s/%d shutting down because %s\n", child.topic, child.partition, result)
-			close(child.trigger)
-			delete(bc.subscriptions, child)
-		case ErrUnknownTopicOrPartition, ErrNotLeaderForPartition, ErrLeaderNotAvailable, ErrReplicaNotAvailable:
-			// not an error, but does need redispatching
-			Logger.Printf("consumer/broker/%d abandoned subscription to %s/%d because %s\n",
-				bc.broker.ID(), child.topic, child.partition, result)
-			child.trigger <- none{}
-			delete(bc.subscriptions, child)
-		default:
-			// dunno, tell the user and try redispatching
-			child.sendError(result)
-			Logger.Printf("consumer/broker/%d abandoned subscription to %s/%d because %s\n",
-				bc.broker.ID(), child.topic, child.partition, result)
-			child.trigger <- none{}
-			delete(bc.subscriptions, child)
-		}
-	}
-}
-
-func (bc *brokerConsumer) abort(err error) {
-	bc.consumer.abandonBrokerConsumer(bc)
-	_ = bc.broker.Close() // we don't care about the error this might return, we already have one
-
-	for child := range bc.subscriptions {
-		child.sendError(err)
-		child.trigger <- none{}
-	}
-
-	for newSubscriptions := range bc.newSubscriptions {
-		if len(newSubscriptions) == 0 {
-			<-bc.wait
-			continue
-		}
-		for _, child := range newSubscriptions {
-			child.sendError(err)
-			child.trigger <- none{}
-		}
-	}
-}
-
-func (bc *brokerConsumer) fetchNewMessages() (*FetchResponse, error) {
-	request := &FetchRequest{
-		MinBytes:    bc.consumer.conf.Consumer.Fetch.Min,
-		MaxWaitTime: int32(bc.consumer.conf.Consumer.MaxWaitTime / time.Millisecond),
-	}
-	if bc.consumer.conf.Version.IsAtLeast(V0_10_0_0) {
-		request.Version = 2
-	}
-	if bc.consumer.conf.Version.IsAtLeast(V0_10_1_0) {
-		request.Version = 3
-		request.MaxBytes = MaxResponseSize
-	}
-	if bc.consumer.conf.Version.IsAtLeast(V0_11_0_0) {
-		request.Version = 4
-		request.Isolation = ReadUncommitted // We don't support yet transactions.
-	}
-
-	for child := range bc.subscriptions {
-		request.AddBlock(child.topic, child.partition, child.offset, child.fetchSize)
-	}
-
-	return bc.broker.Fetch(request)
-}

vendor/github.com/Shopify/sarama/consumer_group_members.go

@@ -1,94 +0,0 @@
-package sarama
-
-type ConsumerGroupMemberMetadata struct {
-	Version  int16
-	Topics   []string
-	UserData []byte
-}
-
-func (m *ConsumerGroupMemberMetadata) encode(pe packetEncoder) error {
-	pe.putInt16(m.Version)
-
-	if err := pe.putStringArray(m.Topics); err != nil {
-		return err
-	}
-
-	if err := pe.putBytes(m.UserData); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func (m *ConsumerGroupMemberMetadata) decode(pd packetDecoder) (err error) {
-	if m.Version, err = pd.getInt16(); err != nil {
-		return
-	}
-
-	if m.Topics, err = pd.getStringArray(); err != nil {
-		return
-	}
-
-	if m.UserData, err = pd.getBytes(); err != nil {
-		return
-	}
-
-	return nil
-}
-
-type ConsumerGroupMemberAssignment struct {
-	Version  int16
-	Topics   map[string][]int32
-	UserData []byte
-}
-
-func (m *ConsumerGroupMemberAssignment) encode(pe packetEncoder) error {
-	pe.putInt16(m.Version)
-
-	if err := pe.putArrayLength(len(m.Topics)); err != nil {
-		return err
-	}
-
-	for topic, partitions := range m.Topics {
-		if err := pe.putString(topic); err != nil {
-			return err
-		}
-		if err := pe.putInt32Array(partitions); err != nil {
-			return err
-		}
-	}
-
-	if err := pe.putBytes(m.UserData); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func (m *ConsumerGroupMemberAssignment) decode(pd packetDecoder) (err error) {
-	if m.Version, err = pd.getInt16(); err != nil {
-		return
-	}
-
-	var topicLen int
-	if topicLen, err = pd.getArrayLength(); err != nil {
-		return
-	}
-
-	m.Topics = make(map[string][]int32, topicLen)
-	for i := 0; i < topicLen; i++ {
-		var topic string
-		if topic, err = pd.getString(); err != nil {
-			return
-		}
-		if m.Topics[topic], err = pd.getInt32Array(); err != nil {
-			return
-		}
-	}
-
-	if m.UserData, err = pd.getBytes(); err != nil {
-		return
-	}
-
-	return nil
-}

vendor/github.com/Shopify/sarama/consumer_group_members_test.go

@@ -1,73 +0,0 @@
-package sarama
-
-import (
-	"bytes"
-	"reflect"
-	"testing"
-)
-
-var (
-	groupMemberMetadata = []byte{
-		0, 1, // Version
-		0, 0, 0, 2, // Topic array length
-		0, 3, 'o', 'n', 'e', // Topic one
-		0, 3, 't', 'w', 'o', // Topic two
-		0, 0, 0, 3, 0x01, 0x02, 0x03, // Userdata
-	}
-	groupMemberAssignment = []byte{
-		0, 1, // Version
-		0, 0, 0, 1, // Topic array length
-		0, 3, 'o', 'n', 'e', // Topic one
-		0, 0, 0, 3, // Topic one, partition array length
-		0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4, // 0, 2, 4
-		0, 0, 0, 3, 0x01, 0x02, 0x03, // Userdata
-	}
-)
-
-func TestConsumerGroupMemberMetadata(t *testing.T) {
-	meta := &ConsumerGroupMemberMetadata{
-		Version:  1,
-		Topics:   []string{"one", "two"},
-		UserData: []byte{0x01, 0x02, 0x03},
-	}
-
-	buf, err := encode(meta, nil)
-	if err != nil {
-		t.Error("Failed to encode data", err)
-	} else if !bytes.Equal(groupMemberMetadata, buf) {
-		t.Errorf("Encoded data does not match expectation\nexpected: %v\nactual: %v", groupMemberMetadata, buf)
-	}
-
-	meta2 := new(ConsumerGroupMemberMetadata)
-	err = decode(buf, meta2)
-	if err != nil {
-		t.Error("Failed to decode data", err)
-	} else if !reflect.DeepEqual(meta, meta2) {
-		t.Errorf("Encoded data does not match expectation\nexpected: %v\nactual: %v", meta, meta2)
-	}
-}
-
-func TestConsumerGroupMemberAssignment(t *testing.T) {
-	amt := &ConsumerGroupMemberAssignment{
-		Version: 1,
-		Topics: map[string][]int32{
-			"one": {0, 2, 4},
-		},
-		UserData: []byte{0x01, 0x02, 0x03},
-	}
-
-	buf, err := encode(amt, nil)
-	if err != nil {
-		t.Error("Failed to encode data", err)
-	} else if !bytes.Equal(groupMemberAssignment, buf) {
-		t.Errorf("Encoded data does not match expectation\nexpected: %v\nactual: %v", groupMemberAssignment, buf)
-	}
-
-	amt2 := new(ConsumerGroupMemberAssignment)
-	err = decode(buf, amt2)
-	if err != nil {
-		t.Error("Failed to decode data", err)
-	} else if !reflect.DeepEqual(amt, amt2) {
-		t.Errorf("Encoded data does not match expectation\nexpected: %v\nactual: %v", amt, amt2)
-	}
-}

vendor/github.com/Shopify/sarama/consumer_metadata_request.go

@@ -1,26 +0,0 @@
-package sarama
-
-type ConsumerMetadataRequest struct {
-	ConsumerGroup string
-}
-
-func (r *ConsumerMetadataRequest) encode(pe packetEncoder) error {
-	return pe.putString(r.ConsumerGroup)
-}
-
-func (r *ConsumerMetadataRequest) decode(pd packetDecoder, version int16) (err error) {
-	r.ConsumerGroup, err = pd.getString()
-	return err
-}
-
-func (r *ConsumerMetadataRequest) key() int16 {
-	return 10
-}
-
-func (r *ConsumerMetadataRequest) version() int16 {
-	return 0
-}
-
-func (r *ConsumerMetadataRequest) requiredVersion() KafkaVersion {
-	return V0_8_2_0
-}

vendor/github.com/Shopify/sarama/consumer_metadata_request_test.go

@@ -1,19 +0,0 @@
-package sarama
-
-import "testing"
-
-var (
-	consumerMetadataRequestEmpty = []byte{
-		0x00, 0x00}
-
-	consumerMetadataRequestString = []byte{
-		0x00, 0x06, 'f', 'o', 'o', 'b', 'a', 'r'}
-)
-
-func TestConsumerMetadataRequest(t *testing.T) {
-	request := new(ConsumerMetadataRequest)
-	testRequest(t, "empty string", request, consumerMetadataRequestEmpty)
-
-	request.ConsumerGroup = "foobar"
-	testRequest(t, "with string", request, consumerMetadataRequestString)
-}

vendor/github.com/Shopify/sarama/consumer_metadata_response.go

@@ -1,85 +0,0 @@
-package sarama
-
-import (
-	"net"
-	"strconv"
-)
-
-type ConsumerMetadataResponse struct {
-	Err             KError
-	Coordinator     *Broker
-	CoordinatorID   int32  // deprecated: use Coordinator.ID()
-	CoordinatorHost string // deprecated: use Coordinator.Addr()
-	CoordinatorPort int32  // deprecated: use Coordinator.Addr()
-}
-
-func (r *ConsumerMetadataResponse) decode(pd packetDecoder, version int16) (err error) {
-	tmp, err := pd.getInt16()
-	if err != nil {
-		return err
-	}
-	r.Err = KError(tmp)
-
-	coordinator := new(Broker)
-	if err := coordinator.decode(pd); err != nil {
-		return err
-	}
-	if coordinator.addr == ":0" {
-		return nil
-	}
-	r.Coordinator = coordinator
-
-	// this can all go away in 2.0, but we have to fill in deprecated fields to maintain
-	// backwards compatibility
-	host, portstr, err := net.SplitHostPort(r.Coordinator.Addr())
-	if err != nil {
-		return err
-	}
-	port, err := strconv.ParseInt(portstr, 10, 32)
-	if err != nil {
-		return err
-	}
-	r.CoordinatorID = r.Coordinator.ID()
-	r.CoordinatorHost = host
-	r.CoordinatorPort = int32(port)
-
-	return nil
-}
-
-func (r *ConsumerMetadataResponse) encode(pe packetEncoder) error {
-	pe.putInt16(int16(r.Err))
-	if r.Coordinator != nil {
-		host, portstr, err := net.SplitHostPort(r.Coordinator.Addr())
-		if err != nil {
-			return err
-		}
-		port, err := strconv.ParseInt(portstr, 10, 32)
-		if err != nil {
-			return err
-		}
-		pe.putInt32(r.Coordinator.ID())
-		if err := pe.putString(host); err != nil {
-			return err
-		}
-		pe.putInt32(int32(port))
-		return nil
-	}
-	pe.putInt32(r.CoordinatorID)
-	if err := pe.putString(r.CoordinatorHost); err != nil {
-		return err
-	}
-	pe.putInt32(r.CoordinatorPort)
-	return nil
-}
-
-func (r *ConsumerMetadataResponse) key() int16 {
-	return 10
-}
-
-func (r *ConsumerMetadataResponse) version() int16 {
-	return 0
-}
-
-func (r *ConsumerMetadataResponse) requiredVersion() KafkaVersion {
-	return V0_8_2_0
-}

vendor/github.com/Shopify/sarama/consumer_metadata_response_test.go

@@ -1,35 +0,0 @@
-package sarama
-
-import "testing"
-
-var (
-	consumerMetadataResponseError = []byte{
-		0x00, 0x0E,
-		0x00, 0x00, 0x00, 0x00,
-		0x00, 0x00,
-		0x00, 0x00, 0x00, 0x00}
-
-	consumerMetadataResponseSuccess = []byte{
-		0x00, 0x00,
-		0x00, 0x00, 0x00, 0xAB,
-		0x00, 0x03, 'f', 'o', 'o',
-		0x00, 0x00, 0xCC, 0xDD}
-)
-
-func TestConsumerMetadataResponseError(t *testing.T) {
-	response := ConsumerMetadataResponse{Err: ErrOffsetsLoadInProgress}
-	testResponse(t, "error", &response, consumerMetadataResponseError)
-}
-
-func TestConsumerMetadataResponseSuccess(t *testing.T) {
-	broker := NewBroker("foo:52445")
-	broker.id = 0xAB
-	response := ConsumerMetadataResponse{
-		Coordinator:     broker,
-		CoordinatorID:   0xAB,
-		CoordinatorHost: "foo",
-		CoordinatorPort: 0xCCDD,
-		Err:             ErrNoError,
-	}
-	testResponse(t, "success", &response, consumerMetadataResponseSuccess)
-}

vendor/github.com/Shopify/sarama/consumer_test.go

@@ -1,971 +0,0 @@
-package sarama
-
-import (
-	"log"
-	"os"
-	"os/signal"
-	"sync"
-	"testing"
-	"time"
-)
-
-var testMsg = StringEncoder("Foo")
-
-// If a particular offset is provided then messages are consumed starting from
-// that offset.
-func TestConsumerOffsetManual(t *testing.T) {
-	// Given
-	broker0 := NewMockBroker(t, 0)
-
-	mockFetchResponse := NewMockFetchResponse(t, 1)
-	for i := 0; i < 10; i++ {
-		mockFetchResponse.SetMessage("my_topic", 0, int64(i+1234), testMsg)
-	}
-
-	broker0.SetHandlerByMap(map[string]MockResponse{
-		"MetadataRequest": NewMockMetadataResponse(t).
-			SetBroker(broker0.Addr(), broker0.BrokerID()).
-			SetLeader("my_topic", 0, broker0.BrokerID()),
-		"OffsetRequest": NewMockOffsetResponse(t).
-			SetOffset("my_topic", 0, OffsetOldest, 0).
-			SetOffset("my_topic", 0, OffsetNewest, 2345),
-		"FetchRequest": mockFetchResponse,
-	})
-
-	// When
-	master, err := NewConsumer([]string{broker0.Addr()}, nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	consumer, err := master.ConsumePartition("my_topic", 0, 1234)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// Then: messages starting from offset 1234 are consumed.
-	for i := 0; i < 10; i++ {
-		select {
-		case message := <-consumer.Messages():
-			assertMessageOffset(t, message, int64(i+1234))
-		case err := <-consumer.Errors():
-			t.Error(err)
-		}
-	}
-
-	safeClose(t, consumer)
-	safeClose(t, master)
-	broker0.Close()
-}
-
-// If `OffsetNewest` is passed as the initial offset then the first consumed
-// message is indeed corresponds to the offset that broker claims to be the
-// newest in its metadata response.
-func TestConsumerOffsetNewest(t *testing.T) {
-	// Given
-	broker0 := NewMockBroker(t, 0)
-	broker0.SetHandlerByMap(map[string]MockResponse{
-		"MetadataRequest": NewMockMetadataResponse(t).
-			SetBroker(broker0.Addr(), broker0.BrokerID()).
-			SetLeader("my_topic", 0, broker0.BrokerID()),
-		"OffsetRequest": NewMockOffsetResponse(t).
-			SetOffset("my_topic", 0, OffsetNewest, 10).
-			SetOffset("my_topic", 0, OffsetOldest, 7),
-		"FetchRequest": NewMockFetchResponse(t, 1).
-			SetMessage("my_topic", 0, 9, testMsg).
-			SetMessage("my_topic", 0, 10, testMsg).
-			SetMessage("my_topic", 0, 11, testMsg).
-			SetHighWaterMark("my_topic", 0, 14),
-	})
-
-	master, err := NewConsumer([]string{broker0.Addr()}, nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// When
-	consumer, err := master.ConsumePartition("my_topic", 0, OffsetNewest)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// Then
-	assertMessageOffset(t, <-consumer.Messages(), 10)
-	if hwmo := consumer.HighWaterMarkOffset(); hwmo != 14 {
-		t.Errorf("Expected high water mark offset 14, found %d", hwmo)
-	}
-
-	safeClose(t, consumer)
-	safeClose(t, master)
-	broker0.Close()
-}
-
-// It is possible to close a partition consumer and create the same anew.
-func TestConsumerRecreate(t *testing.T) {
-	// Given
-	broker0 := NewMockBroker(t, 0)
-	broker0.SetHandlerByMap(map[string]MockResponse{
-		"MetadataRequest": NewMockMetadataResponse(t).
-			SetBroker(broker0.Addr(), broker0.BrokerID()).
-			SetLeader("my_topic", 0, broker0.BrokerID()),
-		"OffsetRequest": NewMockOffsetResponse(t).
-			SetOffset("my_topic", 0, OffsetOldest, 0).
-			SetOffset("my_topic", 0, OffsetNewest, 1000),
-		"FetchRequest": NewMockFetchResponse(t, 1).
-			SetMessage("my_topic", 0, 10, testMsg),
-	})
-
-	c, err := NewConsumer([]string{broker0.Addr()}, nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	pc, err := c.ConsumePartition("my_topic", 0, 10)
-	if err != nil {
-		t.Fatal(err)
-	}
-	assertMessageOffset(t, <-pc.Messages(), 10)
-
-	// When
-	safeClose(t, pc)
-	pc, err = c.ConsumePartition("my_topic", 0, 10)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// Then
-	assertMessageOffset(t, <-pc.Messages(), 10)
-
-	safeClose(t, pc)
-	safeClose(t, c)
-	broker0.Close()
-}
-
-// An attempt to consume the same partition twice should fail.
-func TestConsumerDuplicate(t *testing.T) {
-	// Given
-	broker0 := NewMockBroker(t, 0)
-	broker0.SetHandlerByMap(map[string]MockResponse{
-		"MetadataRequest": NewMockMetadataResponse(t).
-			SetBroker(broker0.Addr(), broker0.BrokerID()).
-			SetLeader("my_topic", 0, broker0.BrokerID()),
-		"OffsetRequest": NewMockOffsetResponse(t).
-			SetOffset("my_topic", 0, OffsetOldest, 0).
-			SetOffset("my_topic", 0, OffsetNewest, 1000),
-		"FetchRequest": NewMockFetchResponse(t, 1),
-	})
-
-	config := NewConfig()
-	config.ChannelBufferSize = 0
-	c, err := NewConsumer([]string{broker0.Addr()}, config)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	pc1, err := c.ConsumePartition("my_topic", 0, 0)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// When
-	pc2, err := c.ConsumePartition("my_topic", 0, 0)
-
-	// Then
-	if pc2 != nil || err != ConfigurationError("That topic/partition is already being consumed") {
-		t.Fatal("A partition cannot be consumed twice at the same time")
-	}
-
-	safeClose(t, pc1)
-	safeClose(t, c)
-	broker0.Close()
-}
-
-// If consumer fails to refresh metadata it keeps retrying with frequency
-// specified by `Config.Consumer.Retry.Backoff`.
-func TestConsumerLeaderRefreshError(t *testing.T) {
-	// Given
-	broker0 := NewMockBroker(t, 100)
-
-	// Stage 1: my_topic/0 served by broker0
-	Logger.Printf("    STAGE 1")
-
-	broker0.SetHandlerByMap(map[string]MockResponse{
-		"MetadataRequest": NewMockMetadataResponse(t).
-			SetBroker(broker0.Addr(), broker0.BrokerID()).
-			SetLeader("my_topic", 0, broker0.BrokerID()),
-		"OffsetRequest": NewMockOffsetResponse(t).
-			SetOffset("my_topic", 0, OffsetOldest, 123).
-			SetOffset("my_topic", 0, OffsetNewest, 1000),
-		"FetchRequest": NewMockFetchResponse(t, 1).
-			SetMessage("my_topic", 0, 123, testMsg),
-	})
-
-	config := NewConfig()
-	config.Net.ReadTimeout = 100 * time.Millisecond
-	config.Consumer.Retry.Backoff = 200 * time.Millisecond
-	config.Consumer.Return.Errors = true
-	config.Metadata.Retry.Max = 0
-	c, err := NewConsumer([]string{broker0.Addr()}, config)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	pc, err := c.ConsumePartition("my_topic", 0, OffsetOldest)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	assertMessageOffset(t, <-pc.Messages(), 123)
-
-	// Stage 2: broker0 says that it is no longer the leader for my_topic/0,
-	// but the requests to retrieve metadata fail with network timeout.
-	Logger.Printf("    STAGE 2")
-
-	fetchResponse2 := &FetchResponse{}
-	fetchResponse2.AddError("my_topic", 0, ErrNotLeaderForPartition)
-
-	broker0.SetHandlerByMap(map[string]MockResponse{
-		"FetchRequest": NewMockWrapper(fetchResponse2),
-	})
-
-	if consErr := <-pc.Errors(); consErr.Err != ErrOutOfBrokers {
-		t.Errorf("Unexpected error: %v", consErr.Err)
-	}
-
-	// Stage 3: finally the metadata returned by broker0 tells that broker1 is
-	// a new leader for my_topic/0. Consumption resumes.
-
-	Logger.Printf("    STAGE 3")
-
-	broker1 := NewMockBroker(t, 101)
-
-	broker1.SetHandlerByMap(map[string]MockResponse{
-		"FetchRequest": NewMockFetchResponse(t, 1).
-			SetMessage("my_topic", 0, 124, testMsg),
-	})
-	broker0.SetHandlerByMap(map[string]MockResponse{
-		"MetadataRequest": NewMockMetadataResponse(t).
-			SetBroker(broker0.Addr(), broker0.BrokerID()).
-			SetBroker(broker1.Addr(), broker1.BrokerID()).
-			SetLeader("my_topic", 0, broker1.BrokerID()),
-	})
-
-	assertMessageOffset(t, <-pc.Messages(), 124)
-
-	safeClose(t, pc)
-	safeClose(t, c)
-	broker1.Close()
-	broker0.Close()
-}
-
-func TestConsumerInvalidTopic(t *testing.T) {
-	// Given
-	broker0 := NewMockBroker(t, 100)
-	broker0.SetHandlerByMap(map[string]MockResponse{
-		"MetadataRequest": NewMockMetadataResponse(t).
-			SetBroker(broker0.Addr(), broker0.BrokerID()),
-	})
-
-	c, err := NewConsumer([]string{broker0.Addr()}, nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// When
-	pc, err := c.ConsumePartition("my_topic", 0, OffsetOldest)
-
-	// Then
-	if pc != nil || err != ErrUnknownTopicOrPartition {
-		t.Errorf("Should fail with, err=%v", err)
-	}
-
-	safeClose(t, c)
-	broker0.Close()
-}
-
-// Nothing bad happens if a partition consumer that has no leader assigned at
-// the moment is closed.
-func TestConsumerClosePartitionWithoutLeader(t *testing.T) {
-	// Given
-	broker0 := NewMockBroker(t, 100)
-	broker0.SetHandlerByMap(map[string]MockResponse{
-		"MetadataRequest": NewMockMetadataResponse(t).
-			SetBroker(broker0.Addr(), broker0.BrokerID()).
-			SetLeader("my_topic", 0, broker0.BrokerID()),
-		"OffsetRequest": NewMockOffsetResponse(t).
-			SetOffset("my_topic", 0, OffsetOldest, 123).
-			SetOffset("my_topic", 0, OffsetNewest, 1000),
-		"FetchRequest": NewMockFetchResponse(t, 1).
-			SetMessage("my_topic", 0, 123, testMsg),
-	})
-
-	config := NewConfig()
-	config.Net.ReadTimeout = 100 * time.Millisecond
-	config.Consumer.Retry.Backoff = 100 * time.Millisecond
-	config.Consumer.Return.Errors = true
-	config.Metadata.Retry.Max = 0
-	c, err := NewConsumer([]string{broker0.Addr()}, config)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	pc, err := c.ConsumePartition("my_topic", 0, OffsetOldest)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	assertMessageOffset(t, <-pc.Messages(), 123)
-
-	// broker0 says that it is no longer the leader for my_topic/0, but the
-	// requests to retrieve metadata fail with network timeout.
-	fetchResponse2 := &FetchResponse{}
-	fetchResponse2.AddError("my_topic", 0, ErrNotLeaderForPartition)
-
-	broker0.SetHandlerByMap(map[string]MockResponse{
-		"FetchRequest": NewMockWrapper(fetchResponse2),
-	})
-
-	// When
-	if consErr := <-pc.Errors(); consErr.Err != ErrOutOfBrokers {
-		t.Errorf("Unexpected error: %v", consErr.Err)
-	}
-
-	// Then: the partition consumer can be closed without any problem.
-	safeClose(t, pc)
-	safeClose(t, c)
-	broker0.Close()
-}
-
-// If the initial offset passed on partition consumer creation is out of the
-// actual offset range for the partition, then the partition consumer stops
-// immediately closing its output channels.
-func TestConsumerShutsDownOutOfRange(t *testing.T) {
-	// Given
-	broker0 := NewMockBroker(t, 0)
-	fetchResponse := new(FetchResponse)
-	fetchResponse.AddError("my_topic", 0, ErrOffsetOutOfRange)
-	broker0.SetHandlerByMap(map[string]MockResponse{
-		"MetadataRequest": NewMockMetadataResponse(t).
-			SetBroker(broker0.Addr(), broker0.BrokerID()).
-			SetLeader("my_topic", 0, broker0.BrokerID()),
-		"OffsetRequest": NewMockOffsetResponse(t).
-			SetOffset("my_topic", 0, OffsetNewest, 1234).
-			SetOffset("my_topic", 0, OffsetOldest, 7),
-		"FetchRequest": NewMockWrapper(fetchResponse),
-	})
-
-	master, err := NewConsumer([]string{broker0.Addr()}, nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// When
-	consumer, err := master.ConsumePartition("my_topic", 0, 101)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// Then: consumer should shut down closing its messages and errors channels.
-	if _, ok := <-consumer.Messages(); ok {
-		t.Error("Expected the consumer to shut down")
-	}
-	safeClose(t, consumer)
-
-	safeClose(t, master)
-	broker0.Close()
-}
-
-// If a fetch response contains messages with offsets that are smaller then
-// requested, then such messages are ignored.
-func TestConsumerExtraOffsets(t *testing.T) {
-	// Given
-	legacyFetchResponse := &FetchResponse{}
-	legacyFetchResponse.AddMessage("my_topic", 0, nil, testMsg, 1)
-	legacyFetchResponse.AddMessage("my_topic", 0, nil, testMsg, 2)
-	legacyFetchResponse.AddMessage("my_topic", 0, nil, testMsg, 3)
-	legacyFetchResponse.AddMessage("my_topic", 0, nil, testMsg, 4)
-	newFetchResponse := &FetchResponse{Version: 4}
-	newFetchResponse.AddRecord("my_topic", 0, nil, testMsg, 1)
-	newFetchResponse.AddRecord("my_topic", 0, nil, testMsg, 2)
-	newFetchResponse.AddRecord("my_topic", 0, nil, testMsg, 3)
-	newFetchResponse.AddRecord("my_topic", 0, nil, testMsg, 4)
-	newFetchResponse.SetLastStableOffset("my_topic", 0, 4)
-	for _, fetchResponse1 := range []*FetchResponse{legacyFetchResponse, newFetchResponse} {
-		var offsetResponseVersion int16
-		cfg := NewConfig()
-		if fetchResponse1.Version >= 4 {
-			cfg.Version = V0_11_0_0
-			offsetResponseVersion = 1
-		}
-
-		broker0 := NewMockBroker(t, 0)
-		fetchResponse2 := &FetchResponse{}
-		fetchResponse2.Version = fetchResponse1.Version
-		fetchResponse2.AddError("my_topic", 0, ErrNoError)
-		broker0.SetHandlerByMap(map[string]MockResponse{
-			"MetadataRequest": NewMockMetadataResponse(t).
-				SetBroker(broker0.Addr(), broker0.BrokerID()).
-				SetLeader("my_topic", 0, broker0.BrokerID()),
-			"OffsetRequest": NewMockOffsetResponse(t).
-				SetVersion(offsetResponseVersion).
-				SetOffset("my_topic", 0, OffsetNewest, 1234).
-				SetOffset("my_topic", 0, OffsetOldest, 0),
-			"FetchRequest": NewMockSequence(fetchResponse1, fetchResponse2),
-		})
-
-		master, err := NewConsumer([]string{broker0.Addr()}, cfg)
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		// When
-		consumer, err := master.ConsumePartition("my_topic", 0, 3)
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		// Then: messages with offsets 1 and 2 are not returned even though they
-		// are present in the response.
-		assertMessageOffset(t, <-consumer.Messages(), 3)
-		assertMessageOffset(t, <-consumer.Messages(), 4)
-
-		safeClose(t, consumer)
-		safeClose(t, master)
-		broker0.Close()
-	}
-}
-
-func TestConsumeMessageWithNewerFetchAPIVersion(t *testing.T) {
-	// Given
-	fetchResponse1 := &FetchResponse{Version: 4}
-	fetchResponse1.AddMessage("my_topic", 0, nil, testMsg, 1)
-	fetchResponse1.AddMessage("my_topic", 0, nil, testMsg, 2)
-
-	cfg := NewConfig()
-	cfg.Version = V0_11_0_0
-
-	broker0 := NewMockBroker(t, 0)
-	fetchResponse2 := &FetchResponse{}
-	fetchResponse2.Version = 4
-	fetchResponse2.AddError("my_topic", 0, ErrNoError)
-	broker0.SetHandlerByMap(map[string]MockResponse{
-		"MetadataRequest": NewMockMetadataResponse(t).
-			SetBroker(broker0.Addr(), broker0.BrokerID()).
-			SetLeader("my_topic", 0, broker0.BrokerID()),
-		"OffsetRequest": NewMockOffsetResponse(t).
-			SetVersion(1).
-			SetOffset("my_topic", 0, OffsetNewest, 1234).
-			SetOffset("my_topic", 0, OffsetOldest, 0),
-		"FetchRequest": NewMockSequence(fetchResponse1, fetchResponse2),
-	})
-
-	master, err := NewConsumer([]string{broker0.Addr()}, cfg)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// When
-	consumer, err := master.ConsumePartition("my_topic", 0, 1)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	assertMessageOffset(t, <-consumer.Messages(), 1)
-	assertMessageOffset(t, <-consumer.Messages(), 2)
-
-	safeClose(t, consumer)
-	safeClose(t, master)
-	broker0.Close()
-}
-
-// It is fine if offsets of fetched messages are not sequential (although
-// strictly increasing!).
-func TestConsumerNonSequentialOffsets(t *testing.T) {
-	// Given
-	legacyFetchResponse := &FetchResponse{}
-	legacyFetchResponse.AddMessage("my_topic", 0, nil, testMsg, 5)
-	legacyFetchResponse.AddMessage("my_topic", 0, nil, testMsg, 7)
-	legacyFetchResponse.AddMessage("my_topic", 0, nil, testMsg, 11)
-	newFetchResponse := &FetchResponse{Version: 4}
-	newFetchResponse.AddRecord("my_topic", 0, nil, testMsg, 5)
-	newFetchResponse.AddRecord("my_topic", 0, nil, testMsg, 7)
-	newFetchResponse.AddRecord("my_topic", 0, nil, testMsg, 11)
-	newFetchResponse.SetLastStableOffset("my_topic", 0, 11)
-	for _, fetchResponse1 := range []*FetchResponse{legacyFetchResponse, newFetchResponse} {
-		var offsetResponseVersion int16
-		cfg := NewConfig()
-		if fetchResponse1.Version >= 4 {
-			cfg.Version = V0_11_0_0
-			offsetResponseVersion = 1
-		}
-
-		broker0 := NewMockBroker(t, 0)
-		fetchResponse2 := &FetchResponse{Version: fetchResponse1.Version}
-		fetchResponse2.AddError("my_topic", 0, ErrNoError)
-		broker0.SetHandlerByMap(map[string]MockResponse{
-			"MetadataRequest": NewMockMetadataResponse(t).
-				SetBroker(broker0.Addr(), broker0.BrokerID()).
-				SetLeader("my_topic", 0, broker0.BrokerID()),
-			"OffsetRequest": NewMockOffsetResponse(t).
-				SetVersion(offsetResponseVersion).
-				SetOffset("my_topic", 0, OffsetNewest, 1234).
-				SetOffset("my_topic", 0, OffsetOldest, 0),
-			"FetchRequest": NewMockSequence(fetchResponse1, fetchResponse2),
-		})
-
-		master, err := NewConsumer([]string{broker0.Addr()}, cfg)
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		// When
-		consumer, err := master.ConsumePartition("my_topic", 0, 3)
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		// Then: messages with offsets 1 and 2 are not returned even though they
-		// are present in the response.
-		assertMessageOffset(t, <-consumer.Messages(), 5)
-		assertMessageOffset(t, <-consumer.Messages(), 7)
-		assertMessageOffset(t, <-consumer.Messages(), 11)
-
-		safeClose(t, consumer)
-		safeClose(t, master)
-		broker0.Close()
-	}
-}
-
-// If leadership for a partition is changing then consumer resolves the new
-// leader and switches to it.
-func TestConsumerRebalancingMultiplePartitions(t *testing.T) {
-	// initial setup
-	seedBroker := NewMockBroker(t, 10)
-	leader0 := NewMockBroker(t, 0)
-	leader1 := NewMockBroker(t, 1)
-
-	seedBroker.SetHandlerByMap(map[string]MockResponse{
-		"MetadataRequest": NewMockMetadataResponse(t).
-			SetBroker(leader0.Addr(), leader0.BrokerID()).
-			SetBroker(leader1.Addr(), leader1.BrokerID()).
-			SetLeader("my_topic", 0, leader0.BrokerID()).
-			SetLeader("my_topic", 1, leader1.BrokerID()),
-	})
-
-	mockOffsetResponse1 := NewMockOffsetResponse(t).
-		SetOffset("my_topic", 0, OffsetOldest, 0).
-		SetOffset("my_topic", 0, OffsetNewest, 1000).
-		SetOffset("my_topic", 1, OffsetOldest, 0).
-		SetOffset("my_topic", 1, OffsetNewest, 1000)
-	leader0.SetHandlerByMap(map[string]MockResponse{
-		"OffsetRequest": mockOffsetResponse1,
-		"FetchRequest":  NewMockFetchResponse(t, 1),
-	})
-	leader1.SetHandlerByMap(map[string]MockResponse{
-		"OffsetRequest": mockOffsetResponse1,
-		"FetchRequest":  NewMockFetchResponse(t, 1),
-	})
-
-	// launch test goroutines
-	config := NewConfig()
-	config.Consumer.Retry.Backoff = 50
-	master, err := NewConsumer([]string{seedBroker.Addr()}, config)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// we expect to end up (eventually) consuming exactly ten messages on each partition
-	var wg sync.WaitGroup
-	for i := int32(0); i < 2; i++ {
-		consumer, err := master.ConsumePartition("my_topic", i, 0)
-		if err != nil {
-			t.Error(err)
-		}
-
-		go func(c PartitionConsumer) {
-			for err := range c.Errors() {
-				t.Error(err)
-			}
-		}(consumer)
-
-		wg.Add(1)
-		go func(partition int32, c PartitionConsumer) {
-			for i := 0; i < 10; i++ {
-				message := <-consumer.Messages()
-				if message.Offset != int64(i) {
-					t.Error("Incorrect message offset!", i, partition, message.Offset)
-				}
-				if message.Partition != partition {
-					t.Error("Incorrect message partition!")
-				}
-			}
-			safeClose(t, consumer)
-			wg.Done()
-		}(i, consumer)
-	}
-
-	time.Sleep(50 * time.Millisecond)
-	Logger.Printf("    STAGE 1")
-	// Stage 1:
-	//   * my_topic/0 -> leader0 serves 4 messages
-	//   * my_topic/1 -> leader1 serves 0 messages
-
-	mockFetchResponse := NewMockFetchResponse(t, 1)
-	for i := 0; i < 4; i++ {
-		mockFetchResponse.SetMessage("my_topic", 0, int64(i), testMsg)
-	}
-	leader0.SetHandlerByMap(map[string]MockResponse{
-		"FetchRequest": mockFetchResponse,
-	})
-
-	time.Sleep(50 * time.Millisecond)
-	Logger.Printf("    STAGE 2")
-	// Stage 2:
-	//   * leader0 says that it is no longer serving my_topic/0
-	//   * seedBroker tells that leader1 is serving my_topic/0 now
-
-	// seed broker tells that the new partition 0 leader is leader1
-	seedBroker.SetHandlerByMap(map[string]MockResponse{
-		"MetadataRequest": NewMockMetadataResponse(t).
-			SetLeader("my_topic", 0, leader1.BrokerID()).
-			SetLeader("my_topic", 1, leader1.BrokerID()),
-	})
-
-	// leader0 says no longer leader of partition 0
-	fetchResponse := new(FetchResponse)
-	fetchResponse.AddError("my_topic", 0, ErrNotLeaderForPartition)
-	leader0.SetHandlerByMap(map[string]MockResponse{
-		"FetchRequest": NewMockWrapper(fetchResponse),
-	})
-
-	time.Sleep(50 * time.Millisecond)
-	Logger.Printf("    STAGE 3")
-	// Stage 3:
-	//   * my_topic/0 -> leader1 serves 3 messages
-	//   * my_topic/1 -> leader1 server 8 messages
-
-	// leader1 provides 3 message on partition 0, and 8 messages on partition 1
-	mockFetchResponse2 := NewMockFetchResponse(t, 2)
-	for i := 4; i < 7; i++ {
-		mockFetchResponse2.SetMessage("my_topic", 0, int64(i), testMsg)
-	}
-	for i := 0; i < 8; i++ {
-		mockFetchResponse2.SetMessage("my_topic", 1, int64(i), testMsg)
-	}
-	leader1.SetHandlerByMap(map[string]MockResponse{
-		"FetchRequest": mockFetchResponse2,
-	})
-
-	time.Sleep(50 * time.Millisecond)
-	Logger.Printf("    STAGE 4")
-	// Stage 4:
-	//   * my_topic/0 -> leader1 serves 3 messages
-	//   * my_topic/1 -> leader1 tells that it is no longer the leader
-	//   * seedBroker tells that leader0 is a new leader for my_topic/1
-
-	// metadata assigns 0 to leader1 and 1 to leader0
-	seedBroker.SetHandlerByMap(map[string]MockResponse{
-		"MetadataRequest": NewMockMetadataResponse(t).
-			SetLeader("my_topic", 0, leader1.BrokerID()).
-			SetLeader("my_topic", 1, leader0.BrokerID()),
-	})
-
-	// leader1 provides three more messages on partition0, says no longer leader of partition1
-	mockFetchResponse3 := NewMockFetchResponse(t, 3).
-		SetMessage("my_topic", 0, int64(7), testMsg).
-		SetMessage("my_topic", 0, int64(8), testMsg).
-		SetMessage("my_topic", 0, int64(9), testMsg)
-	fetchResponse4 := new(FetchResponse)
-	fetchResponse4.AddError("my_topic", 1, ErrNotLeaderForPartition)
-	leader1.SetHandlerByMap(map[string]MockResponse{
-		"FetchRequest": NewMockSequence(mockFetchResponse3, fetchResponse4),
-	})
-
-	// leader0 provides two messages on partition 1
-	mockFetchResponse4 := NewMockFetchResponse(t, 2)
-	for i := 8; i < 10; i++ {
-		mockFetchResponse4.SetMessage("my_topic", 1, int64(i), testMsg)
-	}
-	leader0.SetHandlerByMap(map[string]MockResponse{
-		"FetchRequest": mockFetchResponse4,
-	})
-
-	wg.Wait()
-	safeClose(t, master)
-	leader1.Close()
-	leader0.Close()
-	seedBroker.Close()
-}
-
-// When two partitions have the same broker as the leader, if one partition
-// consumer channel buffer is full then that does not affect the ability to
-// read messages by the other consumer.
-func TestConsumerInterleavedClose(t *testing.T) {
-	// Given
-	broker0 := NewMockBroker(t, 0)
-	broker0.SetHandlerByMap(map[string]MockResponse{
-		"MetadataRequest": NewMockMetadataResponse(t).
-			SetBroker(broker0.Addr(), broker0.BrokerID()).
-			SetLeader("my_topic", 0, broker0.BrokerID()).
-			SetLeader("my_topic", 1, broker0.BrokerID()),
-		"OffsetRequest": NewMockOffsetResponse(t).
-			SetOffset("my_topic", 0, OffsetOldest, 1000).
-			SetOffset("my_topic", 0, OffsetNewest, 1100).
-			SetOffset("my_topic", 1, OffsetOldest, 2000).
-			SetOffset("my_topic", 1, OffsetNewest, 2100),
-		"FetchRequest": NewMockFetchResponse(t, 1).
-			SetMessage("my_topic", 0, 1000, testMsg).
-			SetMessage("my_topic", 0, 1001, testMsg).
-			SetMessage("my_topic", 0, 1002, testMsg).
-			SetMessage("my_topic", 1, 2000, testMsg),
-	})
-
-	config := NewConfig()
-	config.ChannelBufferSize = 0
-	master, err := NewConsumer([]string{broker0.Addr()}, config)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	c0, err := master.ConsumePartition("my_topic", 0, 1000)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	c1, err := master.ConsumePartition("my_topic", 1, 2000)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// When/Then: we can read from partition 0 even if nobody reads from partition 1
-	assertMessageOffset(t, <-c0.Messages(), 1000)
-	assertMessageOffset(t, <-c0.Messages(), 1001)
-	assertMessageOffset(t, <-c0.Messages(), 1002)
-
-	safeClose(t, c1)
-	safeClose(t, c0)
-	safeClose(t, master)
-	broker0.Close()
-}
-
-func TestConsumerBounceWithReferenceOpen(t *testing.T) {
-	broker0 := NewMockBroker(t, 0)
-	broker0Addr := broker0.Addr()
-	broker1 := NewMockBroker(t, 1)
-
-	mockMetadataResponse := NewMockMetadataResponse(t).
-		SetBroker(broker0.Addr(), broker0.BrokerID()).
-		SetBroker(broker1.Addr(), broker1.BrokerID()).
-		SetLeader("my_topic", 0, broker0.BrokerID()).
-		SetLeader("my_topic", 1, broker1.BrokerID())
-
-	mockOffsetResponse := NewMockOffsetResponse(t).
-		SetOffset("my_topic", 0, OffsetOldest, 1000).
-		SetOffset("my_topic", 0, OffsetNewest, 1100).
-		SetOffset("my_topic", 1, OffsetOldest, 2000).
-		SetOffset("my_topic", 1, OffsetNewest, 2100)
-
-	mockFetchResponse := NewMockFetchResponse(t, 1)
-	for i := 0; i < 10; i++ {
-		mockFetchResponse.SetMessage("my_topic", 0, int64(1000+i), testMsg)
-		mockFetchResponse.SetMessage("my_topic", 1, int64(2000+i), testMsg)
-	}
-
-	broker0.SetHandlerByMap(map[string]MockResponse{
-		"OffsetRequest": mockOffsetResponse,
-		"FetchRequest":  mockFetchResponse,
-	})
-	broker1.SetHandlerByMap(map[string]MockResponse{
-		"MetadataRequest": mockMetadataResponse,
-		"OffsetRequest":   mockOffsetResponse,
-		"FetchRequest":    mockFetchResponse,
-	})
-
-	config := NewConfig()
-	config.Consumer.Return.Errors = true
-	config.Consumer.Retry.Backoff = 100 * time.Millisecond
-	config.ChannelBufferSize = 1
-	master, err := NewConsumer([]string{broker1.Addr()}, config)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	c0, err := master.ConsumePartition("my_topic", 0, 1000)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	c1, err := master.ConsumePartition("my_topic", 1, 2000)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// read messages from both partition to make sure that both brokers operate
-	// normally.
-	assertMessageOffset(t, <-c0.Messages(), 1000)
-	assertMessageOffset(t, <-c1.Messages(), 2000)
-
-	// Simulate broker shutdown. Note that metadata response does not change,
-	// that is the leadership does not move to another broker. So partition
-	// consumer will keep retrying to restore the connection with the broker.
-	broker0.Close()
-
-	// Make sure that while the partition/0 leader is down, consumer/partition/1
-	// is capable of pulling messages from broker1.
-	for i := 1; i < 7; i++ {
-		offset := (<-c1.Messages()).Offset
-		if offset != int64(2000+i) {
-			t.Errorf("Expected offset %d from consumer/partition/1", int64(2000+i))
-		}
-	}
-
-	// Bring broker0 back to service.
-	broker0 = NewMockBrokerAddr(t, 0, broker0Addr)
-	broker0.SetHandlerByMap(map[string]MockResponse{
-		"FetchRequest": mockFetchResponse,
-	})
-
-	// Read the rest of messages from both partitions.
-	for i := 7; i < 10; i++ {
-		assertMessageOffset(t, <-c1.Messages(), int64(2000+i))
-	}
-	for i := 1; i < 10; i++ {
-		assertMessageOffset(t, <-c0.Messages(), int64(1000+i))
-	}
-
-	select {
-	case <-c0.Errors():
-	default:
-		t.Errorf("Partition consumer should have detected broker restart")
-	}
-
-	safeClose(t, c1)
-	safeClose(t, c0)
-	safeClose(t, master)
-	broker0.Close()
-	broker1.Close()
-}
-
-func TestConsumerOffsetOutOfRange(t *testing.T) {
-	// Given
-	broker0 := NewMockBroker(t, 2)
-	broker0.SetHandlerByMap(map[string]MockResponse{
-		"MetadataRequest": NewMockMetadataResponse(t).
-			SetBroker(broker0.Addr(), broker0.BrokerID()).
-			SetLeader("my_topic", 0, broker0.BrokerID()),
-		"OffsetRequest": NewMockOffsetResponse(t).
-			SetOffset("my_topic", 0, OffsetNewest, 1234).
-			SetOffset("my_topic", 0, OffsetOldest, 2345),
-	})
-
-	master, err := NewConsumer([]string{broker0.Addr()}, nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// When/Then
-	if _, err := master.ConsumePartition("my_topic", 0, 0); err != ErrOffsetOutOfRange {
-		t.Fatal("Should return ErrOffsetOutOfRange, got:", err)
-	}
-	if _, err := master.ConsumePartition("my_topic", 0, 3456); err != ErrOffsetOutOfRange {
-		t.Fatal("Should return ErrOffsetOutOfRange, got:", err)
-	}
-	if _, err := master.ConsumePartition("my_topic", 0, -3); err != ErrOffsetOutOfRange {
-		t.Fatal("Should return ErrOffsetOutOfRange, got:", err)
-	}
-
-	safeClose(t, master)
-	broker0.Close()
-}
-
-func TestConsumerExpiryTicker(t *testing.T) {
-	// Given
-	broker0 := NewMockBroker(t, 0)
-	fetchResponse1 := &FetchResponse{}
-	for i := 1; i <= 8; i++ {
-		fetchResponse1.AddMessage("my_topic", 0, nil, testMsg, int64(i))
-	}
-	broker0.SetHandlerByMap(map[string]MockResponse{
-		"MetadataRequest": NewMockMetadataResponse(t).
-			SetBroker(broker0.Addr(), broker0.BrokerID()).
-			SetLeader("my_topic", 0, broker0.BrokerID()),
-		"OffsetRequest": NewMockOffsetResponse(t).
-			SetOffset("my_topic", 0, OffsetNewest, 1234).
-			SetOffset("my_topic", 0, OffsetOldest, 1),
-		"FetchRequest": NewMockSequence(fetchResponse1),
-	})
-
-	config := NewConfig()
-	config.ChannelBufferSize = 0
-	config.Consumer.MaxProcessingTime = 10 * time.Millisecond
-	master, err := NewConsumer([]string{broker0.Addr()}, config)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// When
-	consumer, err := master.ConsumePartition("my_topic", 0, 1)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// Then: messages with offsets 1 through 8 are read
-	for i := 1; i <= 8; i++ {
-		assertMessageOffset(t, <-consumer.Messages(), int64(i))
-		time.Sleep(2 * time.Millisecond)
-	}
-
-	safeClose(t, consumer)
-	safeClose(t, master)
-	broker0.Close()
-}
-
-func assertMessageOffset(t *testing.T, msg *ConsumerMessage, expectedOffset int64) {
-	if msg.Offset != expectedOffset {
-		t.Errorf("Incorrect message offset: expected=%d, actual=%d", expectedOffset, msg.Offset)
-	}
-}
-
-// This example shows how to use the consumer to read messages
-// from a single partition.
-func ExampleConsumer() {
-	consumer, err := NewConsumer([]string{"localhost:9092"}, nil)
-	if err != nil {
-		panic(err)
-	}
-
-	defer func() {
-		if err := consumer.Close(); err != nil {
-			log.Fatalln(err)
-		}
-	}()
-
-	partitionConsumer, err := consumer.ConsumePartition("my_topic", 0, OffsetNewest)
-	if err != nil {
-		panic(err)
-	}
-
-	defer func() {
-		if err := partitionConsumer.Close(); err != nil {
-			log.Fatalln(err)
-		}
-	}()
-
-	// Trap SIGINT to trigger a shutdown.
-	signals := make(chan os.Signal, 1)
-	signal.Notify(signals, os.Interrupt)
-
-	consumed := 0
-ConsumerLoop:
-	for {
-		select {
-		case msg := <-partitionConsumer.Messages():
-			log.Printf("Consumed message offset %d\n", msg.Offset)
-			consumed++
-		case <-signals:
-			break ConsumerLoop
-		}
-	}
-
-	log.Printf("Consumed: %d\n", consumed)
-}

vendor/github.com/Shopify/sarama/crc32_field.go

@@ -1,69 +0,0 @@
-package sarama
-
-import (
-	"encoding/binary"
-	"fmt"
-	"hash/crc32"
-)
-
-type crcPolynomial int8
-
-const (
-	crcIEEE crcPolynomial = iota
-	crcCastagnoli
-)
-
-var castagnoliTable = crc32.MakeTable(crc32.Castagnoli)
-
-// crc32Field implements the pushEncoder and pushDecoder interfaces for calculating CRC32s.
-type crc32Field struct {
-	startOffset int
-	polynomial  crcPolynomial
-}
-
-func (c *crc32Field) saveOffset(in int) {
-	c.startOffset = in
-}
-
-func (c *crc32Field) reserveLength() int {
-	return 4
-}
-
-func newCRC32Field(polynomial crcPolynomial) *crc32Field {
-	return &crc32Field{polynomial: polynomial}
-}
-
-func (c *crc32Field) run(curOffset int, buf []byte) error {
-	crc, err := c.crc(curOffset, buf)
-	if err != nil {
-		return err
-	}
-	binary.BigEndian.PutUint32(buf[c.startOffset:], crc)
-	return nil
-}
-
-func (c *crc32Field) check(curOffset int, buf []byte) error {
-	crc, err := c.crc(curOffset, buf)
-	if err != nil {
-		return err
-	}
-
-	expected := binary.BigEndian.Uint32(buf[c.startOffset:])
-	if crc != expected {
-		return PacketDecodingError{fmt.Sprintf("CRC didn't match expected %#x got %#x", expected, crc)}
-	}
-
-	return nil
-}
-func (c *crc32Field) crc(curOffset int, buf []byte) (uint32, error) {
-	var tab *crc32.Table
-	switch c.polynomial {
-	case crcIEEE:
-		tab = crc32.IEEETable
-	case crcCastagnoli:
-		tab = castagnoliTable
-	default:
-		return 0, PacketDecodingError{"invalid CRC type"}
-	}
-	return crc32.Checksum(buf[c.startOffset+4:curOffset], tab), nil
-}

vendor/github.com/Shopify/sarama/create_partitions_request.go

@@ -1,121 +0,0 @@
-package sarama
-
-import "time"
-
-type CreatePartitionsRequest struct {
-	TopicPartitions map[string]*TopicPartition
-	Timeout         time.Duration
-	ValidateOnly    bool
-}
-
-func (c *CreatePartitionsRequest) encode(pe packetEncoder) error {
-	if err := pe.putArrayLength(len(c.TopicPartitions)); err != nil {
-		return err
-	}
-
-	for topic, partition := range c.TopicPartitions {
-		if err := pe.putString(topic); err != nil {
-			return err
-		}
-		if err := partition.encode(pe); err != nil {
-			return err
-		}
-	}
-
-	pe.putInt32(int32(c.Timeout / time.Millisecond))
-
-	pe.putBool(c.ValidateOnly)
-
-	return nil
-}
-
-func (c *CreatePartitionsRequest) decode(pd packetDecoder, version int16) (err error) {
-	n, err := pd.getArrayLength()
-	if err != nil {
-		return err
-	}
-	c.TopicPartitions = make(map[string]*TopicPartition, n)
-	for i := 0; i < n; i++ {
-		topic, err := pd.getString()
-		if err != nil {
-			return err
-		}
-		c.TopicPartitions[topic] = new(TopicPartition)
-		if err := c.TopicPartitions[topic].decode(pd, version); err != nil {
-			return err
-		}
-	}
-
-	timeout, err := pd.getInt32()
-	if err != nil {
-		return err
-	}
-	c.Timeout = time.Duration(timeout) * time.Millisecond
-
-	if c.ValidateOnly, err = pd.getBool(); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func (r *CreatePartitionsRequest) key() int16 {
-	return 37
-}
-
-func (r *CreatePartitionsRequest) version() int16 {
-	return 0
-}
-
-func (r *CreatePartitionsRequest) requiredVersion() KafkaVersion {
-	return V1_0_0_0
-}
-
-type TopicPartition struct {
-	Count      int32
-	Assignment [][]int32
-}
-
-func (t *TopicPartition) encode(pe packetEncoder) error {
-	pe.putInt32(t.Count)
-
-	if len(t.Assignment) == 0 {
-		pe.putInt32(-1)
-		return nil
-	}
-
-	if err := pe.putArrayLength(len(t.Assignment)); err != nil {
-		return err
-	}
-
-	for _, assign := range t.Assignment {
-		if err := pe.putInt32Array(assign); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (t *TopicPartition) decode(pd packetDecoder, version int16) (err error) {
-	if t.Count, err = pd.getInt32(); err != nil {
-		return err
-	}
-
-	n, err := pd.getInt32()
-	if err != nil {
-		return err
-	}
-	if n <= 0 {
-		return nil
-	}
-	t.Assignment = make([][]int32, n)
-
-	for i := 0; i < int(n); i++ {
-		if t.Assignment[i], err = pd.getInt32Array(); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}

vendor/github.com/Shopify/sarama/create_partitions_request_test.go

@@ -1,50 +0,0 @@
-package sarama
-
-import (
-	"testing"
-	"time"
-)
-
-var (
-	createPartitionRequestNoAssignment = []byte{
-		0, 0, 0, 1, // one topic
-		0, 5, 't', 'o', 'p', 'i', 'c',
-		0, 0, 0, 3, // 3 partitions
-		255, 255, 255, 255, // no assignments
-		0, 0, 0, 100, // timeout
-		0, // validate only = false
-	}
-
-	createPartitionRequestAssignment = []byte{
-		0, 0, 0, 1,
-		0, 5, 't', 'o', 'p', 'i', 'c',
-		0, 0, 0, 3, // 3 partitions
-		0, 0, 0, 2,
-		0, 0, 0, 2,
-		0, 0, 0, 2, 0, 0, 0, 3,
-		0, 0, 0, 2,
-		0, 0, 0, 3, 0, 0, 0, 1,
-		0, 0, 0, 100,
-		1, // validate only = true
-	}
-)
-
-func TestCreatePartitionsRequest(t *testing.T) {
-	req := &CreatePartitionsRequest{
-		TopicPartitions: map[string]*TopicPartition{
-			"topic": &TopicPartition{
-				Count: 3,
-			},
-		},
-		Timeout: 100 * time.Millisecond,
-	}
-
-	buf := testRequestEncode(t, "no assignment", req, createPartitionRequestNoAssignment)
-	testRequestDecode(t, "no assignment", req, buf)
-
-	req.ValidateOnly = true
-	req.TopicPartitions["topic"].Assignment = [][]int32{{2, 3}, {3, 1}}
-
-	buf = testRequestEncode(t, "assignment", req, createPartitionRequestAssignment)
-	testRequestDecode(t, "assignment", req, buf)
-}

vendor/github.com/Shopify/sarama/create_partitions_response.go

@@ -1,94 +0,0 @@
-package sarama
-
-import "time"
-
-type CreatePartitionsResponse struct {
-	ThrottleTime         time.Duration
-	TopicPartitionErrors map[string]*TopicPartitionError
-}
-
-func (c *CreatePartitionsResponse) encode(pe packetEncoder) error {
-	pe.putInt32(int32(c.ThrottleTime / time.Millisecond))
-	if err := pe.putArrayLength(len(c.TopicPartitionErrors)); err != nil {
-		return err
-	}
-
-	for topic, partitionError := range c.TopicPartitionErrors {
-		if err := pe.putString(topic); err != nil {
-			return err
-		}
-		if err := partitionError.encode(pe); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (c *CreatePartitionsResponse) decode(pd packetDecoder, version int16) (err error) {
-	throttleTime, err := pd.getInt32()
-	if err != nil {
-		return err
-	}
-	c.ThrottleTime = time.Duration(throttleTime) * time.Millisecond
-
-	n, err := pd.getArrayLength()
-	if err != nil {
-		return err
-	}
-
-	c.TopicPartitionErrors = make(map[string]*TopicPartitionError, n)
-	for i := 0; i < n; i++ {
-		topic, err := pd.getString()
-		if err != nil {
-			return err
-		}
-		c.TopicPartitionErrors[topic] = new(TopicPartitionError)
-		if err := c.TopicPartitionErrors[topic].decode(pd, version); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (r *CreatePartitionsResponse) key() int16 {
-	return 37
-}
-
-func (r *CreatePartitionsResponse) version() int16 {
-	return 0
-}
-
-func (r *CreatePartitionsResponse) requiredVersion() KafkaVersion {
-	return V1_0_0_0
-}
-
-type TopicPartitionError struct {
-	Err    KError
-	ErrMsg *string
-}
-
-func (t *TopicPartitionError) encode(pe packetEncoder) error {
-	pe.putInt16(int16(t.Err))
-
-	if err := pe.putNullableString(t.ErrMsg); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func (t *TopicPartitionError) decode(pd packetDecoder, version int16) (err error) {
-	kerr, err := pd.getInt16()
-	if err != nil {
-		return err
-	}
-	t.Err = KError(kerr)
-
-	if t.ErrMsg, err = pd.getNullableString(); err != nil {
-		return err
-	}
-
-	return nil
-}

vendor/github.com/Shopify/sarama/create_partitions_response_test.go

@@ -1,52 +0,0 @@
-package sarama
-
-import (
-	"reflect"
-	"testing"
-	"time"
-)
-
-var (
-	createPartitionResponseSuccess = []byte{
-		0, 0, 0, 100, // throttleTimeMs
-		0, 0, 0, 1,
-		0, 5, 't', 'o', 'p', 'i', 'c',
-		0, 0, // no error
-		255, 255, // no error message
-	}
-
-	createPartitionResponseFail = []byte{
-		0, 0, 0, 100, // throttleTimeMs
-		0, 0, 0, 1,
-		0, 5, 't', 'o', 'p', 'i', 'c',
-		0, 37, // partition error
-		0, 5, 'e', 'r', 'r', 'o', 'r',
-	}
-)
-
-func TestCreatePartitionsResponse(t *testing.T) {
-	resp := &CreatePartitionsResponse{
-		ThrottleTime: 100 * time.Millisecond,
-		TopicPartitionErrors: map[string]*TopicPartitionError{
-			"topic": &TopicPartitionError{},
-		},
-	}
-
-	testResponse(t, "success", resp, createPartitionResponseSuccess)
-	decodedresp := new(CreatePartitionsResponse)
-	testVersionDecodable(t, "success", decodedresp, createPartitionResponseSuccess, 0)
-	if !reflect.DeepEqual(decodedresp, resp) {
-		t.Errorf("Decoding error: expected %v but got %v", decodedresp, resp)
-	}
-
-	errMsg := "error"
-	resp.TopicPartitionErrors["topic"].Err = ErrInvalidPartitions
-	resp.TopicPartitionErrors["topic"].ErrMsg = &errMsg
-
-	testResponse(t, "with errors", resp, createPartitionResponseFail)
-	decodedresp = new(CreatePartitionsResponse)
-	testVersionDecodable(t, "with errors", decodedresp, createPartitionResponseFail, 0)
-	if !reflect.DeepEqual(decodedresp, resp) {
-		t.Errorf("Decoding error: expected %v but got %v", decodedresp, resp)
-	}
-}

vendor/github.com/Shopify/sarama/describe_groups_request.go

@@ -1,30 +0,0 @@
-package sarama
-
-type DescribeGroupsRequest struct {
-	Groups []string
-}
-
-func (r *DescribeGroupsRequest) encode(pe packetEncoder) error {
-	return pe.putStringArray(r.Groups)
-}
-
-func (r *DescribeGroupsRequest) decode(pd packetDecoder, version int16) (err error) {
-	r.Groups, err = pd.getStringArray()
-	return
-}
-
-func (r *DescribeGroupsRequest) key() int16 {
-	return 15
-}
-
-func (r *DescribeGroupsRequest) version() int16 {
-	return 0
-}
-
-func (r *DescribeGroupsRequest) requiredVersion() KafkaVersion {
-	return V0_9_0_0
-}
-
-func (r *DescribeGroupsRequest) AddGroup(group string) {
-	r.Groups = append(r.Groups, group)
-}

vendor/github.com/Shopify/sarama/describe_groups_request_test.go

@@ -1,34 +0,0 @@
-package sarama
-
-import "testing"
-
-var (
-	emptyDescribeGroupsRequest = []byte{0, 0, 0, 0}
-
-	singleDescribeGroupsRequest = []byte{
-		0, 0, 0, 1, // 1 group
-		0, 3, 'f', 'o', 'o', // group name: foo
-	}
-
-	doubleDescribeGroupsRequest = []byte{
-		0, 0, 0, 2, // 2 groups
-		0, 3, 'f', 'o', 'o', // group name: foo
-		0, 3, 'b', 'a', 'r', // group name: foo
-	}
-)
-
-func TestDescribeGroupsRequest(t *testing.T) {
-	var request *DescribeGroupsRequest
-
-	request = new(DescribeGroupsRequest)
-	testRequest(t, "no groups", request, emptyDescribeGroupsRequest)
-
-	request = new(DescribeGroupsRequest)
-	request.AddGroup("foo")
-	testRequest(t, "one group", request, singleDescribeGroupsRequest)
-
-	request = new(DescribeGroupsRequest)
-	request.AddGroup("foo")
-	request.AddGroup("bar")
-	testRequest(t, "two groups", request, doubleDescribeGroupsRequest)
-}

vendor/github.com/Shopify/sarama/describe_groups_response.go

@@ -1,187 +0,0 @@
-package sarama
-
-type DescribeGroupsResponse struct {
-	Groups []*GroupDescription
-}
-
-func (r *DescribeGroupsResponse) encode(pe packetEncoder) error {
-	if err := pe.putArrayLength(len(r.Groups)); err != nil {
-		return err
-	}
-
-	for _, groupDescription := range r.Groups {
-		if err := groupDescription.encode(pe); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (r *DescribeGroupsResponse) decode(pd packetDecoder, version int16) (err error) {
-	n, err := pd.getArrayLength()
-	if err != nil {
-		return err
-	}
-
-	r.Groups = make([]*GroupDescription, n)
-	for i := 0; i < n; i++ {
-		r.Groups[i] = new(GroupDescription)
-		if err := r.Groups[i].decode(pd); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (r *DescribeGroupsResponse) key() int16 {
-	return 15
-}
-
-func (r *DescribeGroupsResponse) version() int16 {
-	return 0
-}
-
-func (r *DescribeGroupsResponse) requiredVersion() KafkaVersion {
-	return V0_9_0_0
-}
-
-type GroupDescription struct {
-	Err          KError
-	GroupId      string
-	State        string
-	ProtocolType string
-	Protocol     string
-	Members      map[string]*GroupMemberDescription
-}
-
-func (gd *GroupDescription) encode(pe packetEncoder) error {
-	pe.putInt16(int16(gd.Err))
-
-	if err := pe.putString(gd.GroupId); err != nil {
-		return err
-	}
-	if err := pe.putString(gd.State); err != nil {
-		return err
-	}
-	if err := pe.putString(gd.ProtocolType); err != nil {
-		return err
-	}
-	if err := pe.putString(gd.Protocol); err != nil {
-		return err
-	}
-
-	if err := pe.putArrayLength(len(gd.Members)); err != nil {
-		return err
-	}
-
-	for memberId, groupMemberDescription := range gd.Members {
-		if err := pe.putString(memberId); err != nil {
-			return err
-		}
-		if err := groupMemberDescription.encode(pe); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (gd *GroupDescription) decode(pd packetDecoder) (err error) {
-	kerr, err := pd.getInt16()
-	if err != nil {
-		return err
-	}
-
-	gd.Err = KError(kerr)
-
-	if gd.GroupId, err = pd.getString(); err != nil {
-		return
-	}
-	if gd.State, err = pd.getString(); err != nil {
-		return
-	}
-	if gd.ProtocolType, err = pd.getString(); err != nil {
-		return
-	}
-	if gd.Protocol, err = pd.getString(); err != nil {
-		return
-	}
-
-	n, err := pd.getArrayLength()
-	if err != nil {
-		return err
-	}
-	if n == 0 {
-		return nil
-	}
-
-	gd.Members = make(map[string]*GroupMemberDescription)
-	for i := 0; i < n; i++ {
-		memberId, err := pd.getString()
-		if err != nil {
-			return err
-		}
-
-		gd.Members[memberId] = new(GroupMemberDescription)
-		if err := gd.Members[memberId].decode(pd); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-type GroupMemberDescription struct {
-	ClientId         string
-	ClientHost       string
-	MemberMetadata   []byte
-	MemberAssignment []byte
-}
-
-func (gmd *GroupMemberDescription) encode(pe packetEncoder) error {
-	if err := pe.putString(gmd.ClientId); err != nil {
-		return err
-	}
-	if err := pe.putString(gmd.ClientHost); err != nil {
-		return err
-	}
-	if err := pe.putBytes(gmd.MemberMetadata); err != nil {
-		return err
-	}
-	if err := pe.putBytes(gmd.MemberAssignment); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func (gmd *GroupMemberDescription) decode(pd packetDecoder) (err error) {
-	if gmd.ClientId, err = pd.getString(); err != nil {
-		return
-	}
-	if gmd.ClientHost, err = pd.getString(); err != nil {
-		return
-	}
-	if gmd.MemberMetadata, err = pd.getBytes(); err != nil {
-		return
-	}
-	if gmd.MemberAssignment, err = pd.getBytes(); err != nil {
-		return
-	}
-
-	return nil
-}
-
-func (gmd *GroupMemberDescription) GetMemberAssignment() (*ConsumerGroupMemberAssignment, error) {
-	assignment := new(ConsumerGroupMemberAssignment)
-	err := decode(gmd.MemberAssignment, assignment)
-	return assignment, err
-}
-
-func (gmd *GroupMemberDescription) GetMemberMetadata() (*ConsumerGroupMemberMetadata, error) {
-	metadata := new(ConsumerGroupMemberMetadata)
-	err := decode(gmd.MemberMetadata, metadata)
-	return metadata, err
-}

vendor/github.com/Shopify/sarama/describe_groups_response_test.go

@@ -1,91 +0,0 @@
-package sarama
-
-import (
-	"reflect"
-	"testing"
-)
-
-var (
-	describeGroupsResponseEmpty = []byte{
-		0, 0, 0, 0, // no groups
-	}
-
-	describeGroupsResponsePopulated = []byte{
-		0, 0, 0, 2, // 2 groups
-
-		0, 0, // no error
-		0, 3, 'f', 'o', 'o', // Group ID
-		0, 3, 'b', 'a', 'r', // State
-		0, 8, 'c', 'o', 'n', 's', 'u', 'm', 'e', 'r', // ConsumerProtocol type
-		0, 3, 'b', 'a', 'z', // Protocol name
-		0, 0, 0, 1, // 1 member
-		0, 2, 'i', 'd', // Member ID
-		0, 6, 's', 'a', 'r', 'a', 'm', 'a', // Client ID
-		0, 9, 'l', 'o', 'c', 'a', 'l', 'h', 'o', 's', 't', // Client Host
-		0, 0, 0, 3, 0x01, 0x02, 0x03, // MemberMetadata
-		0, 0, 0, 3, 0x04, 0x05, 0x06, // MemberAssignment
-
-		0, 30, // ErrGroupAuthorizationFailed
-		0, 0,
-		0, 0,
-		0, 0,
-		0, 0,
-		0, 0, 0, 0,
-	}
-)
-
-func TestDescribeGroupsResponse(t *testing.T) {
-	var response *DescribeGroupsResponse
-
-	response = new(DescribeGroupsResponse)
-	testVersionDecodable(t, "empty", response, describeGroupsResponseEmpty, 0)
-	if len(response.Groups) != 0 {
-		t.Error("Expected no groups")
-	}
-
-	response = new(DescribeGroupsResponse)
-	testVersionDecodable(t, "populated", response, describeGroupsResponsePopulated, 0)
-	if len(response.Groups) != 2 {
-		t.Error("Expected two groups")
-	}
-
-	group0 := response.Groups[0]
-	if group0.Err != ErrNoError {
-		t.Error("Unxpected groups[0].Err, found", group0.Err)
-	}
-	if group0.GroupId != "foo" {
-		t.Error("Unxpected groups[0].GroupId, found", group0.GroupId)
-	}
-	if group0.State != "bar" {
-		t.Error("Unxpected groups[0].State, found", group0.State)
-	}
-	if group0.ProtocolType != "consumer" {
-		t.Error("Unxpected groups[0].ProtocolType, found", group0.ProtocolType)
-	}
-	if group0.Protocol != "baz" {
-		t.Error("Unxpected groups[0].Protocol, found", group0.Protocol)
-	}
-	if len(group0.Members) != 1 {
-		t.Error("Unxpected groups[0].Members, found", group0.Members)
-	}
-	if group0.Members["id"].ClientId != "sarama" {
-		t.Error("Unxpected groups[0].Members[id].ClientId, found", group0.Members["id"].ClientId)
-	}
-	if group0.Members["id"].ClientHost != "localhost" {
-		t.Error("Unxpected groups[0].Members[id].ClientHost, found", group0.Members["id"].ClientHost)
-	}
-	if !reflect.DeepEqual(group0.Members["id"].MemberMetadata, []byte{0x01, 0x02, 0x03}) {
-		t.Error("Unxpected groups[0].Members[id].MemberMetadata, found", group0.Members["id"].MemberMetadata)
-	}
-	if !reflect.DeepEqual(group0.Members["id"].MemberAssignment, []byte{0x04, 0x05, 0x06}) {
-		t.Error("Unxpected groups[0].Members[id].MemberAssignment, found", group0.Members["id"].MemberAssignment)
-	}
-
-	group1 := response.Groups[1]
-	if group1.Err != ErrGroupAuthorizationFailed {
-		t.Error("Unxpected groups[1].Err, found", group0.Err)
-	}
-	if len(group1.Members) != 0 {
-		t.Error("Unxpected groups[1].Members, found", group0.Members)
-	}
-}

vendor/github.com/Shopify/sarama/dev.yml

@@ -1,10 +0,0 @@
-name: sarama
-
-up:
-  - go:
-      version: '1.9'
-
-commands:
-  test:
-    run: make test
-    desc: 'run unit tests'

vendor/github.com/Shopify/sarama/encoder_decoder.go

@@ -1,89 +0,0 @@
-package sarama
-
-import (
-	"fmt"
-
-	"github.com/rcrowley/go-metrics"
-)
-
-// Encoder is the interface that wraps the basic Encode method.
-// Anything implementing Encoder can be turned into bytes using Kafka's encoding rules.
-type encoder interface {
-	encode(pe packetEncoder) error
-}
-
-// Encode takes an Encoder and turns it into bytes while potentially recording metrics.
-func encode(e encoder, metricRegistry metrics.Registry) ([]byte, error) {
-	if e == nil {
-		return nil, nil
-	}
-
-	var prepEnc prepEncoder
-	var realEnc realEncoder
-
-	err := e.encode(&prepEnc)
-	if err != nil {
-		return nil, err
-	}
-
-	if prepEnc.length < 0 || prepEnc.length > int(MaxRequestSize) {
-		return nil, PacketEncodingError{fmt.Sprintf("invalid request size (%d)", prepEnc.length)}
-	}
-
-	realEnc.raw = make([]byte, prepEnc.length)
-	realEnc.registry = metricRegistry
-	err = e.encode(&realEnc)
-	if err != nil {
-		return nil, err
-	}
-
-	return realEnc.raw, nil
-}
-
-// Decoder is the interface that wraps the basic Decode method.
-// Anything implementing Decoder can be extracted from bytes using Kafka's encoding rules.
-type decoder interface {
-	decode(pd packetDecoder) error
-}
-
-type versionedDecoder interface {
-	decode(pd packetDecoder, version int16) error
-}
-
-// Decode takes bytes and a Decoder and fills the fields of the decoder from the bytes,
-// interpreted using Kafka's encoding rules.
-func decode(buf []byte, in decoder) error {
-	if buf == nil {
-		return nil
-	}
-
-	helper := realDecoder{raw: buf}
-	err := in.decode(&helper)
-	if err != nil {
-		return err
-	}
-
-	if helper.off != len(buf) {
-		return PacketDecodingError{"invalid length"}
-	}
-
-	return nil
-}
-
-func versionedDecode(buf []byte, in versionedDecoder, version int16) error {
-	if buf == nil {
-		return nil
-	}
-
-	helper := realDecoder{raw: buf}
-	err := in.decode(&helper, version)
-	if err != nil {
-		return err
-	}
-
-	if helper.off != len(buf) {
-		return PacketDecodingError{"invalid length"}
-	}
-
-	return nil
-}

vendor/github.com/Shopify/sarama/errors.go

@@ -1,269 +0,0 @@
-package sarama
-
-import (
-	"errors"
-	"fmt"
-)
-
-// ErrOutOfBrokers is the error returned when the client has run out of brokers to talk to because all of them errored
-// or otherwise failed to respond.
-var ErrOutOfBrokers = errors.New("kafka: client has run out of available brokers to talk to (Is your cluster reachable?)")
-
-// ErrClosedClient is the error returned when a method is called on a client that has been closed.
-var ErrClosedClient = errors.New("kafka: tried to use a client that was closed")
-
-// ErrIncompleteResponse is the error returned when the server returns a syntactically valid response, but it does
-// not contain the expected information.
-var ErrIncompleteResponse = errors.New("kafka: response did not contain all the expected topic/partition blocks")
-
-// ErrInvalidPartition is the error returned when a partitioner returns an invalid partition index
-// (meaning one outside of the range [0...numPartitions-1]).
-var ErrInvalidPartition = errors.New("kafka: partitioner returned an invalid partition index")
-
-// ErrAlreadyConnected is the error returned when calling Open() on a Broker that is already connected or connecting.
-var ErrAlreadyConnected = errors.New("kafka: broker connection already initiated")
-
-// ErrNotConnected is the error returned when trying to send or call Close() on a Broker that is not connected.
-var ErrNotConnected = errors.New("kafka: broker not connected")
-
-// ErrInsufficientData is returned when decoding and the packet is truncated. This can be expected
-// when requesting messages, since as an optimization the server is allowed to return a partial message at the end
-// of the message set.
-var ErrInsufficientData = errors.New("kafka: insufficient data to decode packet, more bytes expected")
-
-// ErrShuttingDown is returned when a producer receives a message during shutdown.
-var ErrShuttingDown = errors.New("kafka: message received by producer in process of shutting down")
-
-// ErrMessageTooLarge is returned when the next message to consume is larger than the configured Consumer.Fetch.Max
-var ErrMessageTooLarge = errors.New("kafka: message is larger than Consumer.Fetch.Max")
-
-// PacketEncodingError is returned from a failure while encoding a Kafka packet. This can happen, for example,
-// if you try to encode a string over 2^15 characters in length, since Kafka's encoding rules do not permit that.
-type PacketEncodingError struct {
-	Info string
-}
-
-func (err PacketEncodingError) Error() string {
-	return fmt.Sprintf("kafka: error encoding packet: %s", err.Info)
-}
-
-// PacketDecodingError is returned when there was an error (other than truncated data) decoding the Kafka broker's response.
-// This can be a bad CRC or length field, or any other invalid value.
-type PacketDecodingError struct {
-	Info string
-}
-
-func (err PacketDecodingError) Error() string {
-	return fmt.Sprintf("kafka: error decoding packet: %s", err.Info)
-}
-
-// ConfigurationError is the type of error returned from a constructor (e.g. NewClient, or NewConsumer)
-// when the specified configuration is invalid.
-type ConfigurationError string
-
-func (err ConfigurationError) Error() string {
-	return "kafka: invalid configuration (" + string(err) + ")"
-}
-
-// KError is the type of error that can be returned directly by the Kafka broker.
-// See https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-ErrorCodes
-type KError int16
-
-// Numeric error codes returned by the Kafka server.
-const (
-	ErrNoError                            KError = 0
-	ErrUnknown                            KError = -1
-	ErrOffsetOutOfRange                   KError = 1
-	ErrInvalidMessage                     KError = 2
-	ErrUnknownTopicOrPartition            KError = 3
-	ErrInvalidMessageSize                 KError = 4
-	ErrLeaderNotAvailable                 KError = 5
-	ErrNotLeaderForPartition              KError = 6
-	ErrRequestTimedOut                    KError = 7
-	ErrBrokerNotAvailable                 KError = 8
-	ErrReplicaNotAvailable                KError = 9
-	ErrMessageSizeTooLarge                KError = 10
-	ErrStaleControllerEpochCode           KError = 11
-	ErrOffsetMetadataTooLarge             KError = 12
-	ErrNetworkException                   KError = 13
-	ErrOffsetsLoadInProgress              KError = 14
-	ErrConsumerCoordinatorNotAvailable    KError = 15
-	ErrNotCoordinatorForConsumer          KError = 16
-	ErrInvalidTopic                       KError = 17
-	ErrMessageSetSizeTooLarge             KError = 18
-	ErrNotEnoughReplicas                  KError = 19
-	ErrNotEnoughReplicasAfterAppend       KError = 20
-	ErrInvalidRequiredAcks                KError = 21
-	ErrIllegalGeneration                  KError = 22
-	ErrInconsistentGroupProtocol          KError = 23
-	ErrInvalidGroupId                     KError = 24
-	ErrUnknownMemberId                    KError = 25
-	ErrInvalidSessionTimeout              KError = 26
-	ErrRebalanceInProgress                KError = 27
-	ErrInvalidCommitOffsetSize            KError = 28
-	ErrTopicAuthorizationFailed           KError = 29
-	ErrGroupAuthorizationFailed           KError = 30
-	ErrClusterAuthorizationFailed         KError = 31
-	ErrInvalidTimestamp                   KError = 32
-	ErrUnsupportedSASLMechanism           KError = 33
-	ErrIllegalSASLState                   KError = 34
-	ErrUnsupportedVersion                 KError = 35
-	ErrTopicAlreadyExists                 KError = 36
-	ErrInvalidPartitions                  KError = 37
-	ErrInvalidReplicationFactor           KError = 38
-	ErrInvalidReplicaAssignment           KError = 39
-	ErrInvalidConfig                      KError = 40
-	ErrNotController                      KError = 41
-	ErrInvalidRequest                     KError = 42
-	ErrUnsupportedForMessageFormat        KError = 43
-	ErrPolicyViolation                    KError = 44
-	ErrOutOfOrderSequenceNumber           KError = 45
-	ErrDuplicateSequenceNumber            KError = 46
-	ErrInvalidProducerEpoch               KError = 47
-	ErrInvalidTxnState                    KError = 48
-	ErrInvalidProducerIDMapping           KError = 49
-	ErrInvalidTransactionTimeout          KError = 50
-	ErrConcurrentTransactions             KError = 51
-	ErrTransactionCoordinatorFenced       KError = 52
-	ErrTransactionalIDAuthorizationFailed KError = 53
-	ErrSecurityDisabled                   KError = 54
-	ErrOperationNotAttempted              KError = 55
-	ErrKafkaStorageError                  KError = 56
-	ErrLogDirNotFound                     KError = 57
-	ErrSASLAuthenticationFailed           KError = 58
-	ErrUnknownProducerID                  KError = 59
-	ErrReassignmentInProgress             KError = 60
-)
-
-func (err KError) Error() string {
-	// Error messages stolen/adapted from
-	// https://kafka.apache.org/protocol#protocol_error_codes
-	switch err {
-	case ErrNoError:
-		return "kafka server: Not an error, why are you printing me?"
-	case ErrUnknown:
-		return "kafka server: Unexpected (unknown?) server error."
-	case ErrOffsetOutOfRange:
-		return "kafka server: The requested offset is outside the range of offsets maintained by the server for the given topic/partition."
-	case ErrInvalidMessage:
-		return "kafka server: Message contents does not match its CRC."
-	case ErrUnknownTopicOrPartition:
-		return "kafka server: Request was for a topic or partition that does not exist on this broker."
-	case ErrInvalidMessageSize:
-		return "kafka server: The message has a negative size."
-	case ErrLeaderNotAvailable:
-		return "kafka server: In the middle of a leadership election, there is currently no leader for this partition and hence it is unavailable for writes."
-	case ErrNotLeaderForPartition:
-		return "kafka server: Tried to send a message to a replica that is not the leader for some partition. Your metadata is out of date."
-	case ErrRequestTimedOut:
-		return "kafka server: Request exceeded the user-specified time limit in the request."
-	case ErrBrokerNotAvailable:
-		return "kafka server: Broker not available. Not a client facing error, we should never receive this!!!"
-	case ErrReplicaNotAvailable:
-		return "kafka server: Replica information not available, one or more brokers are down."
-	case ErrMessageSizeTooLarge:
-		return "kafka server: Message was too large, server rejected it to avoid allocation error."
-	case ErrStaleControllerEpochCode:
-		return "kafka server: StaleControllerEpochCode (internal error code for broker-to-broker communication)."
-	case ErrOffsetMetadataTooLarge:
-		return "kafka server: Specified a string larger than the configured maximum for offset metadata."
-	case ErrNetworkException:
-		return "kafka server: The server disconnected before a response was received."
-	case ErrOffsetsLoadInProgress:
-		return "kafka server: The broker is still loading offsets after a leader change for that offset's topic partition."
-	case ErrConsumerCoordinatorNotAvailable:
-		return "kafka server: Offset's topic has not yet been created."
-	case ErrNotCoordinatorForConsumer:
-		return "kafka server: Request was for a consumer group that is not coordinated by this broker."
-	case ErrInvalidTopic:
-		return "kafka server: The request attempted to perform an operation on an invalid topic."
-	case ErrMessageSetSizeTooLarge:
-		return "kafka server: The request included message batch larger than the configured segment size on the server."
-	case ErrNotEnoughReplicas:
-		return "kafka server: Messages are rejected since there are fewer in-sync replicas than required."
-	case ErrNotEnoughReplicasAfterAppend:
-		return "kafka server: Messages are written to the log, but to fewer in-sync replicas than required."
-	case ErrInvalidRequiredAcks:
-		return "kafka server: The number of required acks is invalid (should be either -1, 0, or 1)."
-	case ErrIllegalGeneration:
-		return "kafka server: The provided generation id is not the current generation."
-	case ErrInconsistentGroupProtocol:
-		return "kafka server: The provider group protocol type is incompatible with the other members."
-	case ErrInvalidGroupId:
-		return "kafka server: The provided group id was empty."
-	case ErrUnknownMemberId:
-		return "kafka server: The provided member is not known in the current generation."
-	case ErrInvalidSessionTimeout:
-		return "kafka server: The provided session timeout is outside the allowed range."
-	case ErrRebalanceInProgress:
-		return "kafka server: A rebalance for the group is in progress. Please re-join the group."
-	case ErrInvalidCommitOffsetSize:
-		return "kafka server: The provided commit metadata was too large."
-	case ErrTopicAuthorizationFailed:
-		return "kafka server: The client is not authorized to access this topic."
-	case ErrGroupAuthorizationFailed:
-		return "kafka server: The client is not authorized to access this group."
-	case ErrClusterAuthorizationFailed:
-		return "kafka server: The client is not authorized to send this request type."
-	case ErrInvalidTimestamp:
-		return "kafka server: The timestamp of the message is out of acceptable range."
-	case ErrUnsupportedSASLMechanism:
-		return "kafka server: The broker does not support the requested SASL mechanism."
-	case ErrIllegalSASLState:
-		return "kafka server: Request is not valid given the current SASL state."
-	case ErrUnsupportedVersion:
-		return "kafka server: The version of API is not supported."
-	case ErrTopicAlreadyExists:
-		return "kafka server: Topic with this name already exists."
-	case ErrInvalidPartitions:
-		return "kafka server: Number of partitions is invalid."
-	case ErrInvalidReplicationFactor:
-		return "kafka server: Replication-factor is invalid."
-	case ErrInvalidReplicaAssignment:
-		return "kafka server: Replica assignment is invalid."
-	case ErrInvalidConfig:
-		return "kafka server: Configuration is invalid."
-	case ErrNotController:
-		return "kafka server: This is not the correct controller for this cluster."
-	case ErrInvalidRequest:
-		return "kafka server: This most likely occurs because of a request being malformed by the client library or the message was sent to an incompatible broker. See the broker logs for more details."
-	case ErrUnsupportedForMessageFormat:
-		return "kafka server: The requested operation is not supported by the message format version."
-	case ErrPolicyViolation:
-		return "kafka server: Request parameters do not satisfy the configured policy."
-	case ErrOutOfOrderSequenceNumber:
-		return "kafka server: The broker received an out of order sequence number."
-	case ErrDuplicateSequenceNumber:
-		return "kafka server: The broker received a duplicate sequence number."
-	case ErrInvalidProducerEpoch:
-		return "kafka server: Producer attempted an operation with an old epoch."
-	case ErrInvalidTxnState:
-		return "kafka server: The producer attempted a transactional operation in an invalid state."
-	case ErrInvalidProducerIDMapping:
-		return "kafka server: The producer attempted to use a producer id which is not currently assigned to its transactional id."
-	case ErrInvalidTransactionTimeout:
-		return "kafka server: The transaction timeout is larger than the maximum value allowed by the broker (as configured by max.transaction.timeout.ms)."
-	case ErrConcurrentTransactions:
-		return "kafka server: The producer attempted to update a transaction while another concurrent operation on the same transaction was ongoing."
-	case ErrTransactionCoordinatorFenced:
-		return "kafka server: The transaction coordinator sending a WriteTxnMarker is no longer the current coordinator for a given producer."
-	case ErrTransactionalIDAuthorizationFailed:
-		return "kafka server: Transactional ID authorization failed."
-	case ErrSecurityDisabled:
-		return "kafka server: Security features are disabled."
-	case ErrOperationNotAttempted:
-		return "kafka server: The broker did not attempt to execute this operation."
-	case ErrKafkaStorageError:
-		return "kafka server: Disk error when trying to access log file on the disk."
-	case ErrLogDirNotFound:
-		return "kafka server: The specified log directory is not found in the broker config."
-	case ErrSASLAuthenticationFailed:
-		return "kafka server: SASL Authentication failed."
-	case ErrUnknownProducerID:
-		return "kafka server: The broker could not locate the producer metadata associated with the Producer ID."
-	case ErrReassignmentInProgress:
-		return "kafka server: A partition reassignment is in progress."
-	}
-
-	return fmt.Sprintf("Unknown error, how did this happen? Error code = %d", err)
-}

vendor/github.com/Shopify/sarama/examples/README.md

@@ -1,9 +0,0 @@
-# Sarama examples
-
-This folder contains example applications to demonstrate the use of Sarama. For code snippet examples on how to use the different types in Sarama, see [Sarama's API documentation on godoc.org](https://godoc.org/github.com/Shopify/sarama)
-
-In these examples, we use `github.com/Shopify/sarama` as import path. We do this to ensure all the examples are up to date with the latest changes in Sarama. For your own applications, you may want to use `gopkg.in/Shopify/sarama.v1` to lock into a stable API version.
-
-#### HTTP server
-
-[http_server](./http_server) is a simple HTTP server uses both the sync producer to produce data as part of the request handling cycle, as well as the async producer to maintain an access log. It also uses the [mocks subpackage](https://godoc.org/github.com/Shopify/sarama/mocks) to test both.

vendor/github.com/Shopify/sarama/examples/http_server/.gitignore

@@ -1,2 +0,0 @@
-http_server
-http_server.test

vendor/github.com/Shopify/sarama/examples/http_server/README.md

@@ -1,7 +0,0 @@
-# HTTP server example
-
-This HTTP server example shows you how to use the AsyncProducer and SyncProducer, and how to test them using mocks. The server simply sends the data of the HTTP request's query string to Kafka, and send a 200 result if that succeeds. For every request, it will send an access log entry to Kafka as well in the background.
-
-If you need to know whether a message was successfully sent to the Kafka cluster before you can send your HTTP response, using the `SyncProducer` is probably the simplest way to achieve this. If you don't care, e.g. for the access log, using the `AsyncProducer` will let you fire and forget. You can send the HTTP response, while the message is being produced in the background.
-
-One important thing to note is that both the `SyncProducer` and `AsyncProducer` are **thread-safe**. Go's `http.Server` handles requests concurrently in different goroutines, but you can use a single producer safely. This will actually achieve efficiency gains as the producer will be able to batch messages from concurrent requests together.

vendor/github.com/Shopify/sarama/examples/http_server/http_server.go

@@ -1,247 +0,0 @@
-package main
-
-import (
-	"github.com/Shopify/sarama"
-
-	"crypto/tls"
-	"crypto/x509"
-	"encoding/json"
-	"flag"
-	"fmt"
-	"io/ioutil"
-	"log"
-	"net/http"
-	"os"
-	"strings"
-	"time"
-)
-
-var (
-	addr      = flag.String("addr", ":8080", "The address to bind to")
-	brokers   = flag.String("brokers", os.Getenv("KAFKA_PEERS"), "The Kafka brokers to connect to, as a comma separated list")
-	verbose   = flag.Bool("verbose", false, "Turn on Sarama logging")
-	certFile  = flag.String("certificate", "", "The optional certificate file for client authentication")
-	keyFile   = flag.String("key", "", "The optional key file for client authentication")
-	caFile    = flag.String("ca", "", "The optional certificate authority file for TLS client authentication")
-	verifySsl = flag.Bool("verify", false, "Optional verify ssl certificates chain")
-)
-
-func main() {
-	flag.Parse()
-
-	if *verbose {
-		sarama.Logger = log.New(os.Stdout, "[sarama] ", log.LstdFlags)
-	}
-
-	if *brokers == "" {
-		flag.PrintDefaults()
-		os.Exit(1)
-	}
-
-	brokerList := strings.Split(*brokers, ",")
-	log.Printf("Kafka brokers: %s", strings.Join(brokerList, ", "))
-
-	server := &Server{
-		DataCollector:     newDataCollector(brokerList),
-		AccessLogProducer: newAccessLogProducer(brokerList),
-	}
-	defer func() {
-		if err := server.Close(); err != nil {
-			log.Println("Failed to close server", err)
-		}
-	}()
-
-	log.Fatal(server.Run(*addr))
-}
-
-func createTlsConfiguration() (t *tls.Config) {
-	if *certFile != "" && *keyFile != "" && *caFile != "" {
-		cert, err := tls.LoadX509KeyPair(*certFile, *keyFile)
-		if err != nil {
-			log.Fatal(err)
-		}
-
-		caCert, err := ioutil.ReadFile(*caFile)
-		if err != nil {
-			log.Fatal(err)
-		}
-
-		caCertPool := x509.NewCertPool()
-		caCertPool.AppendCertsFromPEM(caCert)
-
-		t = &tls.Config{
-			Certificates:       []tls.Certificate{cert},
-			RootCAs:            caCertPool,
-			InsecureSkipVerify: *verifySsl,
-		}
-	}
-	// will be nil by default if nothing is provided
-	return t
-}
-
-type Server struct {
-	DataCollector     sarama.SyncProducer
-	AccessLogProducer sarama.AsyncProducer
-}
-
-func (s *Server) Close() error {
-	if err := s.DataCollector.Close(); err != nil {
-		log.Println("Failed to shut down data collector cleanly", err)
-	}
-
-	if err := s.AccessLogProducer.Close(); err != nil {
-		log.Println("Failed to shut down access log producer cleanly", err)
-	}
-
-	return nil
-}
-
-func (s *Server) Handler() http.Handler {
-	return s.withAccessLog(s.collectQueryStringData())
-}
-
-func (s *Server) Run(addr string) error {
-	httpServer := &http.Server{
-		Addr:    addr,
-		Handler: s.Handler(),
-	}
-
-	log.Printf("Listening for requests on %s...\n", addr)
-	return httpServer.ListenAndServe()
-}
-
-func (s *Server) collectQueryStringData() http.Handler {
-	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		if r.URL.Path != "/" {
-			http.NotFound(w, r)
-			return
-		}
-
-		// We are not setting a message key, which means that all messages will
-		// be distributed randomly over the different partitions.
-		partition, offset, err := s.DataCollector.SendMessage(&sarama.ProducerMessage{
-			Topic: "important",
-			Value: sarama.StringEncoder(r.URL.RawQuery),
-		})
-
-		if err != nil {
-			w.WriteHeader(http.StatusInternalServerError)
-			fmt.Fprintf(w, "Failed to store your data:, %s", err)
-		} else {
-			// The tuple (topic, partition, offset) can be used as a unique identifier
-			// for a message in a Kafka cluster.
-			fmt.Fprintf(w, "Your data is stored with unique identifier important/%d/%d", partition, offset)
-		}
-	})
-}
-
-type accessLogEntry struct {
-	Method       string  `json:"method"`
-	Host         string  `json:"host"`
-	Path         string  `json:"path"`
-	IP           string  `json:"ip"`
-	ResponseTime float64 `json:"response_time"`
-
-	encoded []byte
-	err     error
-}
-
-func (ale *accessLogEntry) ensureEncoded() {
-	if ale.encoded == nil && ale.err == nil {
-		ale.encoded, ale.err = json.Marshal(ale)
-	}
-}
-
-func (ale *accessLogEntry) Length() int {
-	ale.ensureEncoded()
-	return len(ale.encoded)
-}
-
-func (ale *accessLogEntry) Encode() ([]byte, error) {
-	ale.ensureEncoded()
-	return ale.encoded, ale.err
-}
-
-func (s *Server) withAccessLog(next http.Handler) http.Handler {
-
-	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		started := time.Now()
-
-		next.ServeHTTP(w, r)
-
-		entry := &accessLogEntry{
-			Method:       r.Method,
-			Host:         r.Host,
-			Path:         r.RequestURI,
-			IP:           r.RemoteAddr,
-			ResponseTime: float64(time.Since(started)) / float64(time.Second),
-		}
-
-		// We will use the client's IP address as key. This will cause
-		// all the access log entries of the same IP address to end up
-		// on the same partition.
-		s.AccessLogProducer.Input() <- &sarama.ProducerMessage{
-			Topic: "access_log",
-			Key:   sarama.StringEncoder(r.RemoteAddr),
-			Value: entry,
-		}
-	})
-}
-
-func newDataCollector(brokerList []string) sarama.SyncProducer {
-
-	// For the data collector, we are looking for strong consistency semantics.
-	// Because we don't change the flush settings, sarama will try to produce messages
-	// as fast as possible to keep latency low.
-	config := sarama.NewConfig()
-	config.Producer.RequiredAcks = sarama.WaitForAll // Wait for all in-sync replicas to ack the message
-	config.Producer.Retry.Max = 10                   // Retry up to 10 times to produce the message
-	config.Producer.Return.Successes = true
-	tlsConfig := createTlsConfiguration()
-	if tlsConfig != nil {
-		config.Net.TLS.Config = tlsConfig
-		config.Net.TLS.Enable = true
-	}
-
-	// On the broker side, you may want to change the following settings to get
-	// stronger consistency guarantees:
-	// - For your broker, set `unclean.leader.election.enable` to false
-	// - For the topic, you could increase `min.insync.replicas`.
-
-	producer, err := sarama.NewSyncProducer(brokerList, config)
-	if err != nil {
-		log.Fatalln("Failed to start Sarama producer:", err)
-	}
-
-	return producer
-}
-
-func newAccessLogProducer(brokerList []string) sarama.AsyncProducer {
-
-	// For the access log, we are looking for AP semantics, with high throughput.
-	// By creating batches of compressed messages, we reduce network I/O at a cost of more latency.
-	config := sarama.NewConfig()
-	tlsConfig := createTlsConfiguration()
-	if tlsConfig != nil {
-		config.Net.TLS.Enable = true
-		config.Net.TLS.Config = tlsConfig
-	}
-	config.Producer.RequiredAcks = sarama.WaitForLocal       // Only wait for the leader to ack
-	config.Producer.Compression = sarama.CompressionSnappy   // Compress messages
-	config.Producer.Flush.Frequency = 500 * time.Millisecond // Flush batches every 500ms
-
-	producer, err := sarama.NewAsyncProducer(brokerList, config)
-	if err != nil {
-		log.Fatalln("Failed to start Sarama producer:", err)
-	}
-
-	// We will just log to STDOUT if we're not able to produce messages.
-	// Note: messages will only be returned here after all retry attempts are exhausted.
-	go func() {
-		for err := range producer.Errors() {
-			log.Println("Failed to write access log entry:", err)
-		}
-	}()
-
-	return producer
-}

vendor/github.com/Shopify/sarama/examples/http_server/http_server_test.go

@@ -1,109 +0,0 @@
-package main
-
-import (
-	"io"
-	"net/http"
-	"net/http/httptest"
-	"testing"
-
-	"github.com/Shopify/sarama"
-	"github.com/Shopify/sarama/mocks"
-)
-
-// In normal operation, we expect one access log entry,
-// and one data collector entry. Let's assume both will succeed.
-// We should return a HTTP 200 status.
-func TestCollectSuccessfully(t *testing.T) {
-	dataCollectorMock := mocks.NewSyncProducer(t, nil)
-	dataCollectorMock.ExpectSendMessageAndSucceed()
-
-	accessLogProducerMock := mocks.NewAsyncProducer(t, nil)
-	accessLogProducerMock.ExpectInputAndSucceed()
-
-	// Now, use dependency injection to use the mocks.
-	s := &Server{
-		DataCollector:     dataCollectorMock,
-		AccessLogProducer: accessLogProducerMock,
-	}
-
-	// The Server's Close call is important; it will call Close on
-	// the two mock producers, which will then validate whether all
-	// expectations are resolved.
-	defer safeClose(t, s)
-
-	req, err := http.NewRequest("GET", "http://example.com/?data", nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-	res := httptest.NewRecorder()
-	s.Handler().ServeHTTP(res, req)
-
-	if res.Code != 200 {
-		t.Errorf("Expected HTTP status 200, found %d", res.Code)
-	}
-
-	if string(res.Body.Bytes()) != "Your data is stored with unique identifier important/0/1" {
-		t.Error("Unexpected response body", res.Body)
-	}
-}
-
-// Now, let's see if we handle the case of not being able to produce
-// to the data collector properly. In this case we should return a 500 status.
-func TestCollectionFailure(t *testing.T) {
-	dataCollectorMock := mocks.NewSyncProducer(t, nil)
-	dataCollectorMock.ExpectSendMessageAndFail(sarama.ErrRequestTimedOut)
-
-	accessLogProducerMock := mocks.NewAsyncProducer(t, nil)
-	accessLogProducerMock.ExpectInputAndSucceed()
-
-	s := &Server{
-		DataCollector:     dataCollectorMock,
-		AccessLogProducer: accessLogProducerMock,
-	}
-	defer safeClose(t, s)
-
-	req, err := http.NewRequest("GET", "http://example.com/?data", nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-	res := httptest.NewRecorder()
-	s.Handler().ServeHTTP(res, req)
-
-	if res.Code != 500 {
-		t.Errorf("Expected HTTP status 500, found %d", res.Code)
-	}
-}
-
-// We don't expect any data collector calls because the path is wrong,
-// so we are not setting any expectations on the dataCollectorMock. It
-// will still generate an access log entry though.
-func TestWrongPath(t *testing.T) {
-	dataCollectorMock := mocks.NewSyncProducer(t, nil)
-
-	accessLogProducerMock := mocks.NewAsyncProducer(t, nil)
-	accessLogProducerMock.ExpectInputAndSucceed()
-
-	s := &Server{
-		DataCollector:     dataCollectorMock,
-		AccessLogProducer: accessLogProducerMock,
-	}
-	defer safeClose(t, s)
-
-	req, err := http.NewRequest("GET", "http://example.com/wrong?data", nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-	res := httptest.NewRecorder()
-
-	s.Handler().ServeHTTP(res, req)
-
-	if res.Code != 404 {
-		t.Errorf("Expected HTTP status 404, found %d", res.Code)
-	}
-}
-
-func safeClose(t *testing.T, o io.Closer) {
-	if err := o.Close(); err != nil {
-		t.Error(err)
-	}
-}

vendor/github.com/Shopify/sarama/fetch_request.go

@@ -1,170 +0,0 @@
-package sarama
-
-type fetchRequestBlock struct {
-	fetchOffset int64
-	maxBytes    int32
-}
-
-func (b *fetchRequestBlock) encode(pe packetEncoder) error {
-	pe.putInt64(b.fetchOffset)
-	pe.putInt32(b.maxBytes)
-	return nil
-}
-
-func (b *fetchRequestBlock) decode(pd packetDecoder) (err error) {
-	if b.fetchOffset, err = pd.getInt64(); err != nil {
-		return err
-	}
-	if b.maxBytes, err = pd.getInt32(); err != nil {
-		return err
-	}
-	return nil
-}
-
-// FetchRequest (API key 1) will fetch Kafka messages. Version 3 introduced the MaxBytes field. See
-// https://issues.apache.org/jira/browse/KAFKA-2063 for a discussion of the issues leading up to that.  The KIP is at
-// https://cwiki.apache.org/confluence/display/KAFKA/KIP-74%3A+Add+Fetch+Response+Size+Limit+in+Bytes
-type FetchRequest struct {
-	MaxWaitTime int32
-	MinBytes    int32
-	MaxBytes    int32
-	Version     int16
-	Isolation   IsolationLevel
-	blocks      map[string]map[int32]*fetchRequestBlock
-}
-
-type IsolationLevel int8
-
-const (
-	ReadUncommitted IsolationLevel = 0
-	ReadCommitted   IsolationLevel = 1
-)
-
-func (r *FetchRequest) encode(pe packetEncoder) (err error) {
-	pe.putInt32(-1) // replica ID is always -1 for clients
-	pe.putInt32(r.MaxWaitTime)
-	pe.putInt32(r.MinBytes)
-	if r.Version >= 3 {
-		pe.putInt32(r.MaxBytes)
-	}
-	if r.Version >= 4 {
-		pe.putInt8(int8(r.Isolation))
-	}
-	err = pe.putArrayLength(len(r.blocks))
-	if err != nil {
-		return err
-	}
-	for topic, blocks := range r.blocks {
-		err = pe.putString(topic)
-		if err != nil {
-			return err
-		}
-		err = pe.putArrayLength(len(blocks))
-		if err != nil {
-			return err
-		}
-		for partition, block := range blocks {
-			pe.putInt32(partition)
-			err = block.encode(pe)
-			if err != nil {
-				return err
-			}
-		}
-	}
-	return nil
-}
-
-func (r *FetchRequest) decode(pd packetDecoder, version int16) (err error) {
-	r.Version = version
-	if _, err = pd.getInt32(); err != nil {
-		return err
-	}
-	if r.MaxWaitTime, err = pd.getInt32(); err != nil {
-		return err
-	}
-	if r.MinBytes, err = pd.getInt32(); err != nil {
-		return err
-	}
-	if r.Version >= 3 {
-		if r.MaxBytes, err = pd.getInt32(); err != nil {
-			return err
-		}
-	}
-	if r.Version >= 4 {
-		isolation, err := pd.getInt8()
-		if err != nil {
-			return err
-		}
-		r.Isolation = IsolationLevel(isolation)
-	}
-	topicCount, err := pd.getArrayLength()
-	if err != nil {
-		return err
-	}
-	if topicCount == 0 {
-		return nil
-	}
-	r.blocks = make(map[string]map[int32]*fetchRequestBlock)
-	for i := 0; i < topicCount; i++ {
-		topic, err := pd.getString()
-		if err != nil {
-			return err
-		}
-		partitionCount, err := pd.getArrayLength()
-		if err != nil {
-			return err
-		}
-		r.blocks[topic] = make(map[int32]*fetchRequestBlock)
-		for j := 0; j < partitionCount; j++ {
-			partition, err := pd.getInt32()
-			if err != nil {
-				return err
-			}
-			fetchBlock := &fetchRequestBlock{}
-			if err = fetchBlock.decode(pd); err != nil {
-				return err
-			}
-			r.blocks[topic][partition] = fetchBlock
-		}
-	}
-	return nil
-}
-
-func (r *FetchRequest) key() int16 {
-	return 1
-}
-
-func (r *FetchRequest) version() int16 {
-	return r.Version
-}
-
-func (r *FetchRequest) requiredVersion() KafkaVersion {
-	switch r.Version {
-	case 1:
-		return V0_9_0_0
-	case 2:
-		return V0_10_0_0
-	case 3:
-		return V0_10_1_0
-	case 4:
-		return V0_11_0_0
-	default:
-		return minVersion
-	}
-}
-
-func (r *FetchRequest) AddBlock(topic string, partitionID int32, fetchOffset int64, maxBytes int32) {
-	if r.blocks == nil {
-		r.blocks = make(map[string]map[int32]*fetchRequestBlock)
-	}
-
-	if r.blocks[topic] == nil {
-		r.blocks[topic] = make(map[int32]*fetchRequestBlock)
-	}
-
-	tmp := new(fetchRequestBlock)
-	tmp.maxBytes = maxBytes
-	tmp.fetchOffset = fetchOffset
-
-	r.blocks[topic][partitionID] = tmp
-}

vendor/github.com/Shopify/sarama/fetch_request_test.go

@@ -1,48 +0,0 @@
-package sarama
-
-import "testing"
-
-var (
-	fetchRequestNoBlocks = []byte{
-		0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-		0x00, 0x00, 0x00, 0x00}
-
-	fetchRequestWithProperties = []byte{
-		0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0xEF,
-		0x00, 0x00, 0x00, 0x00}
-
-	fetchRequestOneBlock = []byte{
-		0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-		0x00, 0x00, 0x00, 0x01,
-		0x00, 0x05, 't', 'o', 'p', 'i', 'c',
-		0x00, 0x00, 0x00, 0x01,
-		0x00, 0x00, 0x00, 0x12, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x34, 0x00, 0x00, 0x00, 0x56}
-
-	fetchRequestOneBlockV4 = []byte{
-		0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-		0x00, 0x00, 0x00, 0xFF,
-		0x01,
-		0x00, 0x00, 0x00, 0x01,
-		0x00, 0x05, 't', 'o', 'p', 'i', 'c',
-		0x00, 0x00, 0x00, 0x01,
-		0x00, 0x00, 0x00, 0x12, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x34, 0x00, 0x00, 0x00, 0x56}
-)
-
-func TestFetchRequest(t *testing.T) {
-	request := new(FetchRequest)
-	testRequest(t, "no blocks", request, fetchRequestNoBlocks)
-
-	request.MaxWaitTime = 0x20
-	request.MinBytes = 0xEF
-	testRequest(t, "with properties", request, fetchRequestWithProperties)
-
-	request.MaxWaitTime = 0
-	request.MinBytes = 0
-	request.AddBlock("topic", 0x12, 0x34, 0x56)
-	testRequest(t, "one block", request, fetchRequestOneBlock)
-
-	request.Version = 4
-	request.MaxBytes = 0xFF
-	request.Isolation = ReadCommitted
-	testRequest(t, "one block v4", request, fetchRequestOneBlockV4)
-}

vendor/github.com/Shopify/sarama/fetch_response.go

@@ -1,315 +0,0 @@
-package sarama
-
-import "time"
-
-type AbortedTransaction struct {
-	ProducerID  int64
-	FirstOffset int64
-}
-
-func (t *AbortedTransaction) decode(pd packetDecoder) (err error) {
-	if t.ProducerID, err = pd.getInt64(); err != nil {
-		return err
-	}
-
-	if t.FirstOffset, err = pd.getInt64(); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func (t *AbortedTransaction) encode(pe packetEncoder) (err error) {
-	pe.putInt64(t.ProducerID)
-	pe.putInt64(t.FirstOffset)
-
-	return nil
-}
-
-type FetchResponseBlock struct {
-	Err                 KError
-	HighWaterMarkOffset int64
-	LastStableOffset    int64
-	AbortedTransactions []*AbortedTransaction
-	Records             Records
-}
-
-func (b *FetchResponseBlock) decode(pd packetDecoder, version int16) (err error) {
-	tmp, err := pd.getInt16()
-	if err != nil {
-		return err
-	}
-	b.Err = KError(tmp)
-
-	b.HighWaterMarkOffset, err = pd.getInt64()
-	if err != nil {
-		return err
-	}
-
-	if version >= 4 {
-		b.LastStableOffset, err = pd.getInt64()
-		if err != nil {
-			return err
-		}
-
-		numTransact, err := pd.getArrayLength()
-		if err != nil {
-			return err
-		}
-
-		if numTransact >= 0 {
-			b.AbortedTransactions = make([]*AbortedTransaction, numTransact)
-		}
-
-		for i := 0; i < numTransact; i++ {
-			transact := new(AbortedTransaction)
-			if err = transact.decode(pd); err != nil {
-				return err
-			}
-			b.AbortedTransactions[i] = transact
-		}
-	}
-
-	recordsSize, err := pd.getInt32()
-	if err != nil {
-		return err
-	}
-
-	recordsDecoder, err := pd.getSubset(int(recordsSize))
-	if err != nil {
-		return err
-	}
-	if recordsSize > 0 {
-		if err = b.Records.decode(recordsDecoder); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (b *FetchResponseBlock) encode(pe packetEncoder, version int16) (err error) {
-	pe.putInt16(int16(b.Err))
-
-	pe.putInt64(b.HighWaterMarkOffset)
-
-	if version >= 4 {
-		pe.putInt64(b.LastStableOffset)
-
-		if err = pe.putArrayLength(len(b.AbortedTransactions)); err != nil {
-			return err
-		}
-		for _, transact := range b.AbortedTransactions {
-			if err = transact.encode(pe); err != nil {
-				return err
-			}
-		}
-	}
-
-	pe.push(&lengthField{})
-	err = b.Records.encode(pe)
-	if err != nil {
-		return err
-	}
-	return pe.pop()
-}
-
-type FetchResponse struct {
-	Blocks       map[string]map[int32]*FetchResponseBlock
-	ThrottleTime time.Duration
-	Version      int16 // v1 requires 0.9+, v2 requires 0.10+
-}
-
-func (r *FetchResponse) decode(pd packetDecoder, version int16) (err error) {
-	r.Version = version
-
-	if r.Version >= 1 {
-		throttle, err := pd.getInt32()
-		if err != nil {
-			return err
-		}
-		r.ThrottleTime = time.Duration(throttle) * time.Millisecond
-	}
-
-	numTopics, err := pd.getArrayLength()
-	if err != nil {
-		return err
-	}
-
-	r.Blocks = make(map[string]map[int32]*FetchResponseBlock, numTopics)
-	for i := 0; i < numTopics; i++ {
-		name, err := pd.getString()
-		if err != nil {
-			return err
-		}
-
-		numBlocks, err := pd.getArrayLength()
-		if err != nil {
-			return err
-		}
-
-		r.Blocks[name] = make(map[int32]*FetchResponseBlock, numBlocks)
-
-		for j := 0; j < numBlocks; j++ {
-			id, err := pd.getInt32()
-			if err != nil {
-				return err
-			}
-
-			block := new(FetchResponseBlock)
-			err = block.decode(pd, version)
-			if err != nil {
-				return err
-			}
-			r.Blocks[name][id] = block
-		}
-	}
-
-	return nil
-}
-
-func (r *FetchResponse) encode(pe packetEncoder) (err error) {
-	if r.Version >= 1 {
-		pe.putInt32(int32(r.ThrottleTime / time.Millisecond))
-	}
-
-	err = pe.putArrayLength(len(r.Blocks))
-	if err != nil {
-		return err
-	}
-
-	for topic, partitions := range r.Blocks {
-		err = pe.putString(topic)
-		if err != nil {
-			return err
-		}
-
-		err = pe.putArrayLength(len(partitions))
-		if err != nil {
-			return err
-		}
-
-		for id, block := range partitions {
-			pe.putInt32(id)
-			err = block.encode(pe, r.Version)
-			if err != nil {
-				return err
-			}
-		}
-
-	}
-	return nil
-}
-
-func (r *FetchResponse) key() int16 {
-	return 1
-}
-
-func (r *FetchResponse) version() int16 {
-	return r.Version
-}
-
-func (r *FetchResponse) requiredVersion() KafkaVersion {
-	switch r.Version {
-	case 1:
-		return V0_9_0_0
-	case 2:
-		return V0_10_0_0
-	case 3:
-		return V0_10_1_0
-	case 4:
-		return V0_11_0_0
-	default:
-		return minVersion
-	}
-}
-
-func (r *FetchResponse) GetBlock(topic string, partition int32) *FetchResponseBlock {
-	if r.Blocks == nil {
-		return nil
-	}
-
-	if r.Blocks[topic] == nil {
-		return nil
-	}
-
-	return r.Blocks[topic][partition]
-}
-
-func (r *FetchResponse) AddError(topic string, partition int32, err KError) {
-	if r.Blocks == nil {
-		r.Blocks = make(map[string]map[int32]*FetchResponseBlock)
-	}
-	partitions, ok := r.Blocks[topic]
-	if !ok {
-		partitions = make(map[int32]*FetchResponseBlock)
-		r.Blocks[topic] = partitions
-	}
-	frb, ok := partitions[partition]
-	if !ok {
-		frb = new(FetchResponseBlock)
-		partitions[partition] = frb
-	}
-	frb.Err = err
-}
-
-func (r *FetchResponse) getOrCreateBlock(topic string, partition int32) *FetchResponseBlock {
-	if r.Blocks == nil {
-		r.Blocks = make(map[string]map[int32]*FetchResponseBlock)
-	}
-	partitions, ok := r.Blocks[topic]
-	if !ok {
-		partitions = make(map[int32]*FetchResponseBlock)
-		r.Blocks[topic] = partitions
-	}
-	frb, ok := partitions[partition]
-	if !ok {
-		frb = new(FetchResponseBlock)
-		partitions[partition] = frb
-	}
-
-	return frb
-}
-
-func encodeKV(key, value Encoder) ([]byte, []byte) {
-	var kb []byte
-	var vb []byte
-	if key != nil {
-		kb, _ = key.Encode()
-	}
-	if value != nil {
-		vb, _ = value.Encode()
-	}
-
-	return kb, vb
-}
-
-func (r *FetchResponse) AddMessage(topic string, partition int32, key, value Encoder, offset int64) {
-	frb := r.getOrCreateBlock(topic, partition)
-	kb, vb := encodeKV(key, value)
-	msg := &Message{Key: kb, Value: vb}
-	msgBlock := &MessageBlock{Msg: msg, Offset: offset}
-	set := frb.Records.msgSet
-	if set == nil {
-		set = &MessageSet{}
-		frb.Records = newLegacyRecords(set)
-	}
-	set.Messages = append(set.Messages, msgBlock)
-}
-
-func (r *FetchResponse) AddRecord(topic string, partition int32, key, value Encoder, offset int64) {
-	frb := r.getOrCreateBlock(topic, partition)
-	kb, vb := encodeKV(key, value)
-	rec := &Record{Key: kb, Value: vb, OffsetDelta: offset}
-	batch := frb.Records.recordBatch
-	if batch == nil {
-		batch = &RecordBatch{Version: 2}
-		frb.Records = newDefaultRecords(batch)
-	}
-	batch.addRecord(rec)
-}
-
-func (r *FetchResponse) SetLastStableOffset(topic string, partition int32, offset int64) {
-	frb := r.getOrCreateBlock(topic, partition)
-	frb.LastStableOffset = offset
-}

vendor/github.com/Shopify/sarama/fetch_response_test.go

@@ -1,248 +0,0 @@
-package sarama
-
-import (
-	"bytes"
-	"testing"
-)
-
-var (
-	emptyFetchResponse = []byte{
-		0x00, 0x00, 0x00, 0x00}
-
-	oneMessageFetchResponse = []byte{
-		0x00, 0x00, 0x00, 0x01,
-		0x00, 0x05, 't', 'o', 'p', 'i', 'c',
-		0x00, 0x00, 0x00, 0x01,
-		0x00, 0x00, 0x00, 0x05,
-		0x00, 0x01,
-		0x00, 0x00, 0x00, 0x00, 0x10, 0x10, 0x10, 0x10,
-		0x00, 0x00, 0x00, 0x1C,
-		// messageSet
-		0x00, 0x00, 0x00, 0x00, 0x00, 0x55, 0x00, 0x00,
-		0x00, 0x00, 0x00, 0x10,
-		// message
-		0x23, 0x96, 0x4a, 0xf7, // CRC
-		0x00,
-		0x00,
-		0xFF, 0xFF, 0xFF, 0xFF,
-		0x00, 0x00, 0x00, 0x02, 0x00, 0xEE}
-
-	oneRecordFetchResponse = []byte{
-		0x00, 0x00, 0x00, 0x00, // ThrottleTime
-		0x00, 0x00, 0x00, 0x01, // Number of Topics
-		0x00, 0x05, 't', 'o', 'p', 'i', 'c', // Topic
-		0x00, 0x00, 0x00, 0x01, // Number of Partitions
-		0x00, 0x00, 0x00, 0x05, // Partition
-		0x00, 0x01, // Error
-		0x00, 0x00, 0x00, 0x00, 0x10, 0x10, 0x10, 0x10, // High Watermark Offset
-		0x00, 0x00, 0x00, 0x00, 0x10, 0x10, 0x10, 0x10, // Last Stable Offset
-		0x00, 0x00, 0x00, 0x00, // Number of Aborted Transactions
-		0x00, 0x00, 0x00, 0x52, // Records length
-		// recordBatch
-		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-		0x00, 0x00, 0x00, 0x46,
-		0x00, 0x00, 0x00, 0x00,
-		0x02,
-		0xDB, 0x47, 0x14, 0xC9,
-		0x00, 0x00,
-		0x00, 0x00, 0x00, 0x00,
-		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0A,
-		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-		0x00, 0x00,
-		0x00, 0x00, 0x00, 0x00,
-		0x00, 0x00, 0x00, 0x01,
-		// record
-		0x28,
-		0x00,
-		0x0A,
-		0x00,
-		0x08, 0x01, 0x02, 0x03, 0x04,
-		0x06, 0x05, 0x06, 0x07,
-		0x02,
-		0x06, 0x08, 0x09, 0x0A,
-		0x04, 0x0B, 0x0C}
-
-	oneMessageFetchResponseV4 = []byte{
-		0x00, 0x00, 0x00, 0x00, // ThrottleTime
-		0x00, 0x00, 0x00, 0x01, // Number of Topics
-		0x00, 0x05, 't', 'o', 'p', 'i', 'c', // Topic
-		0x00, 0x00, 0x00, 0x01, // Number of Partitions
-		0x00, 0x00, 0x00, 0x05, // Partition
-		0x00, 0x01, // Error
-		0x00, 0x00, 0x00, 0x00, 0x10, 0x10, 0x10, 0x10, // High Watermark Offset
-		0x00, 0x00, 0x00, 0x00, 0x10, 0x10, 0x10, 0x10, // Last Stable Offset
-		0x00, 0x00, 0x00, 0x00, // Number of Aborted Transactions
-		0x00, 0x00, 0x00, 0x1C,
-		// messageSet
-		0x00, 0x00, 0x00, 0x00, 0x00, 0x55, 0x00, 0x00,
-		0x00, 0x00, 0x00, 0x10,
-		// message
-		0x23, 0x96, 0x4a, 0xf7, // CRC
-		0x00,
-		0x00,
-		0xFF, 0xFF, 0xFF, 0xFF,
-		0x00, 0x00, 0x00, 0x02, 0x00, 0xEE}
-)
-
-func TestEmptyFetchResponse(t *testing.T) {
-	response := FetchResponse{}
-	testVersionDecodable(t, "empty", &response, emptyFetchResponse, 0)
-
-	if len(response.Blocks) != 0 {
-		t.Error("Decoding produced topic blocks where there were none.")
-	}
-
-}
-
-func TestOneMessageFetchResponse(t *testing.T) {
-	response := FetchResponse{}
-	testVersionDecodable(t, "one message", &response, oneMessageFetchResponse, 0)
-
-	if len(response.Blocks) != 1 {
-		t.Fatal("Decoding produced incorrect number of topic blocks.")
-	}
-
-	if len(response.Blocks["topic"]) != 1 {
-		t.Fatal("Decoding produced incorrect number of partition blocks for topic.")
-	}
-
-	block := response.GetBlock("topic", 5)
-	if block == nil {
-		t.Fatal("GetBlock didn't return block.")
-	}
-	if block.Err != ErrOffsetOutOfRange {
-		t.Error("Decoding didn't produce correct error code.")
-	}
-	if block.HighWaterMarkOffset != 0x10101010 {
-		t.Error("Decoding didn't produce correct high water mark offset.")
-	}
-	partial, err := block.Records.isPartial()
-	if err != nil {
-		t.Fatalf("Unexpected error: %v", err)
-	}
-	if partial {
-		t.Error("Decoding detected a partial trailing message where there wasn't one.")
-	}
-
-	n, err := block.Records.numRecords()
-	if err != nil {
-		t.Fatalf("Unexpected error: %v", err)
-	}
-	if n != 1 {
-		t.Fatal("Decoding produced incorrect number of messages.")
-	}
-	msgBlock := block.Records.msgSet.Messages[0]
-	if msgBlock.Offset != 0x550000 {
-		t.Error("Decoding produced incorrect message offset.")
-	}
-	msg := msgBlock.Msg
-	if msg.Codec != CompressionNone {
-		t.Error("Decoding produced incorrect message compression.")
-	}
-	if msg.Key != nil {
-		t.Error("Decoding produced message key where there was none.")
-	}
-	if !bytes.Equal(msg.Value, []byte{0x00, 0xEE}) {
-		t.Error("Decoding produced incorrect message value.")
-	}
-}
-
-func TestOneRecordFetchResponse(t *testing.T) {
-	response := FetchResponse{}
-	testVersionDecodable(t, "one record", &response, oneRecordFetchResponse, 4)
-
-	if len(response.Blocks) != 1 {
-		t.Fatal("Decoding produced incorrect number of topic blocks.")
-	}
-
-	if len(response.Blocks["topic"]) != 1 {
-		t.Fatal("Decoding produced incorrect number of partition blocks for topic.")
-	}
-
-	block := response.GetBlock("topic", 5)
-	if block == nil {
-		t.Fatal("GetBlock didn't return block.")
-	}
-	if block.Err != ErrOffsetOutOfRange {
-		t.Error("Decoding didn't produce correct error code.")
-	}
-	if block.HighWaterMarkOffset != 0x10101010 {
-		t.Error("Decoding didn't produce correct high water mark offset.")
-	}
-	partial, err := block.Records.isPartial()
-	if err != nil {
-		t.Fatalf("Unexpected error: %v", err)
-	}
-	if partial {
-		t.Error("Decoding detected a partial trailing record where there wasn't one.")
-	}
-
-	n, err := block.Records.numRecords()
-	if err != nil {
-		t.Fatalf("Unexpected error: %v", err)
-	}
-	if n != 1 {
-		t.Fatal("Decoding produced incorrect number of records.")
-	}
-	rec := block.Records.recordBatch.Records[0]
-	if !bytes.Equal(rec.Key, []byte{0x01, 0x02, 0x03, 0x04}) {
-		t.Error("Decoding produced incorrect record key.")
-	}
-	if !bytes.Equal(rec.Value, []byte{0x05, 0x06, 0x07}) {
-		t.Error("Decoding produced incorrect record value.")
-	}
-}
-
-func TestOneMessageFetchResponseV4(t *testing.T) {
-	response := FetchResponse{}
-	testVersionDecodable(t, "one message v4", &response, oneMessageFetchResponseV4, 4)
-
-	if len(response.Blocks) != 1 {
-		t.Fatal("Decoding produced incorrect number of topic blocks.")
-	}
-
-	if len(response.Blocks["topic"]) != 1 {
-		t.Fatal("Decoding produced incorrect number of partition blocks for topic.")
-	}
-
-	block := response.GetBlock("topic", 5)
-	if block == nil {
-		t.Fatal("GetBlock didn't return block.")
-	}
-	if block.Err != ErrOffsetOutOfRange {
-		t.Error("Decoding didn't produce correct error code.")
-	}
-	if block.HighWaterMarkOffset != 0x10101010 {
-		t.Error("Decoding didn't produce correct high water mark offset.")
-	}
-	partial, err := block.Records.isPartial()
-	if err != nil {
-		t.Fatalf("Unexpected error: %v", err)
-	}
-	if partial {
-		t.Error("Decoding detected a partial trailing record where there wasn't one.")
-	}
-
-	n, err := block.Records.numRecords()
-	if err != nil {
-		t.Fatalf("Unexpected error: %v", err)
-	}
-	if n != 1 {
-		t.Fatal("Decoding produced incorrect number of records.")
-	}
-	msgBlock := block.Records.msgSet.Messages[0]
-	if msgBlock.Offset != 0x550000 {
-		t.Error("Decoding produced incorrect message offset.")
-	}
-	msg := msgBlock.Msg
-	if msg.Codec != CompressionNone {
-		t.Error("Decoding produced incorrect message compression.")
-	}
-	if msg.Key != nil {
-		t.Error("Decoding produced message key where there was none.")
-	}
-	if !bytes.Equal(msg.Value, []byte{0x00, 0xEE}) {
-		t.Error("Decoding produced incorrect message value.")
-	}
-}

vendor/github.com/Shopify/sarama/functional_client_test.go

@@ -1,90 +0,0 @@
-package sarama
-
-import (
-	"fmt"
-	"testing"
-	"time"
-)
-
-func TestFuncConnectionFailure(t *testing.T) {
-	setupFunctionalTest(t)
-	defer teardownFunctionalTest(t)
-
-	Proxies["kafka1"].Enabled = false
-	SaveProxy(t, "kafka1")
-
-	config := NewConfig()
-	config.Metadata.Retry.Max = 1
-
-	_, err := NewClient([]string{kafkaBrokers[0]}, config)
-	if err != ErrOutOfBrokers {
-		t.Fatal("Expected returned error to be ErrOutOfBrokers, but was: ", err)
-	}
-}
-
-func TestFuncClientMetadata(t *testing.T) {
-	setupFunctionalTest(t)
-	defer teardownFunctionalTest(t)
-
-	config := NewConfig()
-	config.Metadata.Retry.Max = 1
-	config.Metadata.Retry.Backoff = 10 * time.Millisecond
-	client, err := NewClient(kafkaBrokers, config)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	if err := client.RefreshMetadata("unknown_topic"); err != ErrUnknownTopicOrPartition {
-		t.Error("Expected ErrUnknownTopicOrPartition, got", err)
-	}
-
-	if _, err := client.Leader("unknown_topic", 0); err != ErrUnknownTopicOrPartition {
-		t.Error("Expected ErrUnknownTopicOrPartition, got", err)
-	}
-
-	if _, err := client.Replicas("invalid/topic", 0); err != ErrUnknownTopicOrPartition {
-		t.Error("Expected ErrUnknownTopicOrPartition, got", err)
-	}
-
-	partitions, err := client.Partitions("test.4")
-	if err != nil {
-		t.Error(err)
-	}
-	if len(partitions) != 4 {
-		t.Errorf("Expected test.4 topic to have 4 partitions, found %v", partitions)
-	}
-
-	partitions, err = client.Partitions("test.1")
-	if err != nil {
-		t.Error(err)
-	}
-	if len(partitions) != 1 {
-		t.Errorf("Expected test.1 topic to have 1 partitions, found %v", partitions)
-	}
-
-	safeClose(t, client)
-}
-
-func TestFuncClientCoordinator(t *testing.T) {
-	checkKafkaVersion(t, "0.8.2")
-	setupFunctionalTest(t)
-	defer teardownFunctionalTest(t)
-
-	client, err := NewClient(kafkaBrokers, nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	for i := 0; i < 10; i++ {
-		broker, err := client.Coordinator(fmt.Sprintf("another_new_consumer_group_%d", i))
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		if connected, err := broker.Connected(); !connected || err != nil {
-			t.Errorf("Expected to coordinator %s broker to be properly connected.", broker.Addr())
-		}
-	}
-
-	safeClose(t, client)
-}

vendor/github.com/Shopify/sarama/functional_consumer_test.go

@@ -1,61 +0,0 @@
-package sarama
-
-import (
-	"math"
-	"testing"
-)
-
-func TestFuncConsumerOffsetOutOfRange(t *testing.T) {
-	setupFunctionalTest(t)
-	defer teardownFunctionalTest(t)
-
-	consumer, err := NewConsumer(kafkaBrokers, nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	if _, err := consumer.ConsumePartition("test.1", 0, -10); err != ErrOffsetOutOfRange {
-		t.Error("Expected ErrOffsetOutOfRange, got:", err)
-	}
-
-	if _, err := consumer.ConsumePartition("test.1", 0, math.MaxInt64); err != ErrOffsetOutOfRange {
-		t.Error("Expected ErrOffsetOutOfRange, got:", err)
-	}
-
-	safeClose(t, consumer)
-}
-
-func TestConsumerHighWaterMarkOffset(t *testing.T) {
-	setupFunctionalTest(t)
-	defer teardownFunctionalTest(t)
-
-	p, err := NewSyncProducer(kafkaBrokers, nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer safeClose(t, p)
-
-	_, offset, err := p.SendMessage(&ProducerMessage{Topic: "test.1", Value: StringEncoder("Test")})
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	c, err := NewConsumer(kafkaBrokers, nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer safeClose(t, c)
-
-	pc, err := c.ConsumePartition("test.1", 0, OffsetOldest)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	<-pc.Messages()
-
-	if hwmo := pc.HighWaterMarkOffset(); hwmo != offset+1 {
-		t.Logf("Last produced offset %d; high water mark should be one higher but found %d.", offset, hwmo)
-	}
-
-	safeClose(t, pc)
-}

vendor/github.com/Shopify/sarama/functional_offset_manager_test.go

@@ -1,47 +0,0 @@
-package sarama
-
-import (
-	"testing"
-)
-
-func TestFuncOffsetManager(t *testing.T) {
-	checkKafkaVersion(t, "0.8.2")
-	setupFunctionalTest(t)
-	defer teardownFunctionalTest(t)
-
-	client, err := NewClient(kafkaBrokers, nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	offsetManager, err := NewOffsetManagerFromClient("sarama.TestFuncOffsetManager", client)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	pom1, err := offsetManager.ManagePartition("test.1", 0)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	pom1.MarkOffset(10, "test metadata")
-	safeClose(t, pom1)
-
-	pom2, err := offsetManager.ManagePartition("test.1", 0)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	offset, metadata := pom2.NextOffset()
-
-	if offset != 10 {
-		t.Errorf("Expected the next offset to be 10, found %d.", offset)
-	}
-	if metadata != "test metadata" {
-		t.Errorf("Expected metadata to be 'test metadata', found %s.", metadata)
-	}
-
-	safeClose(t, pom2)
-	safeClose(t, offsetManager)
-	safeClose(t, client)
-}

vendor/github.com/Shopify/sarama/functional_producer_test.go

@@ -1,323 +0,0 @@
-package sarama
-
-import (
-	"fmt"
-	"os"
-	"sync"
-	"testing"
-	"time"
-
-	toxiproxy "github.com/Shopify/toxiproxy/client"
-	"github.com/rcrowley/go-metrics"
-)
-
-const TestBatchSize = 1000
-
-func TestFuncProducing(t *testing.T) {
-	config := NewConfig()
-	testProducingMessages(t, config)
-}
-
-func TestFuncProducingGzip(t *testing.T) {
-	config := NewConfig()
-	config.Producer.Compression = CompressionGZIP
-	testProducingMessages(t, config)
-}
-
-func TestFuncProducingSnappy(t *testing.T) {
-	config := NewConfig()
-	config.Producer.Compression = CompressionSnappy
-	testProducingMessages(t, config)
-}
-
-func TestFuncProducingNoResponse(t *testing.T) {
-	config := NewConfig()
-	config.Producer.RequiredAcks = NoResponse
-	testProducingMessages(t, config)
-}
-
-func TestFuncProducingFlushing(t *testing.T) {
-	config := NewConfig()
-	config.Producer.Flush.Messages = TestBatchSize / 8
-	config.Producer.Flush.Frequency = 250 * time.Millisecond
-	testProducingMessages(t, config)
-}
-
-func TestFuncMultiPartitionProduce(t *testing.T) {
-	setupFunctionalTest(t)
-	defer teardownFunctionalTest(t)
-
-	config := NewConfig()
-	config.ChannelBufferSize = 20
-	config.Producer.Flush.Frequency = 50 * time.Millisecond
-	config.Producer.Flush.Messages = 200
-	config.Producer.Return.Successes = true
-	producer, err := NewSyncProducer(kafkaBrokers, config)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	var wg sync.WaitGroup
-	wg.Add(TestBatchSize)
-
-	for i := 1; i <= TestBatchSize; i++ {
-		go func(i int) {
-			defer wg.Done()
-			msg := &ProducerMessage{Topic: "test.64", Key: nil, Value: StringEncoder(fmt.Sprintf("hur %d", i))}
-			if _, _, err := producer.SendMessage(msg); err != nil {
-				t.Error(i, err)
-			}
-		}(i)
-	}
-
-	wg.Wait()
-	if err := producer.Close(); err != nil {
-		t.Error(err)
-	}
-}
-
-func TestFuncProducingToInvalidTopic(t *testing.T) {
-	setupFunctionalTest(t)
-	defer teardownFunctionalTest(t)
-
-	producer, err := NewSyncProducer(kafkaBrokers, nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	if _, _, err := producer.SendMessage(&ProducerMessage{Topic: "in/valid"}); err != ErrUnknownTopicOrPartition {
-		t.Error("Expected ErrUnknownTopicOrPartition, found", err)
-	}
-
-	if _, _, err := producer.SendMessage(&ProducerMessage{Topic: "in/valid"}); err != ErrUnknownTopicOrPartition {
-		t.Error("Expected ErrUnknownTopicOrPartition, found", err)
-	}
-
-	safeClose(t, producer)
-}
-
-func testProducingMessages(t *testing.T, config *Config) {
-	setupFunctionalTest(t)
-	defer teardownFunctionalTest(t)
-
-	// Configure some latency in order to properly validate the request latency metric
-	for _, proxy := range Proxies {
-		if _, err := proxy.AddToxic("", "latency", "", 1, toxiproxy.Attributes{"latency": 10}); err != nil {
-			t.Fatal("Unable to configure latency toxicity", err)
-		}
-	}
-
-	config.Producer.Return.Successes = true
-	config.Consumer.Return.Errors = true
-
-	client, err := NewClient(kafkaBrokers, config)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// Keep in mind the current offset
-	initialOffset, err := client.GetOffset("test.1", 0, OffsetNewest)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	producer, err := NewAsyncProducerFromClient(client)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	expectedResponses := TestBatchSize
-	for i := 1; i <= TestBatchSize; {
-		msg := &ProducerMessage{Topic: "test.1", Key: nil, Value: StringEncoder(fmt.Sprintf("testing %d", i))}
-		select {
-		case producer.Input() <- msg:
-			i++
-		case ret := <-producer.Errors():
-			t.Fatal(ret.Err)
-		case <-producer.Successes():
-			expectedResponses--
-		}
-	}
-	for expectedResponses > 0 {
-		select {
-		case ret := <-producer.Errors():
-			t.Fatal(ret.Err)
-		case <-producer.Successes():
-			expectedResponses--
-		}
-	}
-	safeClose(t, producer)
-
-	// Validate producer metrics before using the consumer minus the offset request
-	validateMetrics(t, client)
-
-	master, err := NewConsumerFromClient(client)
-	if err != nil {
-		t.Fatal(err)
-	}
-	consumer, err := master.ConsumePartition("test.1", 0, initialOffset)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	for i := 1; i <= TestBatchSize; i++ {
-		select {
-		case <-time.After(10 * time.Second):
-			t.Fatal("Not received any more events in the last 10 seconds.")
-
-		case err := <-consumer.Errors():
-			t.Error(err)
-
-		case message := <-consumer.Messages():
-			if string(message.Value) != fmt.Sprintf("testing %d", i) {
-				t.Fatalf("Unexpected message with index %d: %s", i, message.Value)
-			}
-		}
-
-	}
-	safeClose(t, consumer)
-	safeClose(t, client)
-}
-
-func validateMetrics(t *testing.T, client Client) {
-	// Get the broker used by test1 topic
-	var broker *Broker
-	if partitions, err := client.Partitions("test.1"); err != nil {
-		t.Error(err)
-	} else {
-		for _, partition := range partitions {
-			if b, err := client.Leader("test.1", partition); err != nil {
-				t.Error(err)
-			} else {
-				if broker != nil && b != broker {
-					t.Fatal("Expected only one broker, got at least 2")
-				}
-				broker = b
-			}
-		}
-	}
-
-	metricValidators := newMetricValidators()
-	noResponse := client.Config().Producer.RequiredAcks == NoResponse
-	compressionEnabled := client.Config().Producer.Compression != CompressionNone
-
-	// We are adding 10ms of latency to all requests with toxiproxy
-	minRequestLatencyInMs := 10
-	if noResponse {
-		// but when we do not wait for a response it can be less than 1ms
-		minRequestLatencyInMs = 0
-	}
-
-	// We read at least 1 byte from the broker
-	metricValidators.registerForAllBrokers(broker, minCountMeterValidator("incoming-byte-rate", 1))
-	// in at least 3 global requests (1 for metadata request, 1 for offset request and N for produce request)
-	metricValidators.register(minCountMeterValidator("request-rate", 3))
-	metricValidators.register(minCountHistogramValidator("request-size", 3))
-	metricValidators.register(minValHistogramValidator("request-size", 1))
-	metricValidators.register(minValHistogramValidator("request-latency-in-ms", minRequestLatencyInMs))
-	// and at least 2 requests to the registered broker (offset + produces)
-	metricValidators.registerForBroker(broker, minCountMeterValidator("request-rate", 2))
-	metricValidators.registerForBroker(broker, minCountHistogramValidator("request-size", 2))
-	metricValidators.registerForBroker(broker, minValHistogramValidator("request-size", 1))
-	metricValidators.registerForBroker(broker, minValHistogramValidator("request-latency-in-ms", minRequestLatencyInMs))
-
-	// We send at least 1 batch
-	metricValidators.registerForGlobalAndTopic("test_1", minCountHistogramValidator("batch-size", 1))
-	metricValidators.registerForGlobalAndTopic("test_1", minValHistogramValidator("batch-size", 1))
-	if compressionEnabled {
-		// We record compression ratios between [0.50,-10.00] (50-1000 with a histogram) for at least one "fake" record
-		metricValidators.registerForGlobalAndTopic("test_1", minCountHistogramValidator("compression-ratio", 1))
-		metricValidators.registerForGlobalAndTopic("test_1", minValHistogramValidator("compression-ratio", 50))
-		metricValidators.registerForGlobalAndTopic("test_1", maxValHistogramValidator("compression-ratio", 1000))
-	} else {
-		// We record compression ratios of 1.00 (100 with a histogram) for every TestBatchSize record
-		metricValidators.registerForGlobalAndTopic("test_1", countHistogramValidator("compression-ratio", TestBatchSize))
-		metricValidators.registerForGlobalAndTopic("test_1", minValHistogramValidator("compression-ratio", 100))
-		metricValidators.registerForGlobalAndTopic("test_1", maxValHistogramValidator("compression-ratio", 100))
-	}
-
-	// We send exactly TestBatchSize messages
-	metricValidators.registerForGlobalAndTopic("test_1", countMeterValidator("record-send-rate", TestBatchSize))
-	// We send at least one record per request
-	metricValidators.registerForGlobalAndTopic("test_1", minCountHistogramValidator("records-per-request", 1))
-	metricValidators.registerForGlobalAndTopic("test_1", minValHistogramValidator("records-per-request", 1))
-
-	// We receive at least 1 byte from the broker
-	metricValidators.registerForAllBrokers(broker, minCountMeterValidator("outgoing-byte-rate", 1))
-	if noResponse {
-		// in exactly 2 global responses (metadata + offset)
-		metricValidators.register(countMeterValidator("response-rate", 2))
-		metricValidators.register(minCountHistogramValidator("response-size", 2))
-		metricValidators.register(minValHistogramValidator("response-size", 1))
-		// and exactly 1 offset response for the registered broker
-		metricValidators.registerForBroker(broker, countMeterValidator("response-rate", 1))
-		metricValidators.registerForBroker(broker, minCountHistogramValidator("response-size", 1))
-		metricValidators.registerForBroker(broker, minValHistogramValidator("response-size", 1))
-	} else {
-		// in at least 3 global responses (metadata + offset + produces)
-		metricValidators.register(minCountMeterValidator("response-rate", 3))
-		metricValidators.register(minCountHistogramValidator("response-size", 3))
-		metricValidators.register(minValHistogramValidator("response-size", 1))
-		// and at least 2 for the registered broker
-		metricValidators.registerForBroker(broker, minCountMeterValidator("response-rate", 2))
-		metricValidators.registerForBroker(broker, minCountHistogramValidator("response-size", 2))
-		metricValidators.registerForBroker(broker, minValHistogramValidator("response-size", 1))
-	}
-
-	// Run the validators
-	metricValidators.run(t, client.Config().MetricRegistry)
-}
-
-// Benchmarks
-
-func BenchmarkProducerSmall(b *testing.B) {
-	benchmarkProducer(b, nil, "test.64", ByteEncoder(make([]byte, 128)))
-}
-func BenchmarkProducerMedium(b *testing.B) {
-	benchmarkProducer(b, nil, "test.64", ByteEncoder(make([]byte, 1024)))
-}
-func BenchmarkProducerLarge(b *testing.B) {
-	benchmarkProducer(b, nil, "test.64", ByteEncoder(make([]byte, 8192)))
-}
-func BenchmarkProducerSmallSinglePartition(b *testing.B) {
-	benchmarkProducer(b, nil, "test.1", ByteEncoder(make([]byte, 128)))
-}
-func BenchmarkProducerMediumSnappy(b *testing.B) {
-	conf := NewConfig()
-	conf.Producer.Compression = CompressionSnappy
-	benchmarkProducer(b, conf, "test.1", ByteEncoder(make([]byte, 1024)))
-}
-
-func benchmarkProducer(b *testing.B, conf *Config, topic string, value Encoder) {
-	setupFunctionalTest(b)
-	defer teardownFunctionalTest(b)
-
-	metricsDisable := os.Getenv("METRICS_DISABLE")
-	if metricsDisable != "" {
-		previousUseNilMetrics := metrics.UseNilMetrics
-		Logger.Println("Disabling metrics using no-op implementation")
-		metrics.UseNilMetrics = true
-		// Restore previous setting
-		defer func() {
-			metrics.UseNilMetrics = previousUseNilMetrics
-		}()
-	}
-
-	producer, err := NewAsyncProducer(kafkaBrokers, conf)
-	if err != nil {
-		b.Fatal(err)
-	}
-
-	b.ResetTimer()
-
-	for i := 1; i <= b.N; {
-		msg := &ProducerMessage{Topic: topic, Key: StringEncoder(fmt.Sprintf("%d", i)), Value: value}
-		select {
-		case producer.Input() <- msg:
-			i++
-		case ret := <-producer.Errors():
-			b.Fatal(ret.Err)
-		}
-	}
-	safeClose(b, producer)
-}

vendor/github.com/Shopify/sarama/functional_test.go

@@ -1,148 +0,0 @@
-package sarama
-
-import (
-	"log"
-	"math/rand"
-	"net"
-	"os"
-	"strconv"
-	"strings"
-	"testing"
-	"time"
-
-	toxiproxy "github.com/Shopify/toxiproxy/client"
-)
-
-const (
-	VagrantToxiproxy      = "http://192.168.100.67:8474"
-	VagrantKafkaPeers     = "192.168.100.67:9091,192.168.100.67:9092,192.168.100.67:9093,192.168.100.67:9094,192.168.100.67:9095"
-	VagrantZookeeperPeers = "192.168.100.67:2181,192.168.100.67:2182,192.168.100.67:2183,192.168.100.67:2184,192.168.100.67:2185"
-)
-
-var (
-	kafkaAvailable, kafkaRequired bool
-	kafkaBrokers                  []string
-
-	proxyClient  *toxiproxy.Client
-	Proxies      map[string]*toxiproxy.Proxy
-	ZKProxies    = []string{"zk1", "zk2", "zk3", "zk4", "zk5"}
-	KafkaProxies = []string{"kafka1", "kafka2", "kafka3", "kafka4", "kafka5"}
-)
-
-func init() {
-	if os.Getenv("DEBUG") == "true" {
-		Logger = log.New(os.Stdout, "[sarama] ", log.LstdFlags)
-	}
-
-	seed := time.Now().UTC().UnixNano()
-	if tmp := os.Getenv("TEST_SEED"); tmp != "" {
-		seed, _ = strconv.ParseInt(tmp, 0, 64)
-	}
-	Logger.Println("Using random seed:", seed)
-	rand.Seed(seed)
-
-	proxyAddr := os.Getenv("TOXIPROXY_ADDR")
-	if proxyAddr == "" {
-		proxyAddr = VagrantToxiproxy
-	}
-	proxyClient = toxiproxy.NewClient(proxyAddr)
-
-	kafkaPeers := os.Getenv("KAFKA_PEERS")
-	if kafkaPeers == "" {
-		kafkaPeers = VagrantKafkaPeers
-	}
-	kafkaBrokers = strings.Split(kafkaPeers, ",")
-
-	if c, err := net.DialTimeout("tcp", kafkaBrokers[0], 5*time.Second); err == nil {
-		if err = c.Close(); err == nil {
-			kafkaAvailable = true
-		}
-	}
-
-	kafkaRequired = os.Getenv("CI") != ""
-}
-
-func checkKafkaAvailability(t testing.TB) {
-	if !kafkaAvailable {
-		if kafkaRequired {
-			t.Fatalf("Kafka broker is not available on %s. Set KAFKA_PEERS to connect to Kafka on a different location.", kafkaBrokers[0])
-		} else {
-			t.Skipf("Kafka broker is not available on %s. Set KAFKA_PEERS to connect to Kafka on a different location.", kafkaBrokers[0])
-		}
-	}
-}
-
-func checkKafkaVersion(t testing.TB, requiredVersion string) {
-	kafkaVersion := os.Getenv("KAFKA_VERSION")
-	if kafkaVersion == "" {
-		t.Logf("No KAFKA_VERSION set. This test requires Kafka version %s or higher. Continuing...", requiredVersion)
-	} else {
-		available := parseKafkaVersion(kafkaVersion)
-		required := parseKafkaVersion(requiredVersion)
-		if !available.satisfies(required) {
-			t.Skipf("Kafka version %s is required for this test; you have %s. Skipping...", requiredVersion, kafkaVersion)
-		}
-	}
-}
-
-func resetProxies(t testing.TB) {
-	if err := proxyClient.ResetState(); err != nil {
-		t.Error(err)
-	}
-	Proxies = nil
-}
-
-func fetchProxies(t testing.TB) {
-	var err error
-	Proxies, err = proxyClient.Proxies()
-	if err != nil {
-		t.Fatal(err)
-	}
-}
-
-func SaveProxy(t *testing.T, px string) {
-	if err := Proxies[px].Save(); err != nil {
-		t.Fatal(err)
-	}
-}
-
-func setupFunctionalTest(t testing.TB) {
-	checkKafkaAvailability(t)
-	resetProxies(t)
-	fetchProxies(t)
-}
-
-func teardownFunctionalTest(t testing.TB) {
-	resetProxies(t)
-}
-
-type kafkaVersion []int
-
-func (kv kafkaVersion) satisfies(other kafkaVersion) bool {
-	var ov int
-	for index, v := range kv {
-		if len(other) <= index {
-			ov = 0
-		} else {
-			ov = other[index]
-		}
-
-		if v < ov {
-			return false
-		} else if v > ov {
-			return true
-		}
-	}
-	return true
-}
-
-func parseKafkaVersion(version string) kafkaVersion {
-	numbers := strings.Split(version, ".")
-	result := make(kafkaVersion, 0, len(numbers))
-	for _, number := range numbers {
-		nr, _ := strconv.Atoi(number)
-		result = append(result, nr)
-	}
-
-	return result
-}

vendor/github.com/Shopify/sarama/heartbeat_request.go

@@ -1,47 +0,0 @@
-package sarama
-
-type HeartbeatRequest struct {
-	GroupId      string
-	GenerationId int32
-	MemberId     string
-}
-
-func (r *HeartbeatRequest) encode(pe packetEncoder) error {
-	if err := pe.putString(r.GroupId); err != nil {
-		return err
-	}
-
-	pe.putInt32(r.GenerationId)
-
-	if err := pe.putString(r.MemberId); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func (r *HeartbeatRequest) decode(pd packetDecoder, version int16) (err error) {
-	if r.GroupId, err = pd.getString(); err != nil {
-		return
-	}
-	if r.GenerationId, err = pd.getInt32(); err != nil {
-		return
-	}
-	if r.MemberId, err = pd.getString(); err != nil {
-		return
-	}
-
-	return nil
-}
-
-func (r *HeartbeatRequest) key() int16 {
-	return 12
-}
-
-func (r *HeartbeatRequest) version() int16 {
-	return 0
-}
-
-func (r *HeartbeatRequest) requiredVersion() KafkaVersion {
-	return V0_9_0_0
-}

vendor/github.com/Shopify/sarama/heartbeat_request_test.go

@@ -1,21 +0,0 @@
-package sarama
-
-import "testing"
-
-var (
-	basicHeartbeatRequest = []byte{
-		0, 3, 'f', 'o', 'o', // Group ID
-		0x00, 0x01, 0x02, 0x03, // Generatiuon ID
-		0, 3, 'b', 'a', 'z', // Member ID
-	}
-)
-
-func TestHeartbeatRequest(t *testing.T) {
-	var request *HeartbeatRequest
-
-	request = new(HeartbeatRequest)
-	request.GroupId = "foo"
-	request.GenerationId = 66051
-	request.MemberId = "baz"
-	testRequest(t, "basic", request, basicHeartbeatRequest)
-}

vendor/github.com/Shopify/sarama/heartbeat_response.go

@@ -1,32 +0,0 @@
-package sarama
-
-type HeartbeatResponse struct {
-	Err KError
-}
-
-func (r *HeartbeatResponse) encode(pe packetEncoder) error {
-	pe.putInt16(int16(r.Err))
-	return nil
-}
-
-func (r *HeartbeatResponse) decode(pd packetDecoder, version int16) error {
-	kerr, err := pd.getInt16()
-	if err != nil {
-		return err
-	}
-	r.Err = KError(kerr)
-
-	return nil
-}
-
-func (r *HeartbeatResponse) key() int16 {
-	return 12
-}
-
-func (r *HeartbeatResponse) version() int16 {
-	return 0
-}
-
-func (r *HeartbeatResponse) requiredVersion() KafkaVersion {
-	return V0_9_0_0
-}

vendor/github.com/Shopify/sarama/heartbeat_response_test.go

@@ -1,18 +0,0 @@
-package sarama
-
-import "testing"
-
-var (
-	heartbeatResponseNoError = []byte{
-		0x00, 0x00}
-)
-
-func TestHeartbeatResponse(t *testing.T) {
-	var response *HeartbeatResponse
-
-	response = new(HeartbeatResponse)
-	testVersionDecodable(t, "no error", response, heartbeatResponseNoError, 0)
-	if response.Err != ErrNoError {
-		t.Error("Decoding error failed: no error expected but found", response.Err)
-	}
-}

vendor/github.com/Shopify/sarama/join_group_request.go

@@ -1,143 +0,0 @@
-package sarama
-
-type GroupProtocol struct {
-	Name     string
-	Metadata []byte
-}
-
-func (p *GroupProtocol) decode(pd packetDecoder) (err error) {
-	p.Name, err = pd.getString()
-	if err != nil {
-		return err
-	}
-	p.Metadata, err = pd.getBytes()
-	return err
-}
-
-func (p *GroupProtocol) encode(pe packetEncoder) (err error) {
-	if err := pe.putString(p.Name); err != nil {
-		return err
-	}
-	if err := pe.putBytes(p.Metadata); err != nil {
-		return err
-	}
-	return nil
-}
-
-type JoinGroupRequest struct {
-	GroupId               string
-	SessionTimeout        int32
-	MemberId              string
-	ProtocolType          string
-	GroupProtocols        map[string][]byte // deprecated; use OrderedGroupProtocols
-	OrderedGroupProtocols []*GroupProtocol
-}
-
-func (r *JoinGroupRequest) encode(pe packetEncoder) error {
-	if err := pe.putString(r.GroupId); err != nil {
-		return err
-	}
-	pe.putInt32(r.SessionTimeout)
-	if err := pe.putString(r.MemberId); err != nil {
-		return err
-	}
-	if err := pe.putString(r.ProtocolType); err != nil {
-		return err
-	}
-
-	if len(r.GroupProtocols) > 0 {
-		if len(r.OrderedGroupProtocols) > 0 {
-			return PacketDecodingError{"cannot specify both GroupProtocols and OrderedGroupProtocols on JoinGroupRequest"}
-		}
-
-		if err := pe.putArrayLength(len(r.GroupProtocols)); err != nil {
-			return err
-		}
-		for name, metadata := range r.GroupProtocols {
-			if err := pe.putString(name); err != nil {
-				return err
-			}
-			if err := pe.putBytes(metadata); err != nil {
-				return err
-			}
-		}
-	} else {
-		if err := pe.putArrayLength(len(r.OrderedGroupProtocols)); err != nil {
-			return err
-		}
-		for _, protocol := range r.OrderedGroupProtocols {
-			if err := protocol.encode(pe); err != nil {
-				return err
-			}
-		}
-	}
-
-	return nil
-}
-
-func (r *JoinGroupRequest) decode(pd packetDecoder, version int16) (err error) {
-	if r.GroupId, err = pd.getString(); err != nil {
-		return
-	}
-
-	if r.SessionTimeout, err = pd.getInt32(); err != nil {
-		return
-	}
-
-	if r.MemberId, err = pd.getString(); err != nil {
-		return
-	}
-
-	if r.ProtocolType, err = pd.getString(); err != nil {
-		return
-	}
-
-	n, err := pd.getArrayLength()
-	if err != nil {
-		return err
-	}
-	if n == 0 {
-		return nil
-	}
-
-	r.GroupProtocols = make(map[string][]byte)
-	for i := 0; i < n; i++ {
-		protocol := &GroupProtocol{}
-		if err := protocol.decode(pd); err != nil {
-			return err
-		}
-		r.GroupProtocols[protocol.Name] = protocol.Metadata
-		r.OrderedGroupProtocols = append(r.OrderedGroupProtocols, protocol)
-	}
-
-	return nil
-}
-
-func (r *JoinGroupRequest) key() int16 {
-	return 11
-}
-
-func (r *JoinGroupRequest) version() int16 {
-	return 0
-}
-
-func (r *JoinGroupRequest) requiredVersion() KafkaVersion {
-	return V0_9_0_0
-}
-
-func (r *JoinGroupRequest) AddGroupProtocol(name string, metadata []byte) {
-	r.OrderedGroupProtocols = append(r.OrderedGroupProtocols, &GroupProtocol{
-		Name:     name,
-		Metadata: metadata,
-	})
-}
-
-func (r *JoinGroupRequest) AddGroupProtocolMetadata(name string, metadata *ConsumerGroupMemberMetadata) error {
-	bin, err := encode(metadata, nil)
-	if err != nil {
-		return err
-	}
-
-	r.AddGroupProtocol(name, bin)
-	return nil
-}

vendor/github.com/Shopify/sarama/join_group_request_test.go

@@ -1,57 +0,0 @@
-package sarama
-
-import "testing"
-
-var (
-	joinGroupRequestNoProtocols = []byte{
-		0, 9, 'T', 'e', 's', 't', 'G', 'r', 'o', 'u', 'p', // Group ID
-		0, 0, 0, 100, // Session timeout
-		0, 0, // Member ID
-		0, 8, 'c', 'o', 'n', 's', 'u', 'm', 'e', 'r', // Protocol Type
-		0, 0, 0, 0, // 0 protocol groups
-	}
-
-	joinGroupRequestOneProtocol = []byte{
-		0, 9, 'T', 'e', 's', 't', 'G', 'r', 'o', 'u', 'p', // Group ID
-		0, 0, 0, 100, // Session timeout
-		0, 11, 'O', 'n', 'e', 'P', 'r', 'o', 't', 'o', 'c', 'o', 'l', // Member ID
-		0, 8, 'c', 'o', 'n', 's', 'u', 'm', 'e', 'r', // Protocol Type
-		0, 0, 0, 1, // 1 group protocol
-		0, 3, 'o', 'n', 'e', // Protocol name
-		0, 0, 0, 3, 0x01, 0x02, 0x03, // protocol metadata
-	}
-)
-
-func TestJoinGroupRequest(t *testing.T) {
-	request := new(JoinGroupRequest)
-	request.GroupId = "TestGroup"
-	request.SessionTimeout = 100
-	request.ProtocolType = "consumer"
-	testRequest(t, "no protocols", request, joinGroupRequestNoProtocols)
-}
-
-func TestJoinGroupRequestOneProtocol(t *testing.T) {
-	request := new(JoinGroupRequest)
-	request.GroupId = "TestGroup"
-	request.SessionTimeout = 100
-	request.MemberId = "OneProtocol"
-	request.ProtocolType = "consumer"
-	request.AddGroupProtocol("one", []byte{0x01, 0x02, 0x03})
-	packet := testRequestEncode(t, "one protocol", request, joinGroupRequestOneProtocol)
-	request.GroupProtocols = make(map[string][]byte)
-	request.GroupProtocols["one"] = []byte{0x01, 0x02, 0x03}
-	testRequestDecode(t, "one protocol", request, packet)
-}
-
-func TestJoinGroupRequestDeprecatedEncode(t *testing.T) {
-	request := new(JoinGroupRequest)
-	request.GroupId = "TestGroup"
-	request.SessionTimeout = 100
-	request.MemberId = "OneProtocol"
-	request.ProtocolType = "consumer"
-	request.GroupProtocols = make(map[string][]byte)
-	request.GroupProtocols["one"] = []byte{0x01, 0x02, 0x03}
-	packet := testRequestEncode(t, "one protocol", request, joinGroupRequestOneProtocol)
-	request.AddGroupProtocol("one", []byte{0x01, 0x02, 0x03})
-	testRequestDecode(t, "one protocol", request, packet)
-}

vendor/github.com/Shopify/sarama/join_group_response.go

@@ -1,115 +0,0 @@
-package sarama
-
-type JoinGroupResponse struct {
-	Err           KError
-	GenerationId  int32
-	GroupProtocol string
-	LeaderId      string
-	MemberId      string
-	Members       map[string][]byte
-}
-
-func (r *JoinGroupResponse) GetMembers() (map[string]ConsumerGroupMemberMetadata, error) {
-	members := make(map[string]ConsumerGroupMemberMetadata, len(r.Members))
-	for id, bin := range r.Members {
-		meta := new(ConsumerGroupMemberMetadata)
-		if err := decode(bin, meta); err != nil {
-			return nil, err
-		}
-		members[id] = *meta
-	}
-	return members, nil
-}
-
-func (r *JoinGroupResponse) encode(pe packetEncoder) error {
-	pe.putInt16(int16(r.Err))
-	pe.putInt32(r.GenerationId)
-
-	if err := pe.putString(r.GroupProtocol); err != nil {
-		return err
-	}
-	if err := pe.putString(r.LeaderId); err != nil {
-		return err
-	}
-	if err := pe.putString(r.MemberId); err != nil {
-		return err
-	}
-
-	if err := pe.putArrayLength(len(r.Members)); err != nil {
-		return err
-	}
-
-	for memberId, memberMetadata := range r.Members {
-		if err := pe.putString(memberId); err != nil {
-			return err
-		}
-
-		if err := pe.putBytes(memberMetadata); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (r *JoinGroupResponse) decode(pd packetDecoder, version int16) (err error) {
-	kerr, err := pd.getInt16()
-	if err != nil {
-		return err
-	}
-
-	r.Err = KError(kerr)
-
-	if r.GenerationId, err = pd.getInt32(); err != nil {
-		return
-	}
-
-	if r.GroupProtocol, err = pd.getString(); err != nil {
-		return
-	}
-
-	if r.LeaderId, err = pd.getString(); err != nil {
-		return
-	}
-
-	if r.MemberId, err = pd.getString(); err != nil {
-		return
-	}
-
-	n, err := pd.getArrayLength()
-	if err != nil {
-		return err
-	}
-	if n == 0 {
-		return nil
-	}
-
-	r.Members = make(map[string][]byte)
-	for i := 0; i < n; i++ {
-		memberId, err := pd.getString()
-		if err != nil {
-			return err
-		}
-
-		memberMetadata, err := pd.getBytes()
-		if err != nil {
-			return err
-		}
-
-		r.Members[memberId] = memberMetadata
-	}
-
-	return nil
-}
-
-func (r *JoinGroupResponse) key() int16 {
-	return 11
-}
-
-func (r *JoinGroupResponse) version() int16 {
-	return 0
-}
-
-func (r *JoinGroupResponse) requiredVersion() KafkaVersion {
-	return V0_9_0_0
-}

vendor/github.com/Shopify/sarama/join_group_response_test.go

@@ -1,98 +0,0 @@
-package sarama
-
-import (
-	"reflect"
-	"testing"
-)
-
-var (
-	joinGroupResponseNoError = []byte{
-		0x00, 0x00, // No error
-		0x00, 0x01, 0x02, 0x03, // Generation ID
-		0, 8, 'p', 'r', 'o', 't', 'o', 'c', 'o', 'l', // Protocol name chosen
-		0, 3, 'f', 'o', 'o', // Leader ID
-		0, 3, 'b', 'a', 'r', // Member ID
-		0, 0, 0, 0, // No member info
-	}
-
-	joinGroupResponseWithError = []byte{
-		0, 23, // Error: inconsistent group protocol
-		0x00, 0x00, 0x00, 0x00, // Generation ID
-		0, 0, // Protocol name chosen
-		0, 0, // Leader ID
-		0, 0, // Member ID
-		0, 0, 0, 0, // No member info
-	}
-
-	joinGroupResponseLeader = []byte{
-		0x00, 0x00, // No error
-		0x00, 0x01, 0x02, 0x03, // Generation ID
-		0, 8, 'p', 'r', 'o', 't', 'o', 'c', 'o', 'l', // Protocol name chosen
-		0, 3, 'f', 'o', 'o', // Leader ID
-		0, 3, 'f', 'o', 'o', // Member ID == Leader ID
-		0, 0, 0, 1, // 1 member
-		0, 3, 'f', 'o', 'o', // Member ID
-		0, 0, 0, 3, 0x01, 0x02, 0x03, // Member metadata
-	}
-)
-
-func TestJoinGroupResponse(t *testing.T) {
-	var response *JoinGroupResponse
-
-	response = new(JoinGroupResponse)
-	testVersionDecodable(t, "no error", response, joinGroupResponseNoError, 0)
-	if response.Err != ErrNoError {
-		t.Error("Decoding Err failed: no error expected but found", response.Err)
-	}
-	if response.GenerationId != 66051 {
-		t.Error("Decoding GenerationId failed, found:", response.GenerationId)
-	}
-	if response.LeaderId != "foo" {
-		t.Error("Decoding LeaderId failed, found:", response.LeaderId)
-	}
-	if response.MemberId != "bar" {
-		t.Error("Decoding MemberId failed, found:", response.MemberId)
-	}
-	if len(response.Members) != 0 {
-		t.Error("Decoding Members failed, found:", response.Members)
-	}
-
-	response = new(JoinGroupResponse)
-	testVersionDecodable(t, "with error", response, joinGroupResponseWithError, 0)
-	if response.Err != ErrInconsistentGroupProtocol {
-		t.Error("Decoding Err failed: ErrInconsistentGroupProtocol expected but found", response.Err)
-	}
-	if response.GenerationId != 0 {
-		t.Error("Decoding GenerationId failed, found:", response.GenerationId)
-	}
-	if response.LeaderId != "" {
-		t.Error("Decoding LeaderId failed, found:", response.LeaderId)
-	}
-	if response.MemberId != "" {
-		t.Error("Decoding MemberId failed, found:", response.MemberId)
-	}
-	if len(response.Members) != 0 {
-		t.Error("Decoding Members failed, found:", response.Members)
-	}
-
-	response = new(JoinGroupResponse)
-	testVersionDecodable(t, "with error", response, joinGroupResponseLeader, 0)
-	if response.Err != ErrNoError {
-		t.Error("Decoding Err failed: ErrNoError expected but found", response.Err)
-	}
-	if response.GenerationId != 66051 {
-		t.Error("Decoding GenerationId failed, found:", response.GenerationId)
-	}
-	if response.LeaderId != "foo" {
-		t.Error("Decoding LeaderId failed, found:", response.LeaderId)
-	}
-	if response.MemberId != "foo" {
-		t.Error("Decoding MemberId failed, found:", response.MemberId)
-	}
-	if len(response.Members) != 1 {
-		t.Error("Decoding Members failed, found:", response.Members)
-	}
-	if !reflect.DeepEqual(response.Members["foo"], []byte{0x01, 0x02, 0x03}) {
-		t.Error("Decoding foo member failed, found:", response.Members["foo"])
-	}
-}

vendor/github.com/Shopify/sarama/leave_group_request.go

@@ -1,40 +0,0 @@
-package sarama
-
-type LeaveGroupRequest struct {
-	GroupId  string
-	MemberId string
-}
-
-func (r *LeaveGroupRequest) encode(pe packetEncoder) error {
-	if err := pe.putString(r.GroupId); err != nil {
-		return err
-	}
-	if err := pe.putString(r.MemberId); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func (r *LeaveGroupRequest) decode(pd packetDecoder, version int16) (err error) {
-	if r.GroupId, err = pd.getString(); err != nil {
-		return
-	}
-	if r.MemberId, err = pd.getString(); err != nil {
-		return
-	}
-
-	return nil
-}
-
-func (r *LeaveGroupRequest) key() int16 {
-	return 13
-}
-
-func (r *LeaveGroupRequest) version() int16 {
-	return 0
-}
-
-func (r *LeaveGroupRequest) requiredVersion() KafkaVersion {
-	return V0_9_0_0
-}

vendor/github.com/Shopify/sarama/leave_group_request_test.go

@@ -1,19 +0,0 @@
-package sarama
-
-import "testing"
-
-var (
-	basicLeaveGroupRequest = []byte{
-		0, 3, 'f', 'o', 'o',
-		0, 3, 'b', 'a', 'r',
-	}
-)
-
-func TestLeaveGroupRequest(t *testing.T) {
-	var request *LeaveGroupRequest
-
-	request = new(LeaveGroupRequest)
-	request.GroupId = "foo"
-	request.MemberId = "bar"
-	testRequest(t, "basic", request, basicLeaveGroupRequest)
-}

vendor/github.com/Shopify/sarama/leave_group_response.go

@@ -1,32 +0,0 @@
-package sarama
-
-type LeaveGroupResponse struct {
-	Err KError
-}
-
-func (r *LeaveGroupResponse) encode(pe packetEncoder) error {
-	pe.putInt16(int16(r.Err))
-	return nil
-}
-
-func (r *LeaveGroupResponse) decode(pd packetDecoder, version int16) (err error) {
-	kerr, err := pd.getInt16()
-	if err != nil {
-		return err
-	}
-	r.Err = KError(kerr)
-
-	return nil
-}
-
-func (r *LeaveGroupResponse) key() int16 {
-	return 13
-}
-
-func (r *LeaveGroupResponse) version() int16 {
-	return 0
-}
-
-func (r *LeaveGroupResponse) requiredVersion() KafkaVersion {
-	return V0_9_0_0
-}

vendor/github.com/Shopify/sarama/leave_group_response_test.go

@@ -1,24 +0,0 @@
-package sarama
-
-import "testing"
-
-var (
-	leaveGroupResponseNoError   = []byte{0x00, 0x00}
-	leaveGroupResponseWithError = []byte{0, 25}
-)
-
-func TestLeaveGroupResponse(t *testing.T) {
-	var response *LeaveGroupResponse
-
-	response = new(LeaveGroupResponse)
-	testVersionDecodable(t, "no error", response, leaveGroupResponseNoError, 0)
-	if response.Err != ErrNoError {
-		t.Error("Decoding error failed: no error expected but found", response.Err)
-	}
-
-	response = new(LeaveGroupResponse)
-	testVersionDecodable(t, "with error", response, leaveGroupResponseWithError, 0)
-	if response.Err != ErrUnknownMemberId {
-		t.Error("Decoding error failed: ErrUnknownMemberId expected but found", response.Err)
-	}
-}

vendor/github.com/Shopify/sarama/length_field.go

@@ -1,69 +0,0 @@
-package sarama
-
-import "encoding/binary"
-
-// LengthField implements the PushEncoder and PushDecoder interfaces for calculating 4-byte lengths.
-type lengthField struct {
-	startOffset int
-}
-
-func (l *lengthField) saveOffset(in int) {
-	l.startOffset = in
-}
-
-func (l *lengthField) reserveLength() int {
-	return 4
-}
-
-func (l *lengthField) run(curOffset int, buf []byte) error {
-	binary.BigEndian.PutUint32(buf[l.startOffset:], uint32(curOffset-l.startOffset-4))
-	return nil
-}
-
-func (l *lengthField) check(curOffset int, buf []byte) error {
-	if uint32(curOffset-l.startOffset-4) != binary.BigEndian.Uint32(buf[l.startOffset:]) {
-		return PacketDecodingError{"length field invalid"}
-	}
-
-	return nil
-}
-
-type varintLengthField struct {
-	startOffset int
-	length      int64
-}
-
-func (l *varintLengthField) decode(pd packetDecoder) error {
-	var err error
-	l.length, err = pd.getVarint()
-	return err
-}
-
-func (l *varintLengthField) saveOffset(in int) {
-	l.startOffset = in
-}
-
-func (l *varintLengthField) adjustLength(currOffset int) int {
-	oldFieldSize := l.reserveLength()
-	l.length = int64(currOffset - l.startOffset - oldFieldSize)
-
-	return l.reserveLength() - oldFieldSize
-}
-
-func (l *varintLengthField) reserveLength() int {
-	var tmp [binary.MaxVarintLen64]byte
-	return binary.PutVarint(tmp[:], l.length)
-}
-
-func (l *varintLengthField) run(curOffset int, buf []byte) error {
-	binary.PutVarint(buf[l.startOffset:], l.length)
-	return nil
-}
-
-func (l *varintLengthField) check(curOffset int, buf []byte) error {
-	if int64(curOffset-l.startOffset-l.reserveLength()) != l.length {
-		return PacketDecodingError{"length field invalid"}
-	}
-
-	return nil
-}

vendor/github.com/Shopify/sarama/list_groups_request.go

@@ -1,24 +0,0 @@
-package sarama
-
-type ListGroupsRequest struct {
-}
-
-func (r *ListGroupsRequest) encode(pe packetEncoder) error {
-	return nil
-}
-
-func (r *ListGroupsRequest) decode(pd packetDecoder, version int16) (err error) {
-	return nil
-}
-
-func (r *ListGroupsRequest) key() int16 {
-	return 16
-}
-
-func (r *ListGroupsRequest) version() int16 {
-	return 0
-}
-
-func (r *ListGroupsRequest) requiredVersion() KafkaVersion {
-	return V0_9_0_0
-}

vendor/github.com/Shopify/sarama/list_groups_request_test.go

@@ -1,7 +0,0 @@
-package sarama
-
-import "testing"
-
-func TestListGroupsRequest(t *testing.T) {
-	testRequest(t, "ListGroupsRequest", &ListGroupsRequest{}, []byte{})
-}

vendor/github.com/Shopify/sarama/list_groups_response.go

@@ -1,69 +0,0 @@
-package sarama
-
-type ListGroupsResponse struct {
-	Err    KError
-	Groups map[string]string
-}
-
-func (r *ListGroupsResponse) encode(pe packetEncoder) error {
-	pe.putInt16(int16(r.Err))
-
-	if err := pe.putArrayLength(len(r.Groups)); err != nil {
-		return err
-	}
-	for groupId, protocolType := range r.Groups {
-		if err := pe.putString(groupId); err != nil {
-			return err
-		}
-		if err := pe.putString(protocolType); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (r *ListGroupsResponse) decode(pd packetDecoder, version int16) error {
-	kerr, err := pd.getInt16()
-	if err != nil {
-		return err
-	}
-
-	r.Err = KError(kerr)
-
-	n, err := pd.getArrayLength()
-	if err != nil {
-		return err
-	}
-	if n == 0 {
-		return nil
-	}
-
-	r.Groups = make(map[string]string)
-	for i := 0; i < n; i++ {
-		groupId, err := pd.getString()
-		if err != nil {
-			return err
-		}
-		protocolType, err := pd.getString()
-		if err != nil {
-			return err
-		}
-
-		r.Groups[groupId] = protocolType
-	}
-
-	return nil
-}
-
-func (r *ListGroupsResponse) key() int16 {
-	return 16
-}
-
-func (r *ListGroupsResponse) version() int16 {
-	return 0
-}
-
-func (r *ListGroupsResponse) requiredVersion() KafkaVersion {
-	return V0_9_0_0
-}