adding ctr metrics to rpcs (#1538)

Adding container create metric to grpc adding metrics for ctr preparation duration, creation duration, init start, using atomic.load and store to avoid concurrent thread faults adding atomic store for stats generated in runhot function excluding / path in webserver, removing ochttp for adminserver adding spandata in runner client to handle new proto data
2022-10-28 21:29:17 +03:00 · 2019-08-08 15:04:42 -07:00
parent 5fc7731b17
commit 025b1aefab
10 changed files with 297 additions and 209 deletions
--- a/api/agent/agent.go
+++ b/api/agent/agent.go
@@ -12,6 +12,7 @@ import (
 	"path/filepath"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"time"

 	"github.com/fnproject/fn/api/agent/drivers"
@@ -797,6 +798,8 @@ func (a *agent) runHot(ctx context.Context, caller slotCaller, call *call, tok R
 	var cookie drivers.Cookie
 	var err error

+	ctrCreatePrepStart := time.Now()
+
 	id := id.New().String()
 	logger := logrus.WithFields(logrus.Fields{"container_id": id, "app_id": call.AppID, "fn_id": call.FnID, "image": call.Image, "memory": call.Memory, "cpus": call.CPUs, "idle_timeout": call.IdleTimeout})
 	ctx, cancel := context.WithCancel(common.WithLogger(ctx, logger))
@@ -881,6 +884,7 @@ func (a *agent) runHot(ctx context.Context, caller slotCaller, call *call, tok R
 		return
 	}
 	needsPull, err := cookie.ValidateImage(ctx)
+	atomic.StoreInt64(&call.ctrPrepTime, int64(time.Since(ctrCreatePrepStart)/time.Millisecond))
 	if needsPull {
 		waitStart := time.Now()
 		pullCtx, pullCancel := context.WithTimeout(ctx, a.cfg.HotPullTimeout)
@@ -896,12 +900,13 @@ func (a *agent) runHot(ctx context.Context, caller slotCaller, call *call, tok R
 				err = models.ErrCallTimeoutServerBusy
 			}
 		}
-		call.AddDockerWaitTime(time.Since(waitStart))
+		atomic.StoreInt64(&call.imagePullWaitTime, int64(time.Since(waitStart)/time.Millisecond))
 	}
 	if tryQueueErr(err, errQueue) != nil {
 		return
 	}

+	ctrCreateStart := time.Now()
 	err = cookie.CreateContainer(ctx)
 	if tryQueueErr(err, errQueue) != nil {
 		return
@@ -911,6 +916,7 @@ func (a *agent) runHot(ctx context.Context, caller slotCaller, call *call, tok R
 	if tryQueueErr(err, errQueue) != nil {
 		return
 	}
+	atomic.StoreInt64(&call.ctrCreateTime, int64(time.Since(ctrCreateStart)/time.Millisecond))

 	childDone = make(chan struct{})

@@ -930,15 +936,23 @@ func (a *agent) runHot(ctx context.Context, caller slotCaller, call *call, tok R
 		// INIT BARRIER HERE. Wait for the initialization go-routine signal
 		select {
 		case <-initialized:
-			statsContainerUDSInitLatency(ctx, initStart, time.Now(), "initialized")
+			initTime := time.Now() // Declaring this prior to keep the stats in sync
+			statsContainerUDSInitLatency(ctx, initStart, initTime, "initialized")
+			atomic.StoreInt64(&call.initStartTime, int64(initTime.Sub(initStart)/time.Millisecond))
 		case <-a.shutWg.Closer(): // agent shutdown
-			statsContainerUDSInitLatency(ctx, initStart, time.Now(), "canceled")
+			closerTime := time.Now()
+			statsContainerUDSInitLatency(ctx, initStart, closerTime, "canceled")
+			atomic.StoreInt64(&call.initStartTime, int64(closerTime.Sub(initStart)/time.Millisecond))
 			return
 		case <-ctx.Done():
-			statsContainerUDSInitLatency(ctx, initStart, time.Now(), "canceled")
+			ctxCancelTime := time.Now()
+			statsContainerUDSInitLatency(ctx, initStart, ctxCancelTime, "canceled")
+			atomic.StoreInt64(&call.initStartTime, int64(ctxCancelTime.Sub(initStart)/time.Millisecond))
 			return
 		case <-timer.C:
-			statsContainerUDSInitLatency(ctx, initStart, time.Now(), "timedout")
+			timeoutTime := time.Now()
+			statsContainerUDSInitLatency(ctx, initStart, timeoutTime, "timedout")
+			atomic.StoreInt64(&call.initStartTime, int64(timeoutTime.Sub(initStart)/time.Millisecond))
 			tryQueueErr(models.ErrContainerInitTimeout, errQueue)
 			return
 		}
--- a/api/agent/call.go
+++ b/api/agent/call.go
@@ -295,7 +295,16 @@ type call struct {
 	userExecTime *time.Duration

 	// amount of time attributed to docker-pull wait
-	dockerWaitTime *time.Duration
+	imagePullWaitTime int64
+
+	// amount of time attributed to container preparation
+	ctrPrepTime int64
+
+	// amount of time attributed to container creation
+	ctrCreateTime int64
+
+	// Init wait start timestamp for goroutine in runHot
+	initStartTime int64

 	// LB & Pure Runner Extra Config
 	extensions map[string]string
@@ -352,8 +361,6 @@ func (c *call) Start(ctx context.Context) error {

 	// Check context timeouts, errors
 	if ctx.Err() != nil {
-		span.SetStatus(trace.Status{Code: int32(trace.StatusCodeUnknown),
-			Message: ctx.Err().Error()})
 		return ctx.Err()
 	}

@@ -373,12 +380,8 @@ func (c *call) End(ctx context.Context, errIn error) error {
 	case nil:
 		c.Status = "success"
 	case context.DeadlineExceeded:
-		span.SetStatus(trace.Status{Code: int32(trace.StatusCodeDeadlineExceeded),
-			Message: "context deadline exceeded"})
 		c.Status = "timeout"
 	default:
-		span.SetStatus(trace.Status{Code: int32(trace.StatusCodeUnknown),
-			Message: errIn.Error()})
 		c.Status = "error"
 		c.Error = errIn.Error()
 	}
@@ -390,8 +393,6 @@ func (c *call) End(ctx context.Context, errIn error) error {
 	c.stderr.Close()

 	if err := c.ct.fireAfterCall(ctx, c.Model()); err != nil {
-		span.SetStatus(trace.Status{Code: int32(trace.StatusCodeUnknown),
-			Message: err.Error()})
 		return err
 	}
 	return errIn // original error, important for use in sync call returns
@@ -417,14 +418,3 @@ func GetCallLatencies(c *call) (time.Duration, time.Duration) {
 	}
 	return schedDuration, execDuration
 }
-
-func (c *call) AddDockerWaitTime(dur time.Duration) {
-	if c.dockerWaitTime == nil {
-		c.dockerWaitTime = new(time.Duration)
-	}
-	*c.dockerWaitTime += dur
-	// We expose this on the upstream models.Call also.
-	// CallListeners have access to the latter, but not the internals of the agent, so any
-	// reporting or bean-counting that's going on from there will need access to this.
-	c.Model().DockerWaitDuration = *c.dockerWaitTime
-}
--- a/api/agent/grpc/runner.pb.go
+++ b/api/agent/grpc/runner.pb.go
@@ -320,21 +320,24 @@ func (*CallResultStart) XXX_OneofWrappers() []interface{} {

 // Call has really finished, it might have completed or crashed
 type CallFinished struct {
-	Success              bool     `protobuf:"varint,1,opt,name=success,proto3" json:"success,omitempty"`
-	Details              string   `protobuf:"bytes,2,opt,name=details,proto3" json:"details,omitempty"`
-	ErrorCode            int32    `protobuf:"varint,3,opt,name=errorCode,proto3" json:"errorCode,omitempty"`
-	ErrorStr             string   `protobuf:"bytes,4,opt,name=errorStr,proto3" json:"errorStr,omitempty"`
-	CreatedAt            string   `protobuf:"bytes,5,opt,name=createdAt,proto3" json:"createdAt,omitempty"`
-	StartedAt            string   `protobuf:"bytes,6,opt,name=startedAt,proto3" json:"startedAt,omitempty"`
-	CompletedAt          string   `protobuf:"bytes,7,opt,name=completedAt,proto3" json:"completedAt,omitempty"`
-	SchedulerDuration    int64    `protobuf:"varint,8,opt,name=schedulerDuration,proto3" json:"schedulerDuration,omitempty"`
-	ExecutionDuration    int64    `protobuf:"varint,9,opt,name=executionDuration,proto3" json:"executionDuration,omitempty"`
-	ErrorUser            bool     `protobuf:"varint,10,opt,name=errorUser,proto3" json:"errorUser,omitempty"`
-	Image                string   `protobuf:"bytes,11,opt,name=image,proto3" json:"image,omitempty"`
-	DockerWaitDuration   int64    `protobuf:"varint,12,opt,name=dockerWaitDuration,proto3" json:"dockerWaitDuration,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+	Success               bool     `protobuf:"varint,1,opt,name=success,proto3" json:"success,omitempty"`
+	Details               string   `protobuf:"bytes,2,opt,name=details,proto3" json:"details,omitempty"`
+	ErrorCode             int32    `protobuf:"varint,3,opt,name=errorCode,proto3" json:"errorCode,omitempty"`
+	ErrorStr              string   `protobuf:"bytes,4,opt,name=errorStr,proto3" json:"errorStr,omitempty"`
+	CreatedAt             string   `protobuf:"bytes,5,opt,name=createdAt,proto3" json:"createdAt,omitempty"`
+	StartedAt             string   `protobuf:"bytes,6,opt,name=startedAt,proto3" json:"startedAt,omitempty"`
+	CompletedAt           string   `protobuf:"bytes,7,opt,name=completedAt,proto3" json:"completedAt,omitempty"`
+	SchedulerDuration     int64    `protobuf:"varint,8,opt,name=schedulerDuration,proto3" json:"schedulerDuration,omitempty"`
+	ExecutionDuration     int64    `protobuf:"varint,9,opt,name=executionDuration,proto3" json:"executionDuration,omitempty"`
+	ErrorUser             bool     `protobuf:"varint,10,opt,name=errorUser,proto3" json:"errorUser,omitempty"`
+	Image                 string   `protobuf:"bytes,11,opt,name=image,proto3" json:"image,omitempty"`
+	ImagePullWaitDuration int64    `protobuf:"varint,12,opt,name=imagePullWaitDuration,proto3" json:"imagePullWaitDuration,omitempty"`
+	CtrPrepDuration       int64    `protobuf:"varint,13,opt,name=ctrPrepDuration,proto3" json:"ctrPrepDuration,omitempty"`
+	CtrCreateDuration     int64    `protobuf:"varint,14,opt,name=ctrCreateDuration,proto3" json:"ctrCreateDuration,omitempty"`
+	InitStartTime         int64    `protobuf:"varint,15,opt,name=initStartTime,proto3" json:"initStartTime,omitempty"`
+	XXX_NoUnkeyedLiteral  struct{} `json:"-"`
+	XXX_unrecognized      []byte   `json:"-"`
+	XXX_sizecache         int32    `json:"-"`
 }

 func (m *CallFinished) Reset()         { *m = CallFinished{} }
@@ -439,9 +442,30 @@ func (m *CallFinished) GetImage() string {
 	return ""
 }

-func (m *CallFinished) GetDockerWaitDuration() int64 {
+func (m *CallFinished) GetImagePullWaitDuration() int64 {
 	if m != nil {
-		return m.DockerWaitDuration
+		return m.ImagePullWaitDuration
+	}
+	return 0
+}
+
+func (m *CallFinished) GetCtrPrepDuration() int64 {
+	if m != nil {
+		return m.CtrPrepDuration
+	}
+	return 0
+}
+
+func (m *CallFinished) GetCtrCreateDuration() int64 {
+	if m != nil {
+		return m.CtrCreateDuration
+	}
+	return 0
+}
+
+func (m *CallFinished) GetInitStartTime() int64 {
+	if m != nil {
+		return m.InitStartTime
 	}
 	return 0
 }
@@ -622,26 +646,30 @@ func (*RunnerMsg) XXX_OneofWrappers() []interface{} {
 }

 type RunnerStatus struct {
-	Active               int32             `protobuf:"varint,2,opt,name=active,proto3" json:"active,omitempty"`
-	Failed               bool              `protobuf:"varint,3,opt,name=failed,proto3" json:"failed,omitempty"`
-	Id                   string            `protobuf:"bytes,4,opt,name=id,proto3" json:"id,omitempty"`
-	Details              string            `protobuf:"bytes,5,opt,name=details,proto3" json:"details,omitempty"`
-	ErrorCode            int32             `protobuf:"varint,6,opt,name=errorCode,proto3" json:"errorCode,omitempty"`
-	ErrorStr             string            `protobuf:"bytes,7,opt,name=errorStr,proto3" json:"errorStr,omitempty"`
-	CreatedAt            string            `protobuf:"bytes,8,opt,name=createdAt,proto3" json:"createdAt,omitempty"`
-	StartedAt            string            `protobuf:"bytes,9,opt,name=startedAt,proto3" json:"startedAt,omitempty"`
-	CompletedAt          string            `protobuf:"bytes,10,opt,name=completedAt,proto3" json:"completedAt,omitempty"`
-	Cached               bool              `protobuf:"varint,11,opt,name=cached,proto3" json:"cached,omitempty"`
-	RequestsReceived     uint64            `protobuf:"varint,12,opt,name=requestsReceived,proto3" json:"requestsReceived,omitempty"`
-	RequestsHandled      uint64            `protobuf:"varint,13,opt,name=requestsHandled,proto3" json:"requestsHandled,omitempty"`
-	KdumpsOnDisk         uint64            `protobuf:"varint,14,opt,name=kdumpsOnDisk,proto3" json:"kdumpsOnDisk,omitempty"`
-	SchedulerDuration    int64             `protobuf:"varint,15,opt,name=schedulerDuration,proto3" json:"schedulerDuration,omitempty"`
-	ExecutionDuration    int64             `protobuf:"varint,16,opt,name=executionDuration,proto3" json:"executionDuration,omitempty"`
-	IsNetworkDisabled    bool              `protobuf:"varint,17,opt,name=isNetworkDisabled,proto3" json:"isNetworkDisabled,omitempty"`
-	CustomStatus         map[string]string `protobuf:"bytes,18,rep,name=customStatus,proto3" json:"customStatus,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
-	XXX_NoUnkeyedLiteral struct{}          `json:"-"`
-	XXX_unrecognized     []byte            `json:"-"`
-	XXX_sizecache        int32             `json:"-"`
+	Active                int32             `protobuf:"varint,2,opt,name=active,proto3" json:"active,omitempty"`
+	Failed                bool              `protobuf:"varint,3,opt,name=failed,proto3" json:"failed,omitempty"`
+	Id                    string            `protobuf:"bytes,4,opt,name=id,proto3" json:"id,omitempty"`
+	Details               string            `protobuf:"bytes,5,opt,name=details,proto3" json:"details,omitempty"`
+	ErrorCode             int32             `protobuf:"varint,6,opt,name=errorCode,proto3" json:"errorCode,omitempty"`
+	ErrorStr              string            `protobuf:"bytes,7,opt,name=errorStr,proto3" json:"errorStr,omitempty"`
+	CreatedAt             string            `protobuf:"bytes,8,opt,name=createdAt,proto3" json:"createdAt,omitempty"`
+	StartedAt             string            `protobuf:"bytes,9,opt,name=startedAt,proto3" json:"startedAt,omitempty"`
+	CompletedAt           string            `protobuf:"bytes,10,opt,name=completedAt,proto3" json:"completedAt,omitempty"`
+	Cached                bool              `protobuf:"varint,11,opt,name=cached,proto3" json:"cached,omitempty"`
+	RequestsReceived      uint64            `protobuf:"varint,12,opt,name=requestsReceived,proto3" json:"requestsReceived,omitempty"`
+	RequestsHandled       uint64            `protobuf:"varint,13,opt,name=requestsHandled,proto3" json:"requestsHandled,omitempty"`
+	KdumpsOnDisk          uint64            `protobuf:"varint,14,opt,name=kdumpsOnDisk,proto3" json:"kdumpsOnDisk,omitempty"`
+	SchedulerDuration     int64             `protobuf:"varint,15,opt,name=schedulerDuration,proto3" json:"schedulerDuration,omitempty"`
+	ExecutionDuration     int64             `protobuf:"varint,16,opt,name=executionDuration,proto3" json:"executionDuration,omitempty"`
+	IsNetworkDisabled     bool              `protobuf:"varint,17,opt,name=isNetworkDisabled,proto3" json:"isNetworkDisabled,omitempty"`
+	CustomStatus          map[string]string `protobuf:"bytes,18,rep,name=customStatus,proto3" json:"customStatus,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
+	ImagePullWaitDuration int64             `protobuf:"varint,19,opt,name=imagePullWaitDuration,proto3" json:"imagePullWaitDuration,omitempty"`
+	CtrPrepDuration       int64             `protobuf:"varint,20,opt,name=ctrPrepDuration,proto3" json:"ctrPrepDuration,omitempty"`
+	CtrCreateDuration     int64             `protobuf:"varint,21,opt,name=ctrCreateDuration,proto3" json:"ctrCreateDuration,omitempty"`
+	InitStartTime         int64             `protobuf:"varint,22,opt,name=initStartTime,proto3" json:"initStartTime,omitempty"`
+	XXX_NoUnkeyedLiteral  struct{}          `json:"-"`
+	XXX_unrecognized      []byte            `json:"-"`
+	XXX_sizecache         int32             `json:"-"`
 }

 func (m *RunnerStatus) Reset()         { *m = RunnerStatus{} }
@@ -788,6 +816,34 @@ func (m *RunnerStatus) GetCustomStatus() map[string]string {
 	return nil
 }

+func (m *RunnerStatus) GetImagePullWaitDuration() int64 {
+	if m != nil {
+		return m.ImagePullWaitDuration
+	}
+	return 0
+}
+
+func (m *RunnerStatus) GetCtrPrepDuration() int64 {
+	if m != nil {
+		return m.CtrPrepDuration
+	}
+	return 0
+}
+
+func (m *RunnerStatus) GetCtrCreateDuration() int64 {
+	if m != nil {
+		return m.CtrCreateDuration
+	}
+	return 0
+}
+
+func (m *RunnerStatus) GetInitStartTime() int64 {
+	if m != nil {
+		return m.InitStartTime
+	}
+	return 0
+}
+
 type ConfigMsg struct {
 	Config               map[string]string `protobuf:"bytes,1,rep,name=config,proto3" json:"config,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
 	XXX_NoUnkeyedLiteral struct{}          `json:"-"`
@@ -1296,84 +1352,89 @@ func init() {
 func init() { proto.RegisterFile("runner.proto", fileDescriptor_48eceea7e2abc593) }

 var fileDescriptor_48eceea7e2abc593 = []byte{
-	// 1231 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0x5d, 0x6f, 0x1b, 0x45,
-	0x17, 0xce, 0x7a, 0xed, 0xb5, 0x7d, 0xec, 0x38, 0xce, 0xa8, 0xaa, 0xf6, 0xdd, 0xb7, 0xa2, 0xc6,
-	0x94, 0xca, 0x82, 0x74, 0x4a, 0x03, 0x95, 0x4a, 0x25, 0x40, 0x25, 0x49, 0xe5, 0xa2, 0x96, 0xa2,
-	0x71, 0x0b, 0x97, 0xd1, 0x64, 0x77, 0x62, 0x2f, 0x5e, 0xef, 0x98, 0x99, 0xd9, 0xd0, 0x48, 0xdc,
-	0xf3, 0x1b, 0xe0, 0x0e, 0xee, 0xb8, 0xe7, 0x47, 0x70, 0xc5, 0xcf, 0xe1, 0x1a, 0xcd, 0x87, 0xd7,
-	0x1f, 0x69, 0xd3, 0x56, 0xe2, 0x6e, 0xcf, 0xf3, 0x9c, 0x99, 0x73, 0xf6, 0xcc, 0x73, 0xce, 0x0c,
-	0xb4, 0x45, 0x91, 0xe7, 0x4c, 0xe0, 0xb9, 0xe0, 0x8a, 0x47, 0xff, 0x1f, 0x73, 0x3e, 0xce, 0xd8,
-	0x6d, 0x63, 0x9d, 0x14, 0xa7, 0xb7, 0xd9, 0x6c, 0xae, 0xce, 0x2d, 0xd9, 0xff, 0xdb, 0x83, 0xfa,
-	0x33, 0x71, 0x7e, 0x40, 0xb3, 0x0c, 0x0d, 0xa0, 0x3b, 0xe3, 0x09, 0xcb, 0xe4, 0x71, 0x4c, 0xb3,
-	0xec, 0xf8, 0x7b, 0xc9, 0xf3, 0xd0, 0xeb, 0x79, 0x83, 0x26, 0xe9, 0x58, 0x5c, 0x7b, 0x7d, 0x25,
-	0x79, 0x8e, 0x7a, 0xd0, 0x96, 0x19, 0x57, 0xc7, 0x13, 0x2a, 0x27, 0xc7, 0x69, 0x12, 0x56, 0x8c,
-	0x17, 0x68, 0x6c, 0x48, 0xe5, 0xe4, 0x51, 0x82, 0xee, 0x01, 0xb0, 0x17, 0x8a, 0xe5, 0x32, 0xe5,
-	0xb9, 0x0c, 0xfd, 0x9e, 0x3f, 0x68, 0xed, 0x87, 0xd8, 0x45, 0xc2, 0x47, 0x25, 0x75, 0x94, 0x2b,
-	0x71, 0x4e, 0x56, 0x7c, 0xa3, 0xcf, 0x60, 0x67, 0x83, 0x46, 0x5d, 0xf0, 0xa7, 0xec, 0xdc, 0xe5,
-	0xa2, 0x3f, 0xd1, 0x15, 0xa8, 0x9d, 0xd1, 0xac, 0x60, 0x2e, 0xb2, 0x35, 0xee, 0x57, 0xee, 0x79,
-	0xfd, 0x3b, 0xd0, 0x3c, 0xa4, 0x8a, 0x3e, 0x14, 0x74, 0xc6, 0x10, 0x82, 0x6a, 0x42, 0x15, 0x35,
-	0x2b, 0xdb, 0xc4, 0x7c, 0xeb, 0xcd, 0x18, 0x3f, 0x35, 0x0b, 0x1b, 0x44, 0x7f, 0xf6, 0x3f, 0x01,
-	0x18, 0x2a, 0x35, 0x1f, 0x32, 0x9a, 0x30, 0xf1, 0xa6, 0xc1, 0xfa, 0xdf, 0x42, 0x5b, 0xaf, 0x22,
-	0x4c, 0xce, 0x9f, 0x30, 0x45, 0xd1, 0x75, 0x68, 0x49, 0x45, 0x55, 0x21, 0x8f, 0x63, 0x9e, 0x30,
-	0xb3, 0xbe, 0x46, 0xc0, 0x42, 0x07, 0x3c, 0x61, 0xe8, 0x7d, 0xa8, 0x4f, 0x4c, 0x08, 0x19, 0x56,
-	0x4c, 0x3d, 0x5a, 0x78, 0x19, 0x96, 0x2c, 0xb8, 0xfe, 0xe7, 0xb0, 0xa3, 0x6b, 0x44, 0x98, 0x2c,
-	0x32, 0x35, 0x52, 0x54, 0x28, 0xf4, 0x1e, 0x54, 0x27, 0x4a, 0xcd, 0xc3, 0xa4, 0xe7, 0x0d, 0x5a,
-	0xfb, 0xdb, 0x78, 0x35, 0xee, 0x70, 0x8b, 0x18, 0xf2, 0xcb, 0x00, 0xaa, 0x33, 0xa6, 0x68, 0xff,
-	0x57, 0x1f, 0xda, 0x7a, 0x83, 0x87, 0x69, 0x9e, 0xca, 0x09, 0x4b, 0x50, 0x08, 0x75, 0x59, 0xc4,
-	0x31, 0x93, 0xd2, 0x24, 0xd5, 0x20, 0x0b, 0x53, 0x33, 0x09, 0x53, 0x34, 0xcd, 0xa4, 0xfb, 0xb5,
-	0x85, 0x89, 0xae, 0x41, 0x93, 0x09, 0xc1, 0x85, 0x4e, 0x3c, 0xf4, 0xcd, 0xaf, 0x2c, 0x01, 0x14,
-	0x41, 0xc3, 0x18, 0x23, 0x25, 0xc2, 0xaa, 0x59, 0x58, 0xda, 0x7a, 0x65, 0x2c, 0x18, 0x55, 0x2c,
-	0x79, 0xa0, 0xc2, 0x9a, 0x21, 0x97, 0x80, 0x66, 0xa5, 0xfe, 0x25, 0xc3, 0x06, 0x96, 0x2d, 0x01,
-	0xd4, 0x83, 0x56, 0xcc, 0x67, 0xf3, 0x8c, 0x59, 0xbe, 0x6e, 0xf8, 0x55, 0x08, 0xed, 0xc1, 0xae,
-	0x8c, 0x27, 0x2c, 0x29, 0x32, 0x26, 0x0e, 0x0b, 0x41, 0x55, 0xca, 0xf3, 0xb0, 0xd1, 0xf3, 0x06,
-	0x3e, 0xb9, 0x48, 0x68, 0x6f, 0xf6, 0x82, 0xc5, 0x85, 0x36, 0x4a, 0xef, 0xa6, 0xf5, 0xbe, 0x40,
-	0x94, 0xff, 0xfc, 0x5c, 0x32, 0x11, 0x82, 0xa9, 0xd4, 0x12, 0xd0, 0x22, 0x48, 0x67, 0x74, 0xcc,
-	0xc2, 0x96, 0x15, 0x81, 0x31, 0x10, 0x06, 0x94, 0xf0, 0x78, 0xca, 0xc4, 0x77, 0x34, 0x55, 0x65,
-	0x88, 0xb6, 0x09, 0xf1, 0x12, 0xa6, 0x3f, 0x82, 0xe6, 0x41, 0x96, 0xb2, 0x5c, 0x3d, 0x91, 0x63,
-	0x74, 0x0d, 0x7c, 0x25, 0xac, 0xd2, 0x5a, 0xfb, 0x8d, 0x45, 0x73, 0x0c, 0xb7, 0x88, 0x86, 0x51,
-	0xcf, 0x69, 0xb7, 0x62, 0x68, 0xc0, 0xa5, 0xaa, 0xf5, 0x89, 0x6b, 0x46, 0x9f, 0xf8, 0x09, 0x4f,
-	0xce, 0xfb, 0xbf, 0x78, 0xd0, 0x24, 0xa6, 0xe3, 0xf5, 0xae, 0x77, 0xa1, 0x2d, 0x8c, 0x76, 0x8e,
-	0x4d, 0x61, 0xdd, 0xf6, 0x5d, 0xbc, 0x21, 0xaa, 0xe1, 0x16, 0x69, 0x89, 0x15, 0x8d, 0xbd, 0x36,
-	0x1c, 0xfa, 0x10, 0x1a, 0xa7, 0x4e, 0x53, 0x46, 0x12, 0x5a, 0x89, 0xab, 0x42, 0x1b, 0x6e, 0x91,
-	0xd2, 0xa1, 0xcc, 0xed, 0xf7, 0x1a, 0xb4, 0x6d, 0x6e, 0x23, 0xd3, 0x09, 0xe8, 0x2a, 0x04, 0x34,
-	0x56, 0xe9, 0x99, 0xed, 0xa6, 0x1a, 0x71, 0x96, 0xc6, 0x4f, 0x69, 0x9a, 0xb9, 0xbd, 0x1b, 0xc4,
-	0x59, 0xa8, 0x03, 0x95, 0x34, 0x71, 0x2a, 0xab, 0xa4, 0xc9, 0xaa, 0x66, 0x6b, 0x97, 0x68, 0x36,
-	0xb8, 0x4c, 0xb3, 0xf5, 0xcb, 0x34, 0xdb, 0xb8, 0x54, 0xb3, 0xcd, 0xd7, 0x68, 0x16, 0x2e, 0x6a,
-	0xf6, 0x2a, 0x04, 0x31, 0xd5, 0xda, 0x34, 0xd2, 0x69, 0x10, 0x67, 0xa1, 0x0f, 0xa0, 0x2b, 0xd8,
-	0x0f, 0x05, 0x93, 0x4a, 0x12, 0x16, 0xb3, 0xf4, 0x8c, 0x25, 0x46, 0x39, 0x55, 0x72, 0x01, 0x47,
-	0x03, 0xd8, 0x59, 0x60, 0x43, 0x9a, 0x27, 0xba, 0x4c, 0xdb, 0xc6, 0x75, 0x13, 0x46, 0x7d, 0x68,
-	0x4f, 0x93, 0x62, 0x36, 0x97, 0x4f, 0xf3, 0xc3, 0x54, 0x4e, 0xc3, 0x8e, 0x71, 0x5b, 0xc3, 0x5e,
-	0xde, 0x45, 0x3b, 0x6f, 0xd5, 0x45, 0xdd, 0x57, 0x75, 0xd1, 0x1e, 0xec, 0xa6, 0xf2, 0x6b, 0xa6,
-	0x7e, 0xe4, 0x62, 0x7a, 0x98, 0x4a, 0x7a, 0xa2, 0x73, 0xdd, 0x35, 0x3f, 0x7e, 0x91, 0x40, 0x07,
-	0xd0, 0x8e, 0x0b, 0xa9, 0xf8, 0xcc, 0xaa, 0x23, 0x44, 0x66, 0x30, 0x5e, 0xc7, 0xab, 0x92, 0xc1,
-	0x07, 0x2b, 0x1e, 0xf6, 0xbe, 0x58, 0x5b, 0x14, 0x7d, 0x01, 0xbb, 0x17, 0x5c, 0xde, 0xea, 0xce,
-	0x38, 0x83, 0xe6, 0x01, 0xcf, 0x4f, 0xd3, 0xb1, 0xee, 0x1f, 0x0c, 0x41, 0x6c, 0x8c, 0xd0, 0x33,
-	0xc9, 0x5c, 0xc5, 0x25, 0xe7, 0xbe, 0x6c, 0x0e, 0xce, 0x2b, 0xfa, 0x14, 0x5a, 0x2b, 0xf0, 0x5b,
-	0xc5, 0xed, 0x40, 0xdb, 0x2e, 0xb5, 0x89, 0xf7, 0xff, 0xa8, 0xc0, 0xf6, 0x63, 0x3e, 0x26, 0xf6,
-	0x48, 0x75, 0x32, 0x7b, 0x50, 0x5b, 0xed, 0xe2, 0x2b, 0x78, 0x8d, 0xc6, 0x8b, 0x4e, 0xb6, 0x4e,
-	0xe8, 0x26, 0xf8, 0x34, 0x9e, 0xba, 0x16, 0x46, 0x1b, 0xbe, 0x0f, 0xe2, 0xa9, 0x1e, 0x2d, 0x34,
-	0xd6, 0xe7, 0x5f, 0x13, 0x8c, 0x26, 0xe7, 0xae, 0x8d, 0x37, 0x77, 0x25, 0x9a, 0xd3, 0xbb, 0x1a,
-	0xa7, 0xe8, 0x27, 0xa8, 0xd9, 0x11, 0x71, 0x6f, 0xa3, 0x32, 0xbd, 0x97, 0x65, 0xf3, 0x1f, 0xd7,
-	0x28, 0xaa, 0x81, 0xff, 0x20, 0x9e, 0x46, 0x75, 0xa8, 0x99, 0xb4, 0xca, 0xc1, 0xf2, 0x8f, 0x0f,
-	0x1d, 0x13, 0x5e, 0xce, 0x79, 0x2e, 0x99, 0x2e, 0xd6, 0xad, 0xf2, 0xb6, 0xd7, 0xd9, 0xfd, 0x0f,
-	0xaf, 0xd3, 0x3a, 0x31, 0x45, 0xd3, 0x9c, 0x09, 0x3b, 0xcf, 0xa2, 0x3f, 0x7d, 0x73, 0xec, 0x16,
-	0x43, 0x37, 0x60, 0x9b, 0xce, 0xe7, 0x59, 0x1a, 0x1b, 0x19, 0x3f, 0x4a, 0x5c, 0x76, 0xeb, 0x20,
-	0x7a, 0x07, 0xe0, 0xb4, 0xc8, 0x63, 0xe7, 0xe2, 0x9e, 0x3d, 0x4b, 0xc4, 0x4e, 0x03, 0xb7, 0xe5,
-	0x23, 0x3b, 0xca, 0xcc, 0x34, 0x28, 0x21, 0x74, 0xd7, 0x25, 0x59, 0x35, 0x49, 0xbe, 0xfb, 0xca,
-	0x24, 0xb1, 0x2b, 0xac, 0x4b, 0xf6, 0xe7, 0x0a, 0xd4, 0x1d, 0xa2, 0x07, 0x92, 0xeb, 0xfa, 0x32,
-	0xcd, 0x25, 0x80, 0xee, 0x97, 0x83, 0x5c, 0x07, 0xb8, 0xf9, 0xda, 0x00, 0xf8, 0x71, 0x9a, 0x33,
-	0x17, 0xe5, 0x37, 0x0f, 0xaa, 0xda, 0xd4, 0x21, 0x54, 0x3a, 0x63, 0x52, 0xd1, 0xd9, 0xdc, 0x84,
-	0xf0, 0xc9, 0x12, 0x40, 0x47, 0x10, 0x48, 0x5e, 0x88, 0xd8, 0x1e, 0x57, 0x67, 0xff, 0xd6, 0x9b,
-	0x05, 0xc1, 0x23, 0xb3, 0x88, 0xb8, 0xc5, 0xe5, 0xeb, 0xcc, 0x5f, 0xbe, 0xce, 0xfa, 0x3d, 0x08,
-	0xac, 0x17, 0x02, 0x08, 0x46, 0xcf, 0x0e, 0x9f, 0x3e, 0x7f, 0xd6, 0xdd, 0x72, 0xdf, 0x47, 0x84,
-	0x74, 0xbd, 0xfd, 0xbf, 0x3c, 0xe8, 0xd8, 0xf1, 0xf0, 0x8d, 0x7e, 0xc1, 0xc6, 0x3c, 0x43, 0x37,
-	0x20, 0x38, 0xca, 0xc7, 0xfa, 0x3e, 0x06, 0x5c, 0x5e, 0xaf, 0x11, 0xe0, 0xf2, 0x52, 0x1c, 0x78,
-	0x1f, 0x79, 0xe8, 0x36, 0x04, 0x8b, 0x3b, 0x08, 0xdb, 0x27, 0x31, 0x5e, 0x3c, 0x89, 0xf1, 0x91,
-	0x7e, 0x12, 0x47, 0xdb, 0x6b, 0x73, 0x07, 0xed, 0xc1, 0x8e, 0x55, 0x6d, 0x21, 0x98, 0x25, 0xf4,
-	0xfe, 0x8b, 0x61, 0x10, 0x6d, 0xe3, 0xd5, 0xe6, 0x45, 0x77, 0x00, 0x46, 0x4a, 0x30, 0x3a, 0x7b,
-	0xcc, 0xc7, 0x12, 0x75, 0xd6, 0x7b, 0x23, 0xda, 0xd9, 0x28, 0x91, 0xce, 0xe8, 0x24, 0x30, 0xf1,
-	0x3f, 0xfe, 0x37, 0x00, 0x00, 0xff, 0xff, 0x16, 0x8d, 0xfe, 0x33, 0xb0, 0x0b, 0x00, 0x00,
+	// 1297 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x57, 0x5b, 0x6f, 0x1b, 0x45,
+	0x14, 0x8e, 0x6f, 0x6b, 0xfb, 0xf8, 0x9a, 0xa1, 0x8d, 0x96, 0xa5, 0xa2, 0xc6, 0x94, 0xca, 0x82,
+	0x74, 0x4b, 0x43, 0x2b, 0x95, 0x4a, 0x80, 0x8a, 0x93, 0xca, 0x45, 0x2d, 0xad, 0xc6, 0x29, 0x3c,
+	0x46, 0x93, 0xdd, 0x89, 0x3d, 0x78, 0xbd, 0x6b, 0x66, 0x66, 0x43, 0x23, 0xf1, 0xce, 0x6f, 0xe0,
+	0x05, 0x89, 0x47, 0xde, 0xf9, 0x11, 0x3c, 0xf1, 0xc8, 0x4f, 0xe1, 0x19, 0xcd, 0xc5, 0xeb, 0x5b,
+	0x9a, 0x26, 0x12, 0x6f, 0x73, 0xbe, 0xef, 0xcc, 0x9c, 0x33, 0xc7, 0xdf, 0x39, 0x3b, 0x86, 0x3a,
+	0x4f, 0xe3, 0x98, 0x72, 0x7f, 0xc6, 0x13, 0x99, 0x78, 0xef, 0x8d, 0x92, 0x64, 0x14, 0xd1, 0xbb,
+	0xda, 0x3a, 0x4e, 0x4f, 0xee, 0xd2, 0xe9, 0x4c, 0x9e, 0x19, 0xb2, 0xfb, 0x77, 0x0e, 0xca, 0x87,
+	0xfc, 0xac, 0x4f, 0xa2, 0x08, 0xf5, 0xa0, 0x3d, 0x4d, 0x42, 0x1a, 0x89, 0xa3, 0x80, 0x44, 0xd1,
+	0xd1, 0x0f, 0x22, 0x89, 0xdd, 0x5c, 0x27, 0xd7, 0xab, 0xe2, 0xa6, 0xc1, 0x95, 0xd7, 0x37, 0x22,
+	0x89, 0x51, 0x07, 0xea, 0x22, 0x4a, 0xe4, 0xd1, 0x98, 0x88, 0xf1, 0x11, 0x0b, 0xdd, 0xbc, 0xf6,
+	0x02, 0x85, 0x0d, 0x88, 0x18, 0x3f, 0x0d, 0xd1, 0x43, 0x00, 0xfa, 0x5a, 0xd2, 0x58, 0xb0, 0x24,
+	0x16, 0x6e, 0xa1, 0x53, 0xe8, 0xd5, 0xf6, 0x5c, 0xdf, 0x46, 0xf2, 0x0f, 0x32, 0xea, 0x20, 0x96,
+	0xfc, 0x0c, 0x2f, 0xf9, 0x7a, 0x5f, 0x40, 0x6b, 0x8d, 0x46, 0x6d, 0x28, 0x4c, 0xe8, 0x99, 0xcd,
+	0x45, 0x2d, 0xd1, 0x35, 0x28, 0x9d, 0x92, 0x28, 0xa5, 0x36, 0xb2, 0x31, 0x1e, 0xe5, 0x1f, 0xe6,
+	0xba, 0xf7, 0xa0, 0xba, 0x4f, 0x24, 0x79, 0xc2, 0xc9, 0x94, 0x22, 0x04, 0xc5, 0x90, 0x48, 0xa2,
+	0x77, 0xd6, 0xb1, 0x5e, 0xab, 0xc3, 0x68, 0x72, 0xa2, 0x37, 0x56, 0xb0, 0x5a, 0x76, 0xef, 0x03,
+	0x0c, 0xa4, 0x9c, 0x0d, 0x28, 0x09, 0x29, 0xbf, 0x6c, 0xb0, 0xee, 0x77, 0x50, 0x57, 0xbb, 0x30,
+	0x15, 0xb3, 0xe7, 0x54, 0x12, 0x74, 0x13, 0x6a, 0x42, 0x12, 0x99, 0x8a, 0xa3, 0x20, 0x09, 0xa9,
+	0xde, 0x5f, 0xc2, 0x60, 0xa0, 0x7e, 0x12, 0x52, 0xf4, 0x11, 0x94, 0xc7, 0x3a, 0x84, 0x70, 0xf3,
+	0xba, 0x1e, 0x35, 0x7f, 0x11, 0x16, 0xcf, 0xb9, 0xee, 0x97, 0xd0, 0x52, 0x35, 0xc2, 0x54, 0xa4,
+	0x91, 0x1c, 0x4a, 0xc2, 0x25, 0xfa, 0x10, 0x8a, 0x63, 0x29, 0x67, 0x6e, 0xd8, 0xc9, 0xf5, 0x6a,
+	0x7b, 0x0d, 0x7f, 0x39, 0xee, 0x60, 0x0b, 0x6b, 0xf2, 0x6b, 0x07, 0x8a, 0x53, 0x2a, 0x49, 0xf7,
+	0xb7, 0x22, 0xd4, 0xd5, 0x01, 0x4f, 0x58, 0xcc, 0xc4, 0x98, 0x86, 0xc8, 0x85, 0xb2, 0x48, 0x83,
+	0x80, 0x0a, 0xa1, 0x93, 0xaa, 0xe0, 0xb9, 0xa9, 0x98, 0x90, 0x4a, 0xc2, 0x22, 0x61, 0xaf, 0x36,
+	0x37, 0xd1, 0x0d, 0xa8, 0x52, 0xce, 0x13, 0xae, 0x12, 0x77, 0x0b, 0xfa, 0x2a, 0x0b, 0x00, 0x79,
+	0x50, 0xd1, 0xc6, 0x50, 0x72, 0xb7, 0xa8, 0x37, 0x66, 0xb6, 0xda, 0x19, 0x70, 0x4a, 0x24, 0x0d,
+	0x1f, 0x4b, 0xb7, 0xa4, 0xc9, 0x05, 0xa0, 0x58, 0xa1, 0xae, 0xa4, 0x59, 0xc7, 0xb0, 0x19, 0x80,
+	0x3a, 0x50, 0x0b, 0x92, 0xe9, 0x2c, 0xa2, 0x86, 0x2f, 0x6b, 0x7e, 0x19, 0x42, 0xbb, 0xb0, 0x2d,
+	0x82, 0x31, 0x0d, 0xd3, 0x88, 0xf2, 0xfd, 0x94, 0x13, 0xc9, 0x92, 0xd8, 0xad, 0x74, 0x72, 0xbd,
+	0x02, 0xde, 0x24, 0x94, 0x37, 0x7d, 0x4d, 0x83, 0x54, 0x19, 0x99, 0x77, 0xd5, 0x78, 0x6f, 0x10,
+	0xd9, 0x9d, 0x5f, 0x09, 0xca, 0x5d, 0xd0, 0x95, 0x5a, 0x00, 0x4a, 0x04, 0x6c, 0x4a, 0x46, 0xd4,
+	0xad, 0x19, 0x11, 0x68, 0x03, 0xdd, 0x87, 0xeb, 0x7a, 0xf1, 0x32, 0x8d, 0xa2, 0xef, 0x09, 0x93,
+	0x59, 0x94, 0xba, 0x8e, 0x72, 0x3e, 0x89, 0x7a, 0xd0, 0x0a, 0x24, 0x7f, 0xc9, 0xe9, 0x2c, 0xf3,
+	0x6f, 0x68, 0xff, 0x75, 0x58, 0xdd, 0x20, 0x90, 0xbc, 0xaf, 0xeb, 0x97, 0xf9, 0x36, 0xcd, 0x0d,
+	0x36, 0x08, 0x74, 0x0b, 0x1a, 0x2c, 0x66, 0x46, 0x34, 0x87, 0x6c, 0x4a, 0xdd, 0x96, 0xf6, 0x5c,
+	0x05, 0xbb, 0x43, 0xa8, 0xf6, 0x23, 0x46, 0x63, 0xf9, 0x5c, 0x8c, 0xd0, 0x0d, 0x28, 0x48, 0x6e,
+	0xd4, 0x5e, 0xdb, 0xab, 0xcc, 0x1b, 0x74, 0xb0, 0x85, 0x15, 0x8c, 0x3a, 0xb6, 0x7f, 0xf2, 0x9a,
+	0x06, 0x3f, 0xeb, 0x2c, 0xa5, 0x3a, 0xc5, 0x28, 0xd5, 0x1d, 0x27, 0xe1, 0x59, 0xf7, 0xd7, 0x1c,
+	0x54, 0xb1, 0x9e, 0x3a, 0xea, 0xd4, 0x07, 0x50, 0xe7, 0x5a, 0xbf, 0x47, 0xfa, 0xc7, 0xb5, 0xc7,
+	0xb7, 0xfd, 0x35, 0x61, 0x0f, 0xb6, 0x70, 0x8d, 0x2f, 0xe9, 0xfc, 0xad, 0xe1, 0xd0, 0x27, 0x50,
+	0x39, 0xb1, 0xba, 0xd6, 0xb2, 0x54, 0xdd, 0xb0, 0x2c, 0xf6, 0xc1, 0x16, 0xce, 0x1c, 0xb2, 0xdc,
+	0xfe, 0x71, 0xa0, 0x6e, 0x72, 0x1b, 0xea, 0x6e, 0x44, 0x3b, 0xe0, 0x90, 0x40, 0xb2, 0x53, 0xd3,
+	0xd1, 0x25, 0x6c, 0x2d, 0x85, 0x9f, 0x10, 0x16, 0xd9, 0xb3, 0x2b, 0xd8, 0x5a, 0xa8, 0x09, 0x79,
+	0x16, 0x5a, 0xa5, 0xe7, 0x59, 0xb8, 0xdc, 0x37, 0xa5, 0x0b, 0xfa, 0xc6, 0xb9, 0xa8, 0x6f, 0xca,
+	0x17, 0xf5, 0x4d, 0xe5, 0xc2, 0xbe, 0xa9, 0xbe, 0xa5, 0x6f, 0x60, 0xb3, 0x6f, 0x76, 0xc0, 0x09,
+	0x88, 0xea, 0x0f, 0x2d, 0xdf, 0x0a, 0xb6, 0x16, 0xfa, 0x18, 0xda, 0x9c, 0xfe, 0x98, 0x52, 0x21,
+	0x05, 0xa6, 0x01, 0x65, 0xa7, 0x34, 0xd4, 0xd2, 0x2d, 0xe2, 0x0d, 0x5c, 0xa9, 0x76, 0x8e, 0x0d,
+	0x48, 0x1c, 0xaa, 0x32, 0x35, 0xb4, 0xeb, 0x3a, 0x8c, 0xba, 0x50, 0x9f, 0x84, 0xe9, 0x74, 0x26,
+	0x5e, 0xc4, 0xfb, 0x4c, 0x4c, 0xb4, 0x60, 0x8b, 0x78, 0x05, 0x3b, 0xbf, 0x93, 0x5b, 0x57, 0xea,
+	0xe4, 0xf6, 0x9b, 0x3a, 0x79, 0x17, 0xb6, 0x99, 0xf8, 0x96, 0xca, 0x9f, 0x12, 0x3e, 0xd9, 0x67,
+	0x82, 0x1c, 0xab, 0x5c, 0xb7, 0xf5, 0xc5, 0x37, 0x09, 0xd4, 0x87, 0x7a, 0x90, 0x0a, 0x99, 0x4c,
+	0x8d, 0x3a, 0x5c, 0xa4, 0x87, 0xf3, 0x4d, 0x7f, 0x59, 0x32, 0x7e, 0x7f, 0xc9, 0xc3, 0x7c, 0xb3,
+	0x56, 0x36, 0xbd, 0x79, 0x10, 0xbc, 0x73, 0xc5, 0x41, 0x70, 0xed, 0x0a, 0x83, 0xe0, 0xfa, 0xa5,
+	0x07, 0xc1, 0xce, 0x39, 0x83, 0xc0, 0xfb, 0x0a, 0xb6, 0x37, 0xae, 0x75, 0xa5, 0x6f, 0xed, 0x29,
+	0x54, 0xfb, 0x49, 0x7c, 0xc2, 0x46, 0xaa, 0xe7, 0x7d, 0x70, 0x02, 0x6d, 0xb8, 0x39, 0x5d, 0xc0,
+	0x1d, 0x3f, 0xe3, 0xec, 0xca, 0xd4, 0xcd, 0x7a, 0x79, 0x9f, 0x43, 0x6d, 0x09, 0xbe, 0x52, 0xdc,
+	0x26, 0xd4, 0xcd, 0x56, 0x93, 0x78, 0xf7, 0x8f, 0x3c, 0x34, 0x9e, 0x25, 0x23, 0x6c, 0x64, 0xa8,
+	0x92, 0xd9, 0x85, 0xd2, 0xf2, 0xe4, 0xb9, 0xe6, 0xaf, 0xd0, 0xfe, 0x7c, 0xfa, 0x18, 0x27, 0x74,
+	0x1b, 0x0a, 0x24, 0x98, 0xd8, 0xb1, 0x83, 0xd6, 0x7c, 0x1f, 0x07, 0x13, 0x35, 0x0e, 0x49, 0xa0,
+	0x34, 0x5b, 0xe2, 0x94, 0x84, 0x67, 0x76, 0xf4, 0xac, 0x9f, 0x8a, 0x15, 0xa7, 0x4e, 0xd5, 0x4e,
+	0xde, 0xcf, 0x50, 0x32, 0x63, 0xed, 0xe1, 0x5a, 0x65, 0x3a, 0xe7, 0x65, 0xf3, 0x3f, 0xd7, 0xc8,
+	0x2b, 0x41, 0xe1, 0x71, 0x30, 0xf1, 0xca, 0x50, 0xd2, 0x69, 0x65, 0xc3, 0xf0, 0xdf, 0x02, 0x34,
+	0x75, 0x78, 0x31, 0x4b, 0x62, 0x41, 0x55, 0xb1, 0xee, 0x64, 0xaf, 0x24, 0x95, 0xdd, 0xbb, 0xfe,
+	0x2a, 0xad, 0x12, 0x93, 0x84, 0xc5, 0x94, 0x9b, 0x19, 0xec, 0xfd, 0x59, 0xd0, 0x3f, 0xbb, 0xc1,
+	0x94, 0xd4, 0xc8, 0x6c, 0x16, 0xb1, 0x40, 0x2b, 0xef, 0x69, 0x68, 0xb3, 0x5b, 0x05, 0xd1, 0xfb,
+	0x00, 0x27, 0x69, 0x1c, 0x58, 0x17, 0xfb, 0x5c, 0x5c, 0x20, 0x66, 0x82, 0xd9, 0x23, 0x9f, 0x9a,
+	0xf1, 0xab, 0x27, 0x58, 0x06, 0xa1, 0x07, 0x36, 0xc9, 0xa2, 0x4e, 0xf2, 0x83, 0x37, 0x26, 0xe9,
+	0xdb, 0xc2, 0xda, 0x64, 0x7f, 0xc9, 0x43, 0xd9, 0x22, 0x6a, 0x88, 0xda, 0x49, 0x95, 0xa5, 0xb9,
+	0x00, 0xd0, 0xa3, 0xec, 0xe3, 0xa3, 0x02, 0xdc, 0x7e, 0x6b, 0x00, 0xff, 0x19, 0x8b, 0xa9, 0x8d,
+	0xf2, 0x7b, 0x0e, 0x8a, 0xca, 0x54, 0x21, 0x24, 0x9b, 0x52, 0x21, 0xc9, 0x74, 0xa6, 0x43, 0x14,
+	0xf0, 0x02, 0x40, 0x07, 0xe0, 0x88, 0x24, 0xe5, 0x81, 0xf9, 0xb9, 0x9a, 0x7b, 0x77, 0x2e, 0x17,
+	0xc4, 0x1f, 0xea, 0x4d, 0xd8, 0x6e, 0xce, 0x5e, 0xb5, 0x85, 0xc5, 0xab, 0xb6, 0xdb, 0x01, 0xc7,
+	0x78, 0x21, 0x00, 0x67, 0x78, 0xb8, 0xff, 0xe2, 0xd5, 0x61, 0x7b, 0xcb, 0xae, 0x0f, 0x30, 0x6e,
+	0xe7, 0xf6, 0xfe, 0xca, 0x41, 0xd3, 0x8c, 0xb4, 0x97, 0xea, 0xe5, 0x1f, 0x24, 0x11, 0xba, 0x05,
+	0xce, 0x41, 0x3c, 0x52, 0xef, 0x18, 0xf0, 0xb3, 0x27, 0x81, 0x07, 0x7e, 0xf6, 0x21, 0xef, 0xe5,
+	0x3e, 0xcd, 0xa1, 0xbb, 0xe0, 0xcc, 0xbf, 0x9b, 0xbe, 0xf9, 0x2b, 0xe1, 0xcf, 0xff, 0x4a, 0xf8,
+	0x07, 0xea, 0xaf, 0x84, 0xd7, 0x58, 0x99, 0x95, 0x68, 0x17, 0x5a, 0x46, 0xb5, 0x29, 0xa7, 0x86,
+	0x50, 0xe7, 0xcf, 0x87, 0x81, 0xd7, 0xf0, 0x97, 0x9b, 0x17, 0xdd, 0x03, 0x18, 0x4a, 0x4e, 0xc9,
+	0xf4, 0x59, 0x32, 0x12, 0xa8, 0xb9, 0xda, 0x1b, 0x5e, 0x6b, 0xad, 0x44, 0x2a, 0xa3, 0x63, 0x47,
+	0xc7, 0xff, 0xec, 0xbf, 0x00, 0x00, 0x00, 0xff, 0xff, 0x35, 0xd2, 0x0f, 0x74, 0xe8, 0x0c, 0x00,
+	0x00,
 }

 // Reference imports to suppress errors if they are not otherwise used.
--- a/api/agent/grpc/runner.proto
+++ b/api/agent/grpc/runner.proto
@@ -49,7 +49,10 @@ message CallFinished {
    int64 executionDuration = 9;
    bool errorUser = 10;
    string image = 11;
-    int64 dockerWaitDuration = 12;
+    int64 imagePullWaitDuration = 12;
+    int64 ctrPrepDuration = 13;
+    int64 ctrCreateDuration = 14;
+    int64 initStartTime = 15;
 }

 message ClientMsg {
@@ -85,6 +88,10 @@ message RunnerStatus {
    int64 executionDuration = 16; // the amount of time for function execution
    bool isNetworkDisabled = 17; // returns true if runner network is not ready
    map<string,string> customStatus = 18; // returns any user defined healthcheck status
+    int64 imagePullWaitDuration = 19;
+    int64 ctrPrepDuration = 20;
+    int64 ctrCreateDuration = 21;
+    int64 initStartTime = 22;
 }

 message ConfigMsg {
--- a/api/agent/lb_agent.go
+++ b/api/agent/lb_agent.go
@@ -208,8 +208,6 @@ func (a *lbAgent) Submit(callI Call) error {
 	statsCalls(ctx)

 	if !a.shutWg.AddSession(1) {
-		span.SetStatus(trace.Status{Code: int32(http.StatusServiceUnavailable),
-			Message: "Timed out - server too busy"})
 		statsTooBusy(ctx)
 		return models.ErrCallTimeoutServerBusy
 	}
--- a/api/agent/pure_runner.go
+++ b/api/agent/pure_runner.go
@@ -232,10 +232,13 @@ func (ch *callHandle) enqueueCallResponse(err error) {
 	var image string
 	var details string
 	var errCode int
-	var dockerWaitDuration time.Duration
 	var errStr string
 	var errUser bool
 	var nErr error
+	var imagePullWaitDuration int64
+	var ctrCreateDuration int64
+	var ctrPrepDuration int64
+	var initStartTime int64

 	log := common.Logger(ch.ctx)

@@ -276,24 +279,30 @@ func (ch *callHandle) enqueueCallResponse(err error) {
 		}
 		image = mcall.Image
 		details = mcall.ID
-		dockerWaitDuration = mcall.DockerWaitDuration
+		imagePullWaitDuration = ch.c.imagePullWaitTime
+		ctrCreateDuration = ch.c.ctrCreateTime
+		ctrCreateDuration = ch.c.ctrCreateTime
+		initStartTime = ch.c.initStartTime
 	}
 	log.Debugf("Sending Call Finish details=%v", details)

 	errTmp := ch.enqueueMsgStrict(&runner.RunnerMsg{
 		Body: &runner.RunnerMsg_Finished{Finished: &runner.CallFinished{
-			Success:            nErr == nil,
-			Details:            details,
-			ErrorCode:          int32(errCode),
-			ErrorStr:           errStr,
-			CreatedAt:          createdAt,
-			StartedAt:          startedAt,
-			CompletedAt:        completedAt,
-			Image:              image,
-			SchedulerDuration:  int64(schedulerDuration),
-			ExecutionDuration:  int64(executionDuration),
-			DockerWaitDuration: int64(dockerWaitDuration),
-			ErrorUser:          errUser,
+			CompletedAt:           completedAt,
+			CreatedAt:             createdAt,
+			CtrCreateDuration:     ctrCreateDuration,
+			CtrPrepDuration:       ctrPrepDuration,
+			Details:               details,
+			ErrorCode:             int32(errCode),
+			ErrorStr:              errStr,
+			ErrorUser:             errUser,
+			ExecutionDuration:     int64(executionDuration),
+			Image:                 image,
+			ImagePullWaitDuration: imagePullWaitDuration,
+			InitStartTime:         initStartTime,
+			SchedulerDuration:     int64(schedulerDuration),
+			StartedAt:             startedAt,
+			Success:               nErr == nil,
 		}}})

 	if errTmp != nil {
@@ -931,6 +940,14 @@ func (pr *pureRunner) runStatusCall(ctx context.Context) *runner.RunnerStatus {
 		}
 	}

+	// Loading with runHot metrics if not nil
+	if mcall != nil {
+		result.ImagePullWaitDuration = atomic.LoadInt64(&mcall.imagePullWaitTime)
+		result.CtrCreateDuration = atomic.LoadInt64(&mcall.ctrCreateTime)
+		result.CtrPrepDuration = atomic.LoadInt64(&mcall.ctrPrepTime)
+		result.InitStartTime = atomic.LoadInt64(&mcall.initStartTime)
+	}
+
 	// Status images should not output excessive data since we echo the
 	// data back to caller.
 	body, _ := ioutil.ReadAll(resp.Body)
--- a/api/agent/runner_client.go
+++ b/api/agent/runner_client.go
@@ -121,28 +121,36 @@ func TranslateGRPCStatusToRunnerStatus(status *pb.RunnerStatus) *pool.RunnerStat
 	// These are nanosecond monotonic deltas, they cannot be zero if they were transmitted.
 	runnerSchedLatency := time.Duration(status.GetSchedulerDuration())
 	runnerExecLatency := time.Duration(status.GetExecutionDuration())
+	ctrPrepDuration := time.Duration(status.GetCtrPrepDuration())
+	ctrCreateDuration := time.Duration(status.GetCtrCreateDuration())
+	imagePullWaitDuration := time.Duration(status.GetImagePullWaitDuration())
+	initStartTime := time.Duration(status.GetInitStartTime())

 	creat, _ := common.ParseDateTime(status.CreatedAt)
 	start, _ := common.ParseDateTime(status.StartedAt)
 	compl, _ := common.ParseDateTime(status.CompletedAt)

 	return &pool.RunnerStatus{
-		ActiveRequestCount: status.Active,
-		RequestsReceived:   status.RequestsReceived,
-		RequestsHandled:    status.RequestsHandled,
-		StatusFailed:       status.Failed,
-		KdumpsOnDisk:       status.KdumpsOnDisk,
-		Cached:             status.Cached,
-		StatusId:           status.Id,
-		Details:            status.Details,
-		ErrorCode:          status.ErrorCode,
-		ErrorStr:           status.ErrorStr,
-		CreatedAt:          creat,
-		StartedAt:          start,
-		CompletedAt:        compl,
-		SchedulerDuration:  runnerSchedLatency,
-		ExecutionDuration:  runnerExecLatency,
-		IsNetworkDisabled:  status.IsNetworkDisabled,
+		ActiveRequestCount:    status.Active,
+		RequestsReceived:      status.RequestsReceived,
+		RequestsHandled:       status.RequestsHandled,
+		StatusFailed:          status.Failed,
+		KdumpsOnDisk:          status.KdumpsOnDisk,
+		Cached:                status.Cached,
+		StatusId:              status.Id,
+		Details:               status.Details,
+		ErrorCode:             status.ErrorCode,
+		ErrorStr:              status.ErrorStr,
+		CreatedAt:             creat,
+		StartedAt:             start,
+		CompletedAt:           compl,
+		SchedulerDuration:     runnerSchedLatency,
+		ExecutionDuration:     runnerExecLatency,
+		ImagePullWaitDuration: imagePullWaitDuration,
+		CtrPrepDuration:       ctrPrepDuration,
+		CtrCreateDuration:     ctrCreateDuration,
+		InitStartTime:         initStartTime,
+		IsNetworkDisabled:     status.IsNetworkDisabled,
 	}
 }

@@ -252,7 +260,7 @@ func sendToRunner(ctx context.Context, protocolClient pb.RunnerProtocol_EngageCl
 		// WARNING: blocking read.
 		n, err := bodyReader.Read(writeBuffer)
 		if err != nil && err != io.EOF {
-			errorMsg = fmt.Sprintf("Failed to receive data from http client body")
+			errorMsg = "Failed to receive data from http client body"
 			span.SetStatus(trace.Status{Code: int32(trace.StatusCodeDataLoss), Message: errorMsg})
 			log.WithError(err).Error(errorMsg)
 		}
@@ -441,7 +449,10 @@ DataLoop:
 				trace.BoolAttribute("success", body.Finished.GetSuccess()),
 				trace.Int64Attribute("execution_duration", body.Finished.GetExecutionDuration()),
 				trace.Int64Attribute("scheduler_duration", body.Finished.GetSchedulerDuration()),
-				trace.Int64Attribute("docker_wait_duration", body.Finished.GetDockerWaitDuration()),
+				trace.Int64Attribute("image_pull_wait", body.Finished.GetImagePullWaitDuration()),
+				trace.Int64Attribute("container_create_duration", body.Finished.GetCtrCreateDuration()),
+				trace.Int64Attribute("container_preparation_duration", body.Finished.GetCtrPrepDuration()),
+				trace.Int64Attribute("init_start", body.Finished.GetInitStartTime()),
 				trace.StringAttribute("completed_at", body.Finished.GetCompletedAt()),
 				trace.StringAttribute("created_at", body.Finished.GetCreatedAt()),
 				trace.StringAttribute("started_at", body.Finished.GetStartedAt()),
--- a/api/models/call.go
+++ b/api/models/call.go
@@ -123,9 +123,6 @@ type Call struct {
 	// Duration that user code was running for, in nanoseconds.
 	ExecutionDuration time.Duration `json:"execution_duration,omitempty" db:"execution_duration"`

-	// Duration that image pull has waited, in nanoseconds.
-	DockerWaitDuration time.Duration `json:"docker_wait_duration,omitempty" db:"-"`
-
 	// Stats is a list of metrics from this call's execution, possibly empty.
 	Stats stats.Stats `json:"stats,omitempty" db:"stats"`

--- a/api/runnerpool/runner_pool.go
+++ b/api/runnerpool/runner_pool.go
@@ -26,22 +26,26 @@ type RunnerPool interface {

 // RunnerStatus is general information on Runner health as returned by Runner::Status() call
 type RunnerStatus struct {
-	ActiveRequestCount int32           // Number of active running requests on Runner
-	RequestsReceived   uint64          // Number of requests received by Runner
-	RequestsHandled    uint64          // Number of requests handled without NACK by Runner
-	KdumpsOnDisk       uint64          // Number of kdumps on disk
-	StatusFailed       bool            // True if Status execution failed
-	Cached             bool            // True if Status was provided from cache
-	StatusId           string          // Call ID for Status
-	Details            string          // General/Debug Log information
-	ErrorCode          int32           // If StatusFailed, then error code is set
-	ErrorStr           string          // Error details if StatusFailed and ErrorCode is set
-	CreatedAt          common.DateTime // Status creation date at Runner
-	StartedAt          common.DateTime // Status execution date at Runner
-	CompletedAt        common.DateTime // Status completion date at Runner
-	SchedulerDuration  time.Duration   // Amount of time runner scheduler spent on the request
-	ExecutionDuration  time.Duration   // Amount of time runner spent on function execution
-	IsNetworkDisabled  bool            // True if network on runner is offline
+	ActiveRequestCount    int32           // Number of active running requests on Runner
+	RequestsReceived      uint64          // Number of requests received by Runner
+	RequestsHandled       uint64          // Number of requests handled without NACK by Runner
+	KdumpsOnDisk          uint64          // Number of kdumps on disk
+	StatusFailed          bool            // True if Status execution failed
+	Cached                bool            // True if Status was provided from cache
+	StatusId              string          // Call ID for Status
+	Details               string          // General/Debug Log information
+	ErrorCode             int32           // If StatusFailed, then error code is set
+	ErrorStr              string          // Error details if StatusFailed and ErrorCode is set
+	CreatedAt             common.DateTime // Status creation date at Runner
+	StartedAt             common.DateTime // Status execution date at Runner
+	CompletedAt           common.DateTime // Status completion date at Runner
+	SchedulerDuration     time.Duration   // Amount of time runner scheduler spent on the request
+	ExecutionDuration     time.Duration   // Amount of time runner spent on function execution
+	ImagePullWaitDuration time.Duration   // Amount of time spent waiting for the image pull
+	CtrPrepDuration       time.Duration   // Amount of time spent preparing for the container creation
+	CtrCreateDuration     time.Duration   //Amount of time spent creating the container
+	InitStartTime         time.Duration   // Container Init UDS Latency time
+	IsNetworkDisabled     bool            // True if network on runner is offline
 }

 // Runner is the interface to invoke the execution of a function call on a specific runner
--- a/api/server/server.go
+++ b/api/server/server.go
@@ -861,12 +861,12 @@ func (s *Server) startGears(ctx context.Context, cancel context.CancelFunc) {
 			GetStartOptions: func(r *http.Request) trace.StartOptions {
 				startOptions := trace.StartOptions{}
 				// TODO: Add list of url paths to exclude
-				if r.URL.Path != "/version" {
-					startOptions.Sampler = trace.AlwaysSample()
+				if r.URL.Path == "/" {
+					startOptions.Sampler = trace.NeverSample()
 				}
-				// Defaults to global sampler
 				return startOptions
 			},
+			// TODO: add FormatSpanName to clean up trace exporter operations dash
 		}
 	}

@@ -891,18 +891,7 @@ func (s *Server) startGears(ctx context.Context, cancel context.CancelFunc) {
 		logrus.WithField("type", s.nodeType).Infof("Fn Admin serving on `%v`", s.svcConfigs[AdminServer].Addr)
 		adminServer := s.svcConfigs[AdminServer]
 		if adminServer.Handler == nil {
-			adminServer.Handler = &ochttp.Handler{
-				Handler: s.AdminRouter,
-				GetStartOptions: func(r *http.Request) trace.StartOptions {
-					startOptions := trace.StartOptions{}
-					// TODO: Add list of url paths to exclude
-					if r.URL.Path != "/version" {
-						startOptions.Sampler = trace.AlwaysSample()
-					}
-					// Defaults to global sampler
-					return startOptions
-				},
-			}
+			adminServer.Handler = s.AdminRouter
 		}

 		go func() {