add opentracing spans for metrics

api/server/init.go

@@ -26,6 +26,7 @@ func init() {
 	viper.SetDefault(EnvDBURL, fmt.Sprintf("sqlite3://%s/data/fn.db", cwd))
 	viper.SetDefault(EnvLOGDBURL, "") // default to just using DB url
 	viper.SetDefault(EnvPort, 8080)
+	viper.SetDefault(EnvZipkinURL, "") // off default
 	viper.SetDefault(EnvAPIURL, fmt.Sprintf("http://127.0.0.1:%d", viper.GetInt(EnvPort)))
 	viper.AutomaticEnv() // picks up env vars automatically
 	logLevel, err := logrus.ParseLevel(viper.GetString(EnvLogLevel))

api/server/runner_async_test.go

@@ -32,7 +32,7 @@ func testRouterAsync(ds models.Datastore, mq models.MessageQueue, rnr *runner.Ru
 	r := s.Router
 	r.Use(gin.Logger())
 
-	r.Use(prepareMiddleware(ctx))
+	s.Router.Use(loggerWrap)
 	s.bindHandlers(ctx)
 	return r
 }

api/server/runner_test.go

@@ -18,7 +18,7 @@ func testRunner(t *testing.T) (*runner.Runner, context.CancelFunc) {
 	ctx, cancel := context.WithCancel(context.Background())
 	ds := datastore.NewMock()
 	fnl := logs.NewMock()
-	r, err := runner.New(ctx, runner.NewFuncLogger(fnl), runner.NewMetricLogger(), ds)
+	r, err := runner.New(ctx, runner.NewFuncLogger(fnl), ds)
 	if err != nil {
 		t.Fatal("Test: failed to create new runner")
 	}

api/server/server.go

@@ -15,6 +15,9 @@ import (
 	"github.com/Sirupsen/logrus"
 	"github.com/ccirello/supervisor"
 	"github.com/gin-gonic/gin"
+	"github.com/opentracing/opentracing-go"
+	"github.com/opentracing/opentracing-go/ext"
+	"github.com/openzipkin/zipkin-go-opentracing"
 	"github.com/patrickmn/go-cache"
 	"github.com/spf13/viper"
 	"gitlab-odx.oracle.com/odx/functions/api"
@@ -28,12 +31,13 @@ import (
 )
 
 const (
-	EnvLogLevel = "log_level"
-	EnvMQURL    = "mq_url"
-	EnvDBURL    = "db_url"
-	EnvLOGDBURL = "logstore_url"
-	EnvPort     = "port" // be careful, Gin expects this variable to be "port"
-	EnvAPIURL   = "api_url"
+	EnvLogLevel  = "log_level"
+	EnvMQURL     = "mq_url"
+	EnvDBURL     = "db_url"
+	EnvLOGDBURL  = "logstore_url"
+	EnvPort      = "port" // be careful, Gin expects this variable to be "port"
+	EnvAPIURL    = "api_url"
+	EnvZipkinURL = "zipkin_url"
 )
 
 type Server struct {
@@ -84,10 +88,9 @@ func NewFromEnv(ctx context.Context) *Server {
 
 // New creates a new Functions server with the passed in datastore, message queue and API URL
 func New(ctx context.Context, ds models.Datastore, mq models.MessageQueue, logDB models.FnLog, apiURL string, opts ...ServerOption) *Server {
-	metricLogger := runner.NewMetricLogger()
 	funcLogger := runner.NewFuncLogger(logDB)
 
-	rnr, err := runner.New(ctx, funcLogger, metricLogger, ds)
+	rnr, err := runner.New(ctx, funcLogger, ds)
 	if err != nil {
 		logrus.WithError(err).Fatalln("Failed to create a runner")
 		return nil
@@ -105,7 +108,8 @@ func New(ctx context.Context, ds models.Datastore, mq models.MessageQueue, logDB
 	}
 
 	setMachineId()
-	s.Router.Use(prepareMiddleware(ctx))
+	setTracer()
+	s.Router.Use(loggerWrap, traceWrap)
 	s.bindHandlers(ctx)
 
 	for _, opt := range opts {
@@ -117,6 +121,55 @@ func New(ctx context.Context, ds models.Datastore, mq models.MessageQueue, logDB
 	return s
 }
 
+// we should use http grr
+func traceWrap(c *gin.Context) {
+	// try to grab a span from the request if made from another service, ignore err if not
+	wireContext, _ := opentracing.GlobalTracer().Extract(
+		opentracing.HTTPHeaders,
+		opentracing.HTTPHeadersCarrier(c.Request.Header))
+
+	// Create the span referring to the RPC client if available.
+	// If wireContext == nil, a root span will be created.
+	// TODO we should add more tags?
+	serverSpan := opentracing.StartSpan("serve_http", ext.RPCServerOption(wireContext), opentracing.Tag{"path", c.Request.URL.Path})
+	defer serverSpan.Finish()
+
+	ctx := opentracing.ContextWithSpan(c.Request.Context(), serverSpan)
+	c.Request = c.Request.WithContext(ctx)
+	c.Next()
+}
+
+func setTracer() {
+	var (
+		debugMode          = false
+		serviceName        = "fn-server"
+		serviceHostPort    = "localhost:8080" // meh
+		zipkinHTTPEndpoint = viper.GetString(EnvZipkinURL)
+		// ex: "http://zipkin:9411/api/v1/spans"
+	)
+
+	if zipkinHTTPEndpoint == "" {
+		return
+	}
+
+	logger := zipkintracer.LoggerFunc(func(i ...interface{}) error { logrus.Error(i...); return nil })
+
+	collector, err := zipkintracer.NewHTTPCollector(zipkinHTTPEndpoint, zipkintracer.HTTPLogger(logger))
+	if err != nil {
+		logrus.WithError(err).Fatalln("couldn't start trace collector")
+	}
+	tracer, err := zipkintracer.NewTracer(zipkintracer.NewRecorder(collector, debugMode, serviceHostPort, serviceName),
+		zipkintracer.ClientServerSameSpan(true),
+		zipkintracer.TraceID128Bit(true),
+	)
+	if err != nil {
+		logrus.WithError(err).Fatalln("couldn't start tracer")
+	}
+
+	opentracing.SetGlobalTracer(tracer)
+	logrus.WithFields(logrus.Fields{"url": zipkinHTTPEndpoint}).Info("started tracer")
+}
+
 func setMachineId() {
 	port := uint16(viper.GetInt(EnvPort))
 	addr := whoAmI().To4()
@@ -150,22 +203,19 @@ func whoAmI() net.IP {
 	return nil
 }
 
-// todo: remove this or change name
-func prepareMiddleware(ctx context.Context) gin.HandlerFunc {
-	return func(c *gin.Context) {
-		ctx, _ := common.LoggerWithFields(ctx, extractFields(c))
+func loggerWrap(c *gin.Context) {
+	ctx, _ := common.LoggerWithFields(c.Request.Context(), extractFields(c))
 
-		if appName := c.Param(api.CApp); appName != "" {
-			c.Set(api.AppName, appName)
-		}
-
-		if routePath := c.Param(api.CRoute); routePath != "" {
-			c.Set(api.Path, routePath)
-		}
-
-		c.Request = c.Request.WithContext(ctx)
-		c.Next()
+	if appName := c.Param(api.CApp); appName != "" {
+		c.Set(api.AppName, appName)
 	}
+
+	if routePath := c.Param(api.CRoute); routePath != "" {
+		c.Set(api.Path, routePath)
+	}
+
+	c.Request = c.Request.WithContext(ctx)
+	c.Next()
 }
 
 func DefaultEnqueue(ctx context.Context, mq models.MessageQueue, task *models.Task) (*models.Task, error) {

api/server/server_test.go

@@ -38,7 +38,7 @@ func testServer(ds models.Datastore, mq models.MessageQueue, logDB models.FnLog,
 	r := s.Router
 	r.Use(gin.Logger())
 
-	s.Router.Use(prepareMiddleware(ctx))
+	s.Router.Use(loggerWrap)
 	s.bindHandlers(ctx)
 	return s
 }