Merge pull request #45 from iron-io/listeners-and-handlers

Added support for hooks to customize behavior.

README.md

@@ -90,3 +90,10 @@ TODO
 
 TODO
 
+## FAQ
+
+### Why isn't there monitoring and what not built in?
+
+We didn't want to prescribe how to operate IronFunctions, we wanted to get the core functionality released and stable while
+allowing hooks to customize/extend the core. Also, since we're assuming containers will be used for all deployments, users can
+use container based tools to log, monitor, etc. 

api/datastore/bolt/bolt.go

@@ -15,6 +15,7 @@ type BoltDatastore struct {
 	routesBucket []byte
 	appsBucket   []byte
 	logsBucket   []byte
+	extrasBucket []byte
 	db           *bolt.DB
 	log          logrus.FieldLogger
 }
@@ -33,15 +34,17 @@ func New(url *url.URL) (models.Datastore, error) {
 		log.WithError(err).Errorln("Error on bolt.Open")
 		return nil, err
 	}
-	bucketPrefix := "funcs-"
+	// I don't think we need a prefix here do we? Made it blank. If we do, we should call the query param "prefix" instead of bucket.
+	bucketPrefix := ""
 	if url.Query()["bucket"] != nil {
 		bucketPrefix = url.Query()["bucket"][0]
 	}
 	routesBucketName := []byte(bucketPrefix + "routes")
 	appsBucketName := []byte(bucketPrefix + "apps")
 	logsBucketName := []byte(bucketPrefix + "logs")
+	extrasBucketName := []byte(bucketPrefix + "extras") // todo: think of a better name
 	err = db.Update(func(tx *bolt.Tx) error {
-		for _, name := range [][]byte{routesBucketName, appsBucketName, logsBucketName} {
+		for _, name := range [][]byte{routesBucketName, appsBucketName, logsBucketName, extrasBucketName} {
 			_, err := tx.CreateBucketIfNotExists(name)
 			if err != nil {
 				log.WithError(err).WithFields(logrus.Fields{"name": name}).Error("create bucket")
@@ -59,6 +62,7 @@ func New(url *url.URL) (models.Datastore, error) {
 		routesBucket: routesBucketName,
 		appsBucket:   appsBucketName,
 		logsBucket:   logsBucketName,
+		extrasBucket: extrasBucketName,
 		db:           db,
 		log:          log,
 	}
@@ -164,6 +168,7 @@ func (ds *BoltDatastore) GetApp(name string) (*models.App, error) {
 
 func (ds *BoltDatastore) getRouteBucketForApp(tx *bolt.Tx, appName string) (*bolt.Bucket, error) {
 	var err error
+	// todo: should this be reversed?  Make a bucket for each app that contains sub buckets for routes, etc
 	bp := tx.Bucket(ds.routesBucket)
 	b := bp.Bucket([]byte(appName))
 	if b == nil {
@@ -288,3 +293,22 @@ func (ds *BoltDatastore) GetRoutes(filter *models.RouteFilter) ([]*models.Route,
 	}
 	return res, nil
 }
+
+func (ds *BoltDatastore) Put(key, value []byte) error {
+	ds.db.Update(func(tx *bolt.Tx) error {
+		b := tx.Bucket(ds.extrasBucket) // todo: maybe namespace by app?
+		err := b.Put(key, value)
+		return err
+	})
+	return nil
+}
+
+func (ds *BoltDatastore) Get(key []byte) ([]byte, error) {
+	var ret []byte
+	ds.db.View(func(tx *bolt.Tx) error {
+		b := tx.Bucket(ds.extrasBucket)
+		ret = b.Get(key)
+		return nil
+	})
+	return ret, nil
+}

api/datastore/postgres/postgres.go

@@ -23,6 +23,11 @@ const appsTableCreate = `CREATE TABLE IF NOT EXISTS apps (
     name character varying(256) NOT NULL PRIMARY KEY
 );`
 
+const extrasTableCreate = `CREATE TABLE IF NOT EXISTS extras (
+    key character varying(256) NOT NULL PRIMARY KEY,
+	value character varying(256) NOT NULL
+);`
+
 const routeSelector = `SELECT app_name, path, image, headers FROM routes`
 
 type rowScanner interface {
@@ -56,7 +61,7 @@ func New(url *url.URL) (models.Datastore, error) {
 		db: db,
 	}
 
-	for _, v := range []string{routesTableCreate, appsTableCreate} {
+	for _, v := range []string{routesTableCreate, appsTableCreate, extrasTableCreate} {
 		_, err = db.Exec(v)
 		if err != nil {
 			return nil, err
@@ -158,14 +163,16 @@ func (ds *PostgresDatastore) StoreRoute(route *models.Route) (*models.Route, err
 
 	_, err = ds.db.Exec(`
 		INSERT INTO routes (
-			app_name, path, image,
+			app_name, 
+			path, 
+			image,
 			headers
 		)
-		VALUES ($1, $2, $3, $4, $5)
+		VALUES ($1, $2, $3, $4)
 		ON CONFLICT (name) DO UPDATE SET
-			path = $1,
+			path = $2,
-			image = $2,
+			image = $3,
-			headers = $3;
+			headers = $4;
 		`,
 		route.AppName,
 		route.Path,
@@ -274,3 +281,34 @@ func buildFilterQuery(filter *models.RouteFilter) string {
 
 	return filterQuery
 }
+
+func (ds *PostgresDatastore) Put(key, value []byte) error {
+	_, err := ds.db.Exec(`
+	    INSERT INTO extras (
+			key,
+			value
+		)
+		VALUES ($1, $2)
+		ON CONFLICT (key) DO UPDATE SET
+			value = $1;
+		`, value)
+
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (ds *PostgresDatastore) Get(key []byte) ([]byte, error) {
+	row := ds.db.QueryRow("SELECT value FROM extras WHERE key=$1", key)
+
+	var value []byte
+	err := row.Scan(&value)
+
+	if err != nil {
+		return nil, err
+	}
+
+	return value, nil
+}

api/ifaces/handlers.go

@@ -0,0 +1,25 @@
+package ifaces
+
+import (
+	"net/http"
+
+	"github.com/iron-io/functions/api/models"
+)
+
+type SpecialHandler interface {
+	Handle(c HandlerContext) error
+}
+
+// Each handler can modify the context here so when it gets passed along, it will use the new info.
+// Not using Gin's Context so we don't lock ourselves into Gin, this is a subset of the Gin context.
+type HandlerContext interface {
+	// Request returns the underlying http.Request object
+	Request() *http.Request
+
+	// Datastore returns the models.Datastore object. Not that this has arbitrary key value store methods that can be used to store extra data
+	Datastore() models.Datastore
+
+	// Set and Get values on the context, this can be useful to change behavior for the rest of the request
+	Set(key string, value interface{})
+	Get(key string) (value interface{}, exists bool)
+}

api/ifaces/listeners.go

@@ -0,0 +1,13 @@
+package ifaces
+
+import (
+	"github.com/iron-io/functions/api/models"
+	"golang.org/x/net/context"
+)
+
+type AppListener interface {
+	// BeforeAppUpdate called right before storing App in the database
+	BeforeAppUpdate(ctx context.Context, app *models.App) error
+	// AfterAppUpdate called after storing App in the database
+	AfterAppUpdate(ctx context.Context, app *models.App) error
+}

api/models/datastore.go

@@ -12,6 +12,11 @@ type Datastore interface {
 	GetRoutes(*RouteFilter) (routes []*Route, err error)
 	StoreRoute(*Route) (*Route, error)
 	RemoveRoute(appName, routeName string) error
+
+	// The following provide a generic key value store for arbitrary data, can be used by extensions to store extra data
+	// todo: should we namespace these by app? Then when an app is deleted, it can delete any of this extra data too.
+	Put([]byte, []byte) error
+	Get([]byte) ([]byte, error)
 }
 
 var (

api/server/apps_create.go

@@ -3,13 +3,16 @@ package server
 import (
 	"net/http"
 
-	"github.com/Sirupsen/logrus"
+	"golang.org/x/net/context"
+
 	"github.com/gin-gonic/gin"
 	"github.com/iron-io/functions/api/models"
+	titancommon "github.com/iron-io/titan/common"
 )
 
 func handleAppCreate(c *gin.Context) {
-	log := c.MustGet("log").(logrus.FieldLogger)
+	ctx := c.MustGet("ctx").(context.Context)
+	log := titancommon.Logger(ctx)
 
 	wapp := &models.AppWrapper{}
 
@@ -32,12 +35,26 @@ func handleAppCreate(c *gin.Context) {
 		return
 	}
 
+	err = Api.FireBeforeAppUpdate(ctx, wapp.App)
+	if err != nil {
+		log.WithError(err).Errorln(models.ErrAppsCreate)
+		c.JSON(http.StatusInternalServerError, simpleError(err))
+		return
+	}
+
 	app, err := Api.Datastore.StoreApp(wapp.App)
 	if err != nil {
-		log.WithError(err).Debug(models.ErrAppsCreate)
+		log.WithError(err).Errorln(models.ErrAppsCreate)
 		c.JSON(http.StatusInternalServerError, simpleError(models.ErrAppsCreate))
 		return
 	}
 
+	err = Api.FireAfterAppUpdate(ctx, wapp.App)
+	if err != nil {
+		log.WithError(err).Errorln(models.ErrAppsCreate)
+		c.JSON(http.StatusInternalServerError, simpleError(err))
+		return
+	}
+
 	c.JSON(http.StatusOK, app)
 }

api/server/apps_delete.go

@@ -3,13 +3,16 @@ package server
 import (
 	"net/http"
 
-	"github.com/Sirupsen/logrus"
+	"golang.org/x/net/context"
+
 	"github.com/gin-gonic/gin"
 	"github.com/iron-io/functions/api/models"
+	titancommon "github.com/iron-io/titan/common"
 )
 
 func handleAppDelete(c *gin.Context) {
-	log := c.MustGet("log").(logrus.FieldLogger)
+	ctx := c.MustGet("ctx").(context.Context)
+	log := titancommon.Logger(ctx)
 
 	appName := c.Param("app")
 	err := Api.Datastore.RemoveApp(appName)

api/server/apps_get.go

@@ -3,13 +3,16 @@ package server
 import (
 	"net/http"
 
-	"github.com/Sirupsen/logrus"
+	"golang.org/x/net/context"
+
 	"github.com/gin-gonic/gin"
 	"github.com/iron-io/functions/api/models"
+	titancommon "github.com/iron-io/titan/common"
 )
 
 func handleAppGet(c *gin.Context) {
-	log := c.MustGet("log").(logrus.FieldLogger)
+	ctx := c.MustGet("ctx").(context.Context)
+	log := titancommon.Logger(ctx)
 
 	appName := c.Param("app")
 	app, err := Api.Datastore.GetApp(appName)

api/server/apps_list.go

@@ -3,13 +3,16 @@ package server
 import (
 	"net/http"
 
-	"github.com/Sirupsen/logrus"
+	"golang.org/x/net/context"
+
 	"github.com/gin-gonic/gin"
 	"github.com/iron-io/functions/api/models"
+	titancommon "github.com/iron-io/titan/common"
 )
 
 func handleAppList(c *gin.Context) {
-	log := c.MustGet("log").(logrus.FieldLogger)
+	ctx := c.MustGet("ctx").(context.Context)
+	log := titancommon.Logger(ctx)
 
 	filter := &models.AppFilter{}
 

api/server/apps_update.go

@@ -3,13 +3,16 @@ package server
 import (
 	"net/http"
 
-	"github.com/Sirupsen/logrus"
+	"golang.org/x/net/context"
+
 	"github.com/gin-gonic/gin"
 	"github.com/iron-io/functions/api/models"
+	titancommon "github.com/iron-io/titan/common"
 )
 
 func handleAppUpdate(c *gin.Context) {
-	log := c.MustGet("log").(logrus.FieldLogger)
+	ctx := c.MustGet("ctx").(context.Context)
+	log := titancommon.Logger(ctx)
 
 	app := &models.App{}
 

api/server/routes_create.go

@@ -3,13 +3,16 @@ package server
 import (
 	"net/http"
 
-	"github.com/Sirupsen/logrus"
+	"golang.org/x/net/context"
+
 	"github.com/gin-gonic/gin"
 	"github.com/iron-io/functions/api/models"
+	titancommon "github.com/iron-io/titan/common"
 )
 
 func handleRouteCreate(c *gin.Context) {
-	log := c.MustGet("log").(logrus.FieldLogger)
+	ctx := c.MustGet("ctx").(context.Context)
+	log := titancommon.Logger(ctx)
 
 	var wroute models.RouteWrapper
 

api/server/routes_delete.go

@@ -3,13 +3,16 @@ package server
 import (
 	"net/http"
 
-	"github.com/Sirupsen/logrus"
+	"golang.org/x/net/context"
+
 	"github.com/gin-gonic/gin"
 	"github.com/iron-io/functions/api/models"
+	titancommon "github.com/iron-io/titan/common"
 )
 
 func handleRouteDelete(c *gin.Context) {
-	log := c.MustGet("log").(logrus.FieldLogger)
+	ctx := c.MustGet("ctx").(context.Context)
+	log := titancommon.Logger(ctx)
 
 	appName := c.Param("app")
 	routeName := c.Param("route")

api/server/routes_get.go

@@ -3,13 +3,17 @@ package server
 import (
 	"net/http"
 
+	"golang.org/x/net/context"
+
 	"github.com/Sirupsen/logrus"
 	"github.com/gin-gonic/gin"
 	"github.com/iron-io/functions/api/models"
+	titancommon "github.com/iron-io/titan/common"
 )
 
 func handleRouteGet(c *gin.Context) {
-	log := c.MustGet("log").(logrus.FieldLogger)
+	ctx := c.MustGet("ctx").(context.Context)
+	log := titancommon.Logger(ctx)
 
 	appName := c.Param("app")
 	routeName := c.Param("route")

api/server/routes_list.go

@@ -3,13 +3,17 @@ package server
 import (
 	"net/http"
 
+	"golang.org/x/net/context"
+
 	"github.com/Sirupsen/logrus"
 	"github.com/gin-gonic/gin"
 	"github.com/iron-io/functions/api/models"
+	titancommon "github.com/iron-io/titan/common"
 )
 
 func handleRouteList(c *gin.Context) {
-	log := c.MustGet("log").(logrus.FieldLogger)
+	ctx := c.MustGet("ctx").(context.Context)
+	log := titancommon.Logger(ctx)
 
 	appName := c.Param("app")
 

api/server/routes_update.go

@@ -3,13 +3,16 @@ package server
 import (
 	"net/http"
 
-	"github.com/Sirupsen/logrus"
+	"golang.org/x/net/context"
+
 	"github.com/gin-gonic/gin"
 	"github.com/iron-io/functions/api/models"
+	titancommon "github.com/iron-io/titan/common"
 )
 
 func handleRouteUpdate(c *gin.Context) {
-	log := c.MustGet("log").(logrus.FieldLogger)
+	ctx := c.MustGet("ctx").(context.Context)
+	log := titancommon.Logger(ctx)
 
 	var wroute models.RouteWrapper
 

api/server/runner.go

@@ -7,25 +7,40 @@ import (
 	"strings"
 	"time"
 
+	"golang.org/x/net/context"
+
 	"encoding/json"
 
 	"github.com/Sirupsen/logrus"
 	"github.com/gin-gonic/gin"
 	"github.com/iron-io/functions/api/models"
 	"github.com/iron-io/functions/api/runner"
+	titancommon "github.com/iron-io/titan/common"
 	"github.com/satori/go.uuid"
 )
 
+func handleSpecial(c *gin.Context) {
+	ctx := c.MustGet("ctx").(context.Context)
+	log := titancommon.Logger(ctx)
+
+	err := Api.UseSpecialHandlers(c)
+	if err != nil {
+		log.WithError(err).Errorln("Error using special handler!")
+		// todo: what do we do here?
+	}
+}
+
 func handleRunner(c *gin.Context) {
 	if strings.HasPrefix(c.Request.URL.Path, "/v1") {
 		c.Status(http.StatusNotFound)
 		return
 	}
 
-	log := c.MustGet("log").(logrus.FieldLogger)
+	ctx := c.MustGet("ctx").(context.Context)
+	log := titancommon.Logger(ctx)
 
 	reqID := uuid.NewV5(uuid.Nil, fmt.Sprintf("%s%s%d", c.Request.RemoteAddr, c.Request.URL.Path, time.Now().Unix())).String()
-	c.Set("reqID", reqID)
+	c.Set("reqID", reqID) // todo: put this in the ctx instead of gin's
 
 	log = log.WithFields(logrus.Fields{"request_id": reqID})
 
@@ -54,8 +69,17 @@ func handleRunner(c *gin.Context) {
 
 	appName := c.Param("app")
 	if appName == "" {
-		host := strings.Split(c.Request.Host, ":")[0]
+		// check context, app can be added via special handlers
-		appName = strings.Split(host, ".")[0]
+		a, ok := c.Get("app")
+		if ok {
+			appName = a.(string)
+		}
+	}
+	// if still no appName, we gotta exit
+	if appName == "" {
+		log.WithError(err).Error(models.ErrAppsNotFound)
+		c.JSON(http.StatusBadRequest, simpleError(models.ErrAppsNotFound))
+		return
 	}
 
 	route := c.Param("route")

api/server/server.go

@@ -3,17 +3,25 @@ package server
 import (
 	"path"
 
+	"golang.org/x/net/context"
+
 	"github.com/Sirupsen/logrus"
 	"github.com/gin-gonic/gin"
+	"github.com/iron-io/functions/api/ifaces"
 	"github.com/iron-io/functions/api/models"
+	titancommon "github.com/iron-io/titan/common"
 )
 
+// Would be nice to not have this is a global, but hard to pass things around to the
+// handlers in Gin without it.
 var Api *Server
 
 type Server struct {
-	Router    *gin.Engine
+	Router          *gin.Engine
-	Config    *models.Config
+	Config          *models.Config
-	Datastore models.Datastore
+	Datastore       models.Datastore
+	AppListeners    []ifaces.AppListener
+	SpecialHandlers []ifaces.SpecialHandler
 }
 
 func New(ds models.Datastore, config *models.Config) *Server {
@@ -25,6 +33,49 @@ func New(ds models.Datastore, config *models.Config) *Server {
 	return Api
 }
 
+// AddAppListener adds a listener that will be notified on App changes.
+func (s *Server) AddAppListener(listener ifaces.AppListener) {
+	s.AppListeners = append(s.AppListeners, listener)
+}
+
+func (s *Server) FireBeforeAppUpdate(ctx context.Context, app *models.App) error {
+	for _, l := range s.AppListeners {
+		err := l.BeforeAppUpdate(ctx, app)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (s *Server) FireAfterAppUpdate(ctx context.Context, app *models.App) error {
+	for _, l := range s.AppListeners {
+		err := l.AfterAppUpdate(ctx, app)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (s *Server) AddSpecialHandler(handler ifaces.SpecialHandler) {
+	s.SpecialHandlers = append(s.SpecialHandlers, handler)
+}
+
+func (s *Server) UseSpecialHandlers(ginC *gin.Context) error {
+	c := &SpecialHandlerContext{
+		server:     s,
+		ginContext: ginC,
+	}
+	for _, l := range s.SpecialHandlers {
+		err := l.Handle(c)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 func extractFields(c *gin.Context) logrus.Fields {
 	fields := logrus.Fields{"action": path.Base(c.HandlerName())}
 	for _, param := range c.Params {
@@ -33,9 +84,11 @@ func extractFields(c *gin.Context) logrus.Fields {
 	return fields
 }
 
-func (s *Server) Run() {
+func (s *Server) Run(ctx context.Context) {
+
 	s.Router.Use(func(c *gin.Context) {
-		c.Set("log", logrus.WithFields(extractFields(c)))
+		ctx, _ := titancommon.LoggerWithFields(ctx, extractFields(c))
+		c.Set("ctx", ctx)
 		c.Next()
 	})
 
@@ -66,11 +119,12 @@ func bindHandlers(engine *gin.Engine) {
 			apps.PUT("/routes/*route", handleRouteUpdate)
 			apps.DELETE("/routes/*route", handleRouteDelete)
 		}
-
 	}
 
 	engine.Any("/r/:app/*route", handleRunner)
-	engine.NoRoute(handleRunner)
+
+	// This final route is used for extensions, see Server.Add
+	engine.NoRoute(handleSpecial)
 }
 
 func simpleError(err error) *models.Error {

api/server/special_handler_context.go

@@ -0,0 +1,28 @@
+package server
+
+import (
+	"net/http"
+
+	"github.com/gin-gonic/gin"
+	"github.com/iron-io/functions/api/models"
+)
+
+type SpecialHandlerContext struct {
+	server     *Server
+	ginContext *gin.Context
+}
+
+func (c *SpecialHandlerContext) Request() *http.Request {
+	return c.ginContext.Request
+}
+
+func (c *SpecialHandlerContext) Datastore() models.Datastore {
+	return c.server.Datastore
+}
+
+func (c *SpecialHandlerContext) Set(key string, value interface{}) {
+	c.ginContext.Set(key, value)
+}
+func (c *SpecialHandlerContext) Get(key string) (value interface{}, exists bool) {
+	return c.ginContext.Get(key)
+}

api/server/version.go

@@ -6,6 +6,9 @@ import (
 	"github.com/gin-gonic/gin"
 )
 
+// Version of IronFunctions
+var Version = "0.0.1"
+
 func handleVersion(c *gin.Context) {
-	c.JSON(http.StatusNotImplemented, "Not Implemented")
+	c.JSON(http.StatusOK, gin.H{"version": Version})
 }

docs/api.md

@@ -0,0 +1 @@
+TODO: Add swagger URL. 

docs/extending.md

@@ -0,0 +1,13 @@
+IronFunctions is extensible so you can add custom functionality and extend the project without needing to modify the core. 
+
+## Listeners
+
+This is the main way to do it. To add listeners, copy main.go and use one of the following functions on the Server. 
+
+### AppListener
+
+Implement `ifaces/AppListener` interface, then add it using:
+
+```go
+server.AddAppListener(myAppListener)
+```

main.go

@@ -7,10 +7,11 @@ import (
 	"github.com/iron-io/functions/api/models"
 	"github.com/iron-io/functions/api/server"
 	"github.com/spf13/viper"
+	"golang.org/x/net/context"
 )
 
-// See comments below for how to extend Functions
 func main() {
+	ctx := context.Background()
 	c := &models.Config{}
 
 	config.InitConfig()
@@ -20,11 +21,11 @@ func main() {
 		log.WithError(err).Fatalln("Invalid config.")
 	}
 
-	ds, err := datastore.New(viper.GetString("db"))
+	ds, err := datastore.New(viper.GetString("DB"))
 	if err != nil {
 		log.WithError(err).Fatalln("Invalid DB url.")
 	}
 
 	srv := server.New(ds, c)
-	srv.Run()
+	srv.Run(ctx)
 }