Inverting deps on SQL, Log and MQ plugins to make them optional dependencies of extended servers, Removing some dead code that brought in unused dependencies Filtering out some non-linux transitive deps. (#1057)

* initial Db helper split - make SQL and datastore packages optional

* abstracting log store

* break out DB, MQ and log drivers as extensions

* cleanup

* fewer deps

* fixing docker test

* hmm dbness

* updating db startup

* Consolidate all your extensions into one convenient package

* cleanup

* clean up dep constraints

api/mqs/bolt/bolt.go

@@ -1,4 +1,4 @@
-package mqs
+package bolt
 
 import (
 	"context"
@@ -14,9 +14,12 @@ import (
 	"github.com/boltdb/bolt"
 	"github.com/fnproject/fn/api/common"
 	"github.com/fnproject/fn/api/models"
+	"github.com/fnproject/fn/api/mqs"
 	"github.com/sirupsen/logrus"
 )
 
+type boltProvider int
+
 type BoltDbMQ struct {
 	db     *bolt.DB
 	ticker *time.Ticker
@@ -52,7 +55,19 @@ func timeoutName(i int) []byte {
 	return []byte(fmt.Sprintf("functions_%d_timeout", i))
 }
 
-func NewBoltMQ(url *url.URL) (*BoltDbMQ, error) {
+func (boltProvider) Supports(url *url.URL) bool {
+	switch url.Scheme {
+	case "bolt":
+		return true
+	}
+	return false
+}
+
+func (boltProvider) String() string {
+	return "bolt"
+}
+
+func (boltProvider) New(url *url.URL) (models.MessageQueue, error) {
 	dir := filepath.Dir(url.Path)
 	log := logrus.WithFields(logrus.Fields{"mq": url.Scheme, "dir": dir})
 	err := os.MkdirAll(dir, 0755)
@@ -354,3 +369,7 @@ func (mq *BoltDbMQ) Close() error {
 	mq.ticker.Stop()
 	return mq.db.Close()
 }
+
+func init() {
+	mqs.AddProvider(boltProvider(0))
+}

api/mqs/memory/memory.go

@@ -1,18 +1,21 @@
-package mqs
+package memory
 
 import (
 	"context"
 	"errors"
-	"math/rand"
 	"sync"
 	"time"
 
 	"github.com/fnproject/fn/api/common"
 	"github.com/fnproject/fn/api/models"
+	"github.com/fnproject/fn/api/mqs"
 	"github.com/google/btree"
 	"github.com/sirupsen/logrus"
+	"net/url"
 )
 
+type memoryProvider int
+
 type MemoryMQ struct {
 	// WorkQueue A buffered channel that we can send work requests on.
 	PriorityQueues []chan *models.Call
@@ -26,20 +29,17 @@ type MemoryMQ struct {
 	Mutex sync.Mutex
 }
 
-var letters = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
-
-func randSeq(n int) string {
-	rand.Seed(time.Now().Unix())
-	b := make([]rune, n)
-	for i := range b {
-		b[i] = letters[rand.Intn(len(letters))]
-	}
-	return string(b)
-}
-
 const NumPriorities = 3
 
-func NewMemoryMQ() *MemoryMQ {
+func (memoryProvider) Supports(url *url.URL) bool {
+	switch url.Scheme {
+	case "memory":
+		return true
+	}
+	return false
+}
+
+func (memoryProvider) New(url *url.URL) (models.MessageQueue, error) {
 	var queues []chan *models.Call
 	for i := 0; i < NumPriorities; i++ {
 		queues = append(queues, make(chan *models.Call, 5000))
@@ -53,7 +53,11 @@ func NewMemoryMQ() *MemoryMQ {
 	}
 	mq.start()
 	logrus.Info("MemoryMQ initialized")
-	return mq
+	return mq, nil
+}
+
+func (memoryProvider) String() string {
+	return "memory"
 }
 
 func (mq *MemoryMQ) start() {
@@ -198,3 +202,7 @@ func (mq *MemoryMQ) Close() error {
 	mq.Ticker.Stop()
 	return nil
 }
+
+func init() {
+	mqs.AddProvider(memoryProvider(0))
+}

api/mqs/new.go

@@ -11,6 +11,22 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
+// Provider for message queue extensions
+type Provider interface {
+	fmt.Stringer
+	//Supports indicates if this provider can handle a specific URL scheme
+	Supports(url *url.URL) bool
+	//New creates a new message queue from a given URL
+	New(url *url.URL) (models.MessageQueue, error)
+}
+
+var mqProviders []Provider
+
+// AddProvider registers a new global message queue provider
+func AddProvider(p Provider) {
+	mqProviders = append(mqProviders, p)
+}
+
 // New will parse the URL and return the correct MQ implementation.
 func New(mqURL string) (models.MessageQueue, error) {
 	mq, err := newmq(mqURL)
@@ -27,15 +43,11 @@ func newmq(mqURL string) (models.MessageQueue, error) {
 		logrus.WithError(err).WithFields(logrus.Fields{"url": mqURL}).Fatal("bad MQ URL")
 	}
 	logrus.WithFields(logrus.Fields{"mq": u.Scheme}).Debug("selecting MQ")
-	switch u.Scheme {
-	case "memory":
-		return NewMemoryMQ(), nil
-	case "redis":
-		return NewRedisMQ(u)
-	case "bolt":
-		return NewBoltMQ(u)
+	for _, p := range mqProviders {
+		if p.Supports(u) {
+			return p.New(u)
+		}
 	}
-
 	return nil, fmt.Errorf("mq type not supported %v", u.Scheme)
 }
 

api/mqs/redis/redis.go

@@ -1,4 +1,4 @@
-package mqs
+package redis
 
 import (
 	"context"
@@ -11,6 +11,7 @@ import (
 
 	"github.com/fnproject/fn/api/common"
 	"github.com/fnproject/fn/api/models"
+	"github.com/fnproject/fn/api/mqs"
 	"github.com/garyburd/redigo/redis"
 	"github.com/sirupsen/logrus"
 )
@@ -22,7 +23,21 @@ type RedisMQ struct {
 	prefix    string
 }
 
-func NewRedisMQ(url *url.URL) (*RedisMQ, error) {
+type redisProvider int
+
+func (redisProvider) Supports(url *url.URL) bool {
+	switch url.Scheme {
+	case "redis":
+		return true
+	}
+	return false
+}
+
+func (redisProvider) String() string {
+	return "redis"
+}
+
+func (redisProvider) New(url *url.URL) (models.MessageQueue, error) {
 	pool := &redis.Pool{
 		MaxIdle: 512,
 		// I'm not sure if allowing the pool to block if more than 16 connections are required is a good idea.
@@ -314,3 +329,7 @@ func (mq *RedisMQ) Close() error {
 	mq.ticker.Stop()
 	return mq.pool.Close()
 }
+
+func init() {
+	mqs.AddProvider(redisProvider(0))
+}