Inverting deps on SQL, Log and MQ plugins to make them optional dependencies of extended servers, Removing some dead code that brought in unused dependencies Filtering out some non-linux transitive deps. (#1057)

* initial Db helper split - make SQL and datastore packages optional

* abstracting log store

* break out DB, MQ and log drivers as extensions

* cleanup

* fewer deps

* fixing docker test

* hmm dbness

* updating db startup

* Consolidate all your extensions into one convenient package

* cleanup

* clean up dep constraints

api/logs/log.go

@@ -7,14 +7,30 @@ import (
 	"context"
 
 	"github.com/fnproject/fn/api/common"
-	"github.com/fnproject/fn/api/datastore/sql"
 	"github.com/fnproject/fn/api/logs/metrics"
-	"github.com/fnproject/fn/api/logs/s3"
 	"github.com/fnproject/fn/api/logs/validator"
 	"github.com/fnproject/fn/api/models"
 	"github.com/sirupsen/logrus"
 )
 
+// Provider defines a source that can create log stores
+type Provider interface {
+	fmt.Stringer
+	// Supports indicates if this provider can handle a specific URL scheme
+	Supports(url *url.URL) bool
+	//Create a new log store from the corresponding URL
+	New(ctx context.Context, url *url.URL) (models.LogStore, error)
+}
+
+var providers []Provider
+
+// AddProvider globally registers a new LogStore provider
+func AddProvider(pf Provider) {
+	logrus.Info("Adding log provider %s", pf)
+	providers = append(providers, pf)
+}
+
+// New Creates a new log store based on a given URL
 func New(ctx context.Context, dbURL string) (models.LogStore, error) {
 	log := common.Logger(ctx)
 	u, err := url.Parse(dbURL)
@@ -22,17 +38,13 @@ func New(ctx context.Context, dbURL string) (models.LogStore, error) {
 		log.WithError(err).WithFields(logrus.Fields{"url": dbURL}).Fatal("bad DB URL")
 	}
 	log.WithFields(logrus.Fields{"db": u.Scheme}).Debug("creating log store")
-	var ls models.LogStore
-	switch u.Scheme {
-	case "sqlite3", "postgres", "mysql":
-		ls, err = sql.New(ctx, u)
-	case "s3":
-		ls, err = s3.New(u)
-	default:
-		err = fmt.Errorf("db type not supported %v", u.Scheme)
-	}
 
-	return ls, err
+	for _, p := range providers {
+		if p.Supports(u) {
+			return p.New(ctx, u)
+		}
+	}
+	return nil, fmt.Errorf("no log store provider available for url %s", dbURL)
 }
 
 func Wrap(ls models.LogStore) models.LogStore {

api/logs/s3/s3.go

@@ -21,6 +21,7 @@ import (
 	"github.com/aws/aws-sdk-go/service/s3/s3manager"
 	"github.com/fnproject/fn/api/common"
 	"github.com/fnproject/fn/api/id"
+	"github.com/fnproject/fn/api/logs"
 	"github.com/fnproject/fn/api/models"
 	"github.com/sirupsen/logrus"
 	"go.opencensus.io/stats"
@@ -49,6 +50,8 @@ type store struct {
 	bucket     string
 }
 
+type s3StoreProvider int
+
 // decorator around the Reader interface that keeps track of the number of bytes read
 // in order to avoid double buffering and track Reader size
 type countingReader struct {
@@ -80,10 +83,18 @@ func createStore(bucketName, endpoint, region, accessKeyID, secretAccessKey stri
 	}
 }
 
+func (s3StoreProvider) String() string {
+	return "s3"
+}
+
+func (s3StoreProvider) Supports(u *url.URL) bool {
+	return u.Scheme == "s3"
+}
+
 // New returns an s3 api compatible log store.
 // url format: s3://access_key_id:secret_access_key@host/region/bucket_name?ssl=true
 // Note that access_key_id and secret_access_key must be URL encoded if they contain unsafe characters!
-func New(u *url.URL) (models.LogStore, error) {
+func (s3StoreProvider) New(ctx context.Context, u *url.URL) (models.LogStore, error) {
 	endpoint := u.Host
 
 	var accessKeyID, secretAccessKey string
@@ -464,3 +475,7 @@ func init() {
 func (s *store) Close() error {
 	return nil
 }
+
+func init() {
+	logs.AddProvider(s3StoreProvider(0))
+}

api/logs/s3/s3_test.go

@@ -5,6 +5,7 @@ import (
 	"os"
 	"testing"
 
+	"context"
 	logTesting "github.com/fnproject/fn/api/logs/testing"
 )
 
@@ -20,7 +21,7 @@ func TestS3(t *testing.T) {
 		t.Fatalf("failed to parse url: %v", err)
 	}
 
-	ls, err := New(uLog)
+	ls, err := s3StoreProvider(0).New(context.Background(), uLog)
 	if err != nil {
 		t.Fatalf("failed to create s3 datastore: %v", err)
 	}