alihan/fn-serverless
1
0
Fork 0
mirror of https://github.com/fnproject/fn.git synced 2022-10-28 21:29:17 +03:00
Code Issues Packages Projects Releases Wiki Activity

Inverting deps on SQL, Log and MQ plugins to make them optional dependencies of extended servers, Removing some dead code that brought in unused dependencies Filtering out some non-linux transitive deps. (#1057)

Browse Source 
* initial Db helper split - make SQL and datastore packages optional

* abstracting log store

* break out DB, MQ and log drivers as extensions

* cleanup

* fewer deps

* fixing docker test

* hmm dbness

* updating db startup

* Consolidate all your extensions into one convenient package

* cleanup

* clean up dep constraints
This commit is contained in:
Owen Cliffe
2018-06-11 18:23:28 +01:00
committed by Reed Allman
parent 5b2691037e
commit 1ad27f4f0d
519 changed files with 907 additions and 91042 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
api/agent/drivers/docker/docker_test.go
View File

@@ -9,7 +9,6 @@ import (
"time"
"github.com/fnproject/fn/api/agent/drivers"
"github.com/fsouza/go-dockerclient"
)
type taskDockerTest struct {
@@ -170,20 +169,21 @@ func TestRunnerDockerStdin(t *testing.T) {
}
}
func TestRegistry(t *testing.T) {
image := "fnproject/fn-test-utils"
sizer, err := CheckRegistry(context.Background(), image, docker.AuthConfiguration{})
if err != nil {
t.Fatal("expected registry check not to fail, got:", err)
}
size, err := sizer.Size()
if err != nil {
t.Fatal("expected sizer not to fail, got:", err)
}
if size <= 0 {
t.Fatal("expected positive size for image that exists, got size:", size)
}
}
//
//func TestRegistry(t *testing.T) {
// image := "fnproject/fn-test-utils"
//
// sizer, err := CheckRegistry(context.Background(), image, docker.AuthConfiguration{})
// if err != nil {
// t.Fatal("expected registry check not to fail, got:", err)
// }
//
// size, err := sizer.Size()
// if err != nil {
// t.Fatal("expected sizer not to fail, got:", err)
// }
//
// if size <= 0 {
// t.Fatal("expected positive size for image that exists, got size:", size)
// }
//}

181
api/agent/drivers/docker/registry.go
View File

@@ -1,193 +1,12 @@
package docker
import (
"context"
"crypto/tls"
"io"
"io/ioutil"
"net"
"net/http"
"net/url"
"strings"
"time"
"github.com/docker/distribution"
"github.com/docker/distribution/manifest/schema1"
"github.com/docker/distribution/manifest/schema2"
"github.com/docker/distribution/reference"
registry "github.com/docker/distribution/registry/client"
"github.com/docker/distribution/registry/client/auth"
"github.com/docker/distribution/registry/client/auth/challenge"
"github.com/docker/distribution/registry/client/transport"
"github.com/fnproject/fn/api/agent/drivers"
docker "github.com/fsouza/go-dockerclient"
)
var (
// we need these imported so that they can be unmarshaled properly (yes, docker is mean)
_ = schema1.SchemaVersion
_ = schema2.SchemaVersion
registryTransport = &http.Transport{
Dial: (&net.Dialer{
Timeout: 10 * time.Second,
KeepAlive: 2 * time.Minute,
}).Dial,
TLSClientConfig: &tls.Config{
ClientSessionCache: tls.NewLRUClientSessionCache(8192),
},
TLSHandshakeTimeout: 10 * time.Second,
MaxIdleConnsPerHost: 32, // TODO tune; we will likely be making lots of requests to same place
Proxy: http.ProxyFromEnvironment,
IdleConnTimeout: 90 * time.Second,
MaxIdleConns: 512,
ExpectContinueTimeout: 1 * time.Second,
}
)
const hubURL = "https://registry.hub.docker.com"
// CheckRegistry will return a sizer, which can be used to check the size of an
// image if the returned error is nil. If the error returned is nil, then
// authentication against the given credentials was successful, if the
// configuration or image do not specify a config.ServerAddress,
// https://hub.docker.com will be tried. CheckRegistry is a package level
// method since rkt can also use docker images, we may be interested in using
// rkt w/o a docker driver configured; also, we don't have to tote around a
// driver in any tasker that may be interested in registry information (2/2
// cases thus far).
func CheckRegistry(ctx context.Context, image string, config docker.AuthConfiguration) (Sizer, error) {
regURL, repoName, tag := drivers.ParseImage(image)
repoNamed, err := reference.WithName(repoName)
if err != nil {
return nil, err
}
if regURL == "" {
// image address overrides credential address
regURL = config.ServerAddress
}
regURL, err = registryURL(regURL)
if err != nil {
return nil, err
}
cm := challenge.NewSimpleManager()
creds := newCreds(config.Username, config.Password)
tran := transport.NewTransport(registryTransport,
auth.NewAuthorizer(cm,
auth.NewTokenHandler(registryTransport,
creds,
repoNamed.Name(),
"pull",
),
auth.NewBasicHandler(creds),
),
)
tran = &retryWrap{cm, tran}
repo, err := registry.NewRepository(repoNamed, regURL, tran)
if err != nil {
return nil, err
}
manis, err := repo.Manifests(ctx)
if err != nil {
return nil, err
}
mani, err := manis.Get(context.TODO(), "", distribution.WithTag(tag))
if err != nil {
return nil, err
}
blobs := repo.Blobs(ctx)
// most registries aren't that great, and won't provide a size for the top
// level digest, so we need to sum up all the layers. let this be optional
// with the sizer, since tag is good enough to check existence / auth.
return &sizer{mani, blobs}, nil
}
type retryWrap struct {
cm challenge.Manager
tran http.RoundTripper
}
func (d *retryWrap) RoundTrip(req *http.Request) (*http.Response, error) {
resp, err := d.tran.RoundTrip(req)
// if it's not authed, we have to add this to the challenge manager,
// and then retry it (it will get authed and the challenge then accepted).
// why the docker distribution transport doesn't do this for you is
// a real testament to what sadists those docker people are.
if resp != nil && resp.StatusCode == http.StatusUnauthorized {
pingPath := req.URL.Path
if v2Root := strings.Index(req.URL.Path, "/v2/"); v2Root != -1 {
pingPath = pingPath[:v2Root+4]
} else if v1Root := strings.Index(req.URL.Path, "/v1/"); v1Root != -1 {
pingPath = pingPath[:v1Root] + "/v2/"
}
// seriously, we have to rewrite this to the ping path,
// since looking up challenges strips to this path. YUP. GLHF.
ogURL := req.URL.Path
resp.Request.URL.Path = pingPath
d.cm.AddResponse(resp)
io.Copy(ioutil.Discard, resp.Body)
resp.Body.Close()
// put the original URL path back and try again now...
req.URL.Path = ogURL
resp, err = d.tran.RoundTrip(req)
}
return resp, err
}
func newCreds(user, pass string) *creds {
return &creds{m: make(map[string]string), user: user, pass: pass}
}
// implement auth.CredentialStore
type creds struct {
m map[string]string
user, pass string
}
func (c *creds) Basic(u *url.URL) (string, string) { return c.user, c.pass }
func (c *creds) RefreshToken(u *url.URL, service string) string { return c.m[service] }
func (c *creds) SetRefreshToken(u *url.URL, service, token string) { c.m[service] = token }
// Sizer returns size information. This interface is liable to contain more
// than a size at some point, change as needed.
type Sizer interface {
Size() (int64, error)
}
type sizer struct {
mani distribution.Manifest
blobs distribution.BlobStore
}
func (s *sizer) Size() (int64, error) {
var sum int64
for _, r := range s.mani.References() {
desc, err := s.blobs.Stat(context.TODO(), r.Digest)
if err != nil {
return 0, err
}
sum += desc.Size
}
return sum, nil
}
func registryURL(addr string) (string, error) {
if addr == "" || strings.Contains(addr, "hub.docker.com") || strings.Contains(addr, "index.docker.io") {
return hubURL, nil

44
api/datastore/datastore.go
View File

@@ -2,35 +2,49 @@ package datastore
import (
"context"
"fmt"
"net/url"
"fmt"
"github.com/fnproject/fn/api/common"
"github.com/fnproject/fn/api/datastore/internal/datastoreutil"
"github.com/fnproject/fn/api/datastore/sql"
"github.com/fnproject/fn/api/models"
"github.com/sirupsen/logrus"
)
// New creates a DataStore from the specified URL
func New(ctx context.Context, dbURL string) (models.Datastore, error) {
return newds(ctx, dbURL) // teehee
}
func Wrap(ds models.Datastore) models.Datastore {
return datastoreutil.MetricDS(datastoreutil.NewValidator(ds))
}
func newds(ctx context.Context, dbURL string) (models.Datastore, error) {
log := common.Logger(ctx)
u, err := url.Parse(dbURL)
if err != nil {
log.WithError(err).WithFields(logrus.Fields{"url": dbURL}).Fatal("bad DB URL")
}
log.WithFields(logrus.Fields{"db": u.Scheme}).Debug("creating new datastore")
switch u.Scheme {
case "sqlite3", "postgres", "mysql":
return sql.New(ctx, u)
default:
return nil, fmt.Errorf("db type not supported %v", u.Scheme)
for _, provider := range providers {
if provider.Supports(u) {
return provider.New(ctx, u)
}
}
return nil, fmt.Errorf("no data store provider found for storage url %s", u)
}
func Wrap(ds models.Datastore) models.Datastore {
return datastoreutil.MetricDS(datastoreutil.NewValidator(ds))
}
// Provider is a datastore provider
type Provider interface {
fmt.Stringer
// Supports indicates if this provider can handle a given data store.
Supports(url *url.URL) bool
// New creates a new data store from the specified URL
New(ctx context.Context, url *url.URL) (models.Datastore, error)
}
var providers []Provider
// AddProvider globally registers a data store provider
func AddProvider(provider Provider) {
logrus.Infof("Adding DataStore provider %s", provider)
providers = append(providers, provider)
}

43
api/datastore/sql/dbhelper/dbhelper.go Normal file
View File

@@ -0,0 +1,43 @@
// dbhelpers wrap SQL and specific capabilities of an SQL db
package dbhelper
import (
"fmt"
"github.com/jmoiron/sqlx"
"github.com/sirupsen/logrus"
"net/url"
)
var sqlHelpers []Helper
//Add registers a new SQL helper
func Add(helper Helper) {
logrus.Infof("Registering DB helper %s", helper)
sqlHelpers = append(sqlHelpers, helper)
}
//Helper provides DB-specific SQL capabilities
type Helper interface {
fmt.Stringer
//Supports indicates if this helper supports this driver name
Supports(driverName string) bool
//PreConnect calculates the connect URL for the db from a canonical URL used in Fn config
PreConnect(url *url.URL) (string, error)
//PostCreate Apply any configuration to the DB prior to use
PostCreate(db *sqlx.DB) (*sqlx.DB, error)
//CheckTableExists checks if a table exists in the DB
CheckTableExists(tx *sqlx.Tx, table string) (bool, error)
//IsDuplicateKeyError determines if an error indicates if the prior error was caused by a duplicate key insert
IsDuplicateKeyError(err error) bool
}
//GetHelper returns a helper for a specific driver
func GetHelper(driverName string) (Helper, bool) {
for _, helper := range sqlHelpers {
if helper.Supports(driverName) {
return helper, true
}
logrus.Printf("%s does not support %s", helper, driverName)
}
return nil, false
}

34
api/datastore/sql/migratex/migrate.go
View File

@@ -9,9 +9,8 @@ import (
"sort"
"strings"
"github.com/go-sql-driver/mysql"
"github.com/fnproject/fn/api/datastore/sql/dbhelper"
"github.com/jmoiron/sqlx"
"github.com/lib/pq"
"github.com/sirupsen/logrus"
)
@@ -302,28 +301,29 @@ func SetVersion(ctx context.Context, tx *sqlx.Tx, version int64, dirty bool) err
}
func Version(ctx context.Context, tx *sqlx.Tx) (version int64, dirty bool, err error) {
helper, ok := dbhelper.GetHelper(tx.DriverName())
if !ok {
return 0, false, fmt.Errorf("no db helper registered for for %s", tx.DriverName())
}
tableExists, err := helper.CheckTableExists(tx, MigrationsTable)
if err != nil {
return 0, false, err
}
if !tableExists {
return NilVersion, false, nil
}
query := tx.Rebind(`SELECT version, dirty FROM ` + MigrationsTable + ` LIMIT 1`)
err = tx.QueryRowContext(ctx, query).Scan(&version, &dirty)
switch {
case err == sql.ErrNoRows:
return NilVersion, false, nil
case err != nil:
if e, ok := err.(*mysql.MySQLError); ok {
if e.Number == 0 {
return NilVersion, false, nil
}
}
if e, ok := err.(*pq.Error); ok {
if e.Code.Name() == "undefined_table" {
return NilVersion, false, nil
}
}
// sqlite3 returns 'no such table' but the error is not typed
if strings.Contains(err.Error(), "no such table") {
return NilVersion, false, nil
}
return 0, false, err
default:

2
api/datastore/sql/migratex/migrate_test.go
View File

@@ -6,8 +6,8 @@ import (
"fmt"
"testing"
_ "github.com/fnproject/fn/api/datastore/sql/sqlite"
"github.com/jmoiron/sqlx"
_ "github.com/mattn/go-sqlite3"
)
const testsqlite3 = "file::memory:?mode=memory&cache=shared"

61
api/datastore/sql/mysql/mysql.go Normal file
View File

@@ -0,0 +1,61 @@
package mysql
import (
"fmt"
"github.com/fnproject/fn/api/datastore/sql/dbhelper"
"github.com/go-sql-driver/mysql"
_ "github.com/go-sql-driver/mysql"
"github.com/jmoiron/sqlx"
"net/url"
"strings"
)
type mysqlHelper int
func (mysqlHelper) Supports(scheme string) bool {
return scheme == "mysql"
}
func (mysqlHelper) PreConnect(url *url.URL) (string, error) {
return strings.TrimPrefix(url.String(), url.Scheme+"://"), nil
}
func (mysqlHelper) PostCreate(db *sqlx.DB) (*sqlx.DB, error) {
return db, nil
}
func (mysqlHelper) CheckTableExists(tx *sqlx.Tx, table string) (bool, error) {
query := tx.Rebind(fmt.Sprintf(`SELECT count(*)
FROM information_schema.TABLES
WHERE TABLE_NAME = '%s'
`, table))
row := tx.QueryRow(query)
var count int
err := row.Scan(&count)
if err != nil {
return false, err
}
exists := count > 0
return exists, nil
}
func (mysqlHelper) String() string {
return "mysql"
}
func (mysqlHelper) IsDuplicateKeyError(err error) bool {
switch mErr := err.(type) {
case *mysql.MySQLError:
if mErr.Number == 1062 {
return true
}
}
return false
}
func init() {
dbhelper.Add(mysqlHelper(0))
}

63
api/datastore/sql/postgres/postgres.go Normal file
View File

@@ -0,0 +1,63 @@
package postgres
import (
"github.com/fnproject/fn/api/datastore/sql/dbhelper"
_ "github.com/go-sql-driver/mysql"
"github.com/jmoiron/sqlx"
"github.com/lib/pq"
"net/url"
)
type postgresHelper int
func (postgresHelper) Supports(scheme string) bool {
switch scheme {
case "postgres", "pgx":
return true
}
return false
}
func (postgresHelper) PreConnect(url *url.URL) (string, error) {
return url.String(), nil
}
func (postgresHelper) PostCreate(db *sqlx.DB) (*sqlx.DB, error) {
return db, nil
}
func (postgresHelper) CheckTableExists(tx *sqlx.Tx, table string) (bool, error) {
query := tx.Rebind(`SELECT count(*)
FROM information_schema.TABLES
WHERE TABLE_NAME = 'apps'
`)
row := tx.QueryRow(query)
var count int
err := row.Scan(&count)
if err != nil {
return false, err
}
exists := count > 0
return exists, nil
}
func (postgresHelper) String() string {
return "postgres"
}
func (postgresHelper) IsDuplicateKeyError(err error) bool {
switch dbErr := err.(type) {
case *pq.Error:
if dbErr.Code == "23505" {
return true
}
}
return false
}
func init() {
dbhelper.Add(postgresHelper(0))
}

137
api/datastore/sql/sql.go
View File

@@ -4,12 +4,10 @@ import (
"bytes"
"context"
"database/sql"
"errors"
"fmt"
"io"
"net/url"
"os"
"path/filepath"
"strconv"
"strings"
"time"
@@ -18,13 +16,11 @@ import (
"github.com/fnproject/fn/api/datastore/sql/migratex"
"github.com/fnproject/fn/api/datastore/sql/migrations"
"github.com/fnproject/fn/api/models"
"github.com/go-sql-driver/mysql"
_ "github.com/go-sql-driver/mysql"
"github.com/jmoiron/sqlx"
"github.com/lib/pq"
_ "github.com/lib/pq"
"github.com/mattn/go-sqlite3"
_ "github.com/mattn/go-sqlite3"
"github.com/fnproject/fn/api/datastore"
"github.com/fnproject/fn/api/datastore/sql/dbhelper"
"github.com/fnproject/fn/api/logs"
"github.com/sirupsen/logrus"
)
@@ -103,13 +99,44 @@ var ( // compiler will yell nice things about our upbringing as a child
)
type SQLStore struct {
db *sqlx.DB
helper dbhelper.Helper
db *sqlx.DB
}
type sqlDsProvider int
// New will open the db specified by url, create any tables necessary
// and return a models.Datastore safe for concurrent usage.
func New(ctx context.Context, url *url.URL) (*SQLStore, error) {
return newDS(ctx, url)
func New(ctx context.Context, u *url.URL) (*SQLStore, error) {
return newDS(ctx, u)
}
func (sqlDsProvider) Supports(u *url.URL) bool {
_, ok := dbhelper.GetHelper(u.Scheme)
return ok
}
func (sqlDsProvider) New(ctx context.Context, u *url.URL) (models.Datastore, error) {
return newDS(ctx, u)
}
func (sqlDsProvider) String() string {
return "sql"
}
type sqlLogsProvider int
func (sqlLogsProvider) String() string {
return "sql"
}
func (sqlLogsProvider) Supports(u *url.URL) bool {
_, ok := dbhelper.GetHelper(u.Scheme)
return ok
}
func (sqlLogsProvider) New(ctx context.Context, u *url.URL) (models.LogStore, error) {
return newDS(ctx, u)
}
// for test methods, return concrete type, but don't expose
@@ -117,27 +144,19 @@ func newDS(ctx context.Context, url *url.URL) (*SQLStore, error) {
driver := url.Scheme
log := common.Logger(ctx)
// driver must be one of these for sqlx to work, double check:
switch driver {
case "postgres", "pgx", "mysql", "sqlite3":
default:
return nil, errors.New("invalid db driver, refer to the code")
helper, ok := dbhelper.GetHelper(driver)
if !ok {
return nil, fmt.Errorf("DB helper '%s' is not supported", driver)
}
if driver == "sqlite3" {
// make all the dirs so we can make the file..
dir := filepath.Dir(url.Path)
err := os.MkdirAll(dir, 0755)
if err != nil {
return nil, err
}
uri, err := helper.PreConnect(url)
if err != nil {
return nil, fmt.Errorf("failed to initialise db helper %s : %s", driver, err)
}
uri := url.String()
if driver != "postgres" {
// postgres seems to need this as a prefix in lib/pq, everyone else wants it stripped of scheme
uri = strings.TrimPrefix(url.String(), url.Scheme+"://")
}
log.WithFields(logrus.Fields{"url": uri}).Info("Connecting to DB")
sqldb, err := sql.Open(driver, uri)
if err != nil {
@@ -158,12 +177,12 @@ func newDS(ctx context.Context, url *url.URL) (*SQLStore, error) {
db.SetMaxIdleConns(maxIdleConns)
log.WithFields(logrus.Fields{"max_idle_connections": maxIdleConns, "datastore": driver}).Info("datastore dialed")
switch driver { // NOTE: fixes weird sqlite3 behavior
case "sqlite3":
db.SetMaxOpenConns(1)
db, err = helper.PostCreate(db)
if err != nil {
log.WithFields(logrus.Fields{"url": uri}).WithError(err).Error("couldn't initialize db")
return nil, err
}
sdb := &SQLStore{db: db}
sdb := &SQLStore{db: db, helper: helper}
// NOTE: runMigrations happens before we create all the tables, so that it
// can detect whether the db did not exist and insert the latest version of
@@ -231,41 +250,11 @@ func pingWithRetry(ctx context.Context, db *sqlx.DB) (err error) {
return err
}
// checkExistence checks if tables have been created yet, it is not concerned
// about the existence of the schema migration version (since migrations were
// added to existing dbs, we need to know whether the db exists without migrations
// or if it's brand new).
func checkExistence(tx *sqlx.Tx) (bool, error) {
query := tx.Rebind(`SELECT count(*)
FROM information_schema.TABLES
WHERE TABLE_NAME = 'apps'
`)
if tx.DriverName() == "sqlite3" {
// sqlite3 is special, of course
query = tx.Rebind(`SELECT count(*)
FROM sqlite_master
WHERE name = 'apps'
`)
}
row := tx.QueryRow(query)
var count int
err := row.Scan(&count)
if err != nil {
return false, err
}
exists := count > 0
return exists, nil
}
// check if the db already existed, if the db is brand new then we can skip
// over all the migrations BUT we must be sure to set the right migration
// number so that only current migrations are skipped, not any future ones.
func (ds *SQLStore) runMigrations(ctx context.Context, tx *sqlx.Tx, migrations []migratex.Migration) error {
dbExists, err := checkExistence(tx)
dbExists, err := ds.helper.CheckTableExists(tx, "apps")
if err != nil {
return err
}
@@ -355,19 +344,8 @@ func (ds *SQLStore) InsertApp(ctx context.Context, app *models.App) (*models.App
);`)
_, err := ds.db.NamedExecContext(ctx, query, app)
if err != nil {
switch err := err.(type) {
case *mysql.MySQLError:
if err.Number == 1062 {
return nil, models.ErrAppsAlreadyExists
}
case *pq.Error:
if err.Code == "23505" {
return nil, models.ErrAppsAlreadyExists
}
case sqlite3.Error:
if err.ExtendedCode == sqlite3.ErrConstraintUnique || err.ExtendedCode == sqlite3.ErrConstraintPrimaryKey {
return nil, models.ErrAppsAlreadyExists
}
if ds.helper.IsDuplicateKeyError(err) {
return nil, models.ErrAppsAlreadyExists
}
return nil, err
}
@@ -924,3 +902,8 @@ func (ds *SQLStore) GetDatabase() *sqlx.DB {
func (ds *SQLStore) Close() error {
return ds.db.Close()
}
func init() {
datastore.AddProvider(sqlDsProvider(0))
logs.AddProvider(sqlLogsProvider(0))
}

3
api/datastore/sql/sql_test.go
View File

@@ -10,6 +10,9 @@ import (
"github.com/fnproject/fn/api/datastore/internal/datastoreutil"
"github.com/fnproject/fn/api/datastore/sql/migratex"
"github.com/fnproject/fn/api/datastore/sql/migrations"
_ "github.com/fnproject/fn/api/datastore/sql/mysql"
_ "github.com/fnproject/fn/api/datastore/sql/postgres"
_ "github.com/fnproject/fn/api/datastore/sql/sqlite"
logstoretest "github.com/fnproject/fn/api/logs/testing"
"github.com/fnproject/fn/api/models"
"github.com/jmoiron/sqlx"

74
api/datastore/sql/sqlite/sqlite.go Normal file
View File

@@ -0,0 +1,74 @@
package sqlite
import (
"fmt"
"github.com/fnproject/fn/api/datastore/sql/dbhelper"
"github.com/jmoiron/sqlx"
"github.com/mattn/go-sqlite3"
"net/url"
"os"
"path/filepath"
"strings"
)
type sqliteHelper int
func (sqliteHelper) Supports(scheme string) bool {
switch scheme {
case "sqlite3", "sqlite":
return true
}
return false
}
func (sqliteHelper) PreConnect(url *url.URL) (string, error) {
// make all the dirs so we can make the file..
dir := filepath.Dir(url.Path)
err := os.MkdirAll(dir, 0755)
if err != nil {
return "", err
}
return strings.TrimPrefix(url.String(), url.Scheme+"://"), nil
}
func (sqliteHelper) PostCreate(db *sqlx.DB) (*sqlx.DB, error) {
db.SetMaxOpenConns(1)
return db, nil
}
func (sqliteHelper) CheckTableExists(tx *sqlx.Tx, table string) (bool, error) {
query := tx.Rebind(fmt.Sprintf(`SELECT count(*)
FROM sqlite_master
WHERE name = '%s'
`, table))
row := tx.QueryRow(query)
var count int
err := row.Scan(&count)
if err != nil {
return false, err
}
exists := count > 0
return exists, nil
}
func (sqliteHelper) String() string {
return "sqlite"
}
func (sqliteHelper) IsDuplicateKeyError(err error) bool {
sqliteErr, ok := err.(sqlite3.Error)
if ok {
if sqliteErr.ExtendedCode == sqlite3.ErrConstraintUnique || sqliteErr.ExtendedCode == sqlite3.ErrConstraintPrimaryKey {
return true
}
}
return false
}
func init() {
dbhelper.Add(sqliteHelper(0))
}

36
api/logs/log.go
View File

@@ -7,14 +7,30 @@ import (
"context"
"github.com/fnproject/fn/api/common"
"github.com/fnproject/fn/api/datastore/sql"
"github.com/fnproject/fn/api/logs/metrics"
"github.com/fnproject/fn/api/logs/s3"
"github.com/fnproject/fn/api/logs/validator"
"github.com/fnproject/fn/api/models"
"github.com/sirupsen/logrus"
)
// Provider defines a source that can create log stores
type Provider interface {
fmt.Stringer
// Supports indicates if this provider can handle a specific URL scheme
Supports(url *url.URL) bool
//Create a new log store from the corresponding URL
New(ctx context.Context, url *url.URL) (models.LogStore, error)
}
var providers []Provider
// AddProvider globally registers a new LogStore provider
func AddProvider(pf Provider) {
logrus.Info("Adding log provider %s", pf)
providers = append(providers, pf)
}
// New Creates a new log store based on a given URL
func New(ctx context.Context, dbURL string) (models.LogStore, error) {
log := common.Logger(ctx)
u, err := url.Parse(dbURL)
@@ -22,17 +38,13 @@ func New(ctx context.Context, dbURL string) (models.LogStore, error) {
log.WithError(err).WithFields(logrus.Fields{"url": dbURL}).Fatal("bad DB URL")
}
log.WithFields(logrus.Fields{"db": u.Scheme}).Debug("creating log store")
var ls models.LogStore
switch u.Scheme {
case "sqlite3", "postgres", "mysql":
ls, err = sql.New(ctx, u)
case "s3":
ls, err = s3.New(u)
default:
err = fmt.Errorf("db type not supported %v", u.Scheme)
}
return ls, err
for _, p := range providers {
if p.Supports(u) {
return p.New(ctx, u)
}
}
return nil, fmt.Errorf("no log store provider available for url %s", dbURL)
}
func Wrap(ls models.LogStore) models.LogStore {

17
api/logs/s3/s3.go
View File

@@ -21,6 +21,7 @@ import (
"github.com/aws/aws-sdk-go/service/s3/s3manager"
"github.com/fnproject/fn/api/common"
"github.com/fnproject/fn/api/id"
"github.com/fnproject/fn/api/logs"
"github.com/fnproject/fn/api/models"
"github.com/sirupsen/logrus"
"go.opencensus.io/stats"
@@ -49,6 +50,8 @@ type store struct {
bucket string
}
type s3StoreProvider int
// decorator around the Reader interface that keeps track of the number of bytes read
// in order to avoid double buffering and track Reader size
type countingReader struct {
@@ -80,10 +83,18 @@ func createStore(bucketName, endpoint, region, accessKeyID, secretAccessKey stri
}
}
func (s3StoreProvider) String() string {
return "s3"
}
func (s3StoreProvider) Supports(u *url.URL) bool {
return u.Scheme == "s3"
}
// New returns an s3 api compatible log store.
// url format: s3://access_key_id:secret_access_key@host/region/bucket_name?ssl=true
// Note that access_key_id and secret_access_key must be URL encoded if they contain unsafe characters!
func New(u *url.URL) (models.LogStore, error) {
func (s3StoreProvider) New(ctx context.Context, u *url.URL) (models.LogStore, error) {
endpoint := u.Host
var accessKeyID, secretAccessKey string
@@ -464,3 +475,7 @@ func init() {
func (s *store) Close() error {
return nil
}
func init() {
logs.AddProvider(s3StoreProvider(0))
}

3
api/logs/s3/s3_test.go
View File

@@ -5,6 +5,7 @@ import (
"os"
"testing"
"context"
logTesting "github.com/fnproject/fn/api/logs/testing"
)
@@ -20,7 +21,7 @@ func TestS3(t *testing.T) {
t.Fatalf("failed to parse url: %v", err)
}
ls, err := New(uLog)
ls, err := s3StoreProvider(0).New(context.Background(), uLog)
if err != nil {
t.Fatalf("failed to create s3 datastore: %v", err)
}

42
api/models/reason.go
View File

@@ -1,42 +0,0 @@
package models
import (
"encoding/json"
strfmt "github.com/go-openapi/strfmt"
"github.com/go-openapi/validate"
)
/*Reason Machine usable reason for job being in this state.
Valid values for error status are `timeout | killed | bad_exit`.
Valid values for cancelled status are `client_request`.
For everything else, this is undefined.
swagger:model Reason
*/
type Reason string
// for schema
var reasonEnum []interface{}
func (m Reason) validateReasonEnum(path, location string, value Reason) error {
if reasonEnum == nil {
var res []Reason
if err := json.Unmarshal([]byte(`["timeout","killed","bad_exit","client_request"]`), &res); err != nil {
return err
}
for _, v := range res {
reasonEnum = append(reasonEnum, v)
}
}
err := validate.Enum(path, location, value, reasonEnum)
return err
}
// Validate validates this reason
func (m Reason) Validate(formats strfmt.Registry) error {
// value enum
return m.validateReasonEnum("", "body", m)
}

23
api/mqs/bolt.go → api/mqs/bolt/bolt.go
View File

@@ -1,4 +1,4 @@
package mqs
package bolt
import (
"context"
@@ -14,9 +14,12 @@ import (
"github.com/boltdb/bolt"
"github.com/fnproject/fn/api/common"
"github.com/fnproject/fn/api/models"
"github.com/fnproject/fn/api/mqs"
"github.com/sirupsen/logrus"
)
type boltProvider int
type BoltDbMQ struct {
db *bolt.DB
ticker *time.Ticker
@@ -52,7 +55,19 @@ func timeoutName(i int) []byte {
return []byte(fmt.Sprintf("functions_%d_timeout", i))
}
func NewBoltMQ(url *url.URL) (*BoltDbMQ, error) {
func (boltProvider) Supports(url *url.URL) bool {
switch url.Scheme {
case "bolt":
return true
}
return false
}
func (boltProvider) String() string {
return "bolt"
}
func (boltProvider) New(url *url.URL) (models.MessageQueue, error) {
dir := filepath.Dir(url.Path)
log := logrus.WithFields(logrus.Fields{"mq": url.Scheme, "dir": dir})
err := os.MkdirAll(dir, 0755)
@@ -354,3 +369,7 @@ func (mq *BoltDbMQ) Close() error {
mq.ticker.Stop()
return mq.db.Close()
}
func init() {
mqs.AddProvider(boltProvider(0))
}

38
api/mqs/memory.go → api/mqs/memory/memory.go
View File

@@ -1,18 +1,21 @@
package mqs
package memory
import (
"context"
"errors"
"math/rand"
"sync"
"time"
"github.com/fnproject/fn/api/common"
"github.com/fnproject/fn/api/models"
"github.com/fnproject/fn/api/mqs"
"github.com/google/btree"
"github.com/sirupsen/logrus"
"net/url"
)
type memoryProvider int
type MemoryMQ struct {
// WorkQueue A buffered channel that we can send work requests on.
PriorityQueues []chan *models.Call
@@ -26,20 +29,17 @@ type MemoryMQ struct {
Mutex sync.Mutex
}
var letters = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
func randSeq(n int) string {
rand.Seed(time.Now().Unix())
b := make([]rune, n)
for i := range b {
b[i] = letters[rand.Intn(len(letters))]
}
return string(b)
}
const NumPriorities = 3
func NewMemoryMQ() *MemoryMQ {
func (memoryProvider) Supports(url *url.URL) bool {
switch url.Scheme {
case "memory":
return true
}
return false
}
func (memoryProvider) New(url *url.URL) (models.MessageQueue, error) {
var queues []chan *models.Call
for i := 0; i < NumPriorities; i++ {
queues = append(queues, make(chan *models.Call, 5000))
@@ -53,7 +53,11 @@ func NewMemoryMQ() *MemoryMQ {
}
mq.start()
logrus.Info("MemoryMQ initialized")
return mq
return mq, nil
}
func (memoryProvider) String() string {
return "memory"
}
func (mq *MemoryMQ) start() {
@@ -198,3 +202,7 @@ func (mq *MemoryMQ) Close() error {
mq.Ticker.Stop()
return nil
}
func init() {
mqs.AddProvider(memoryProvider(0))
}

28
api/mqs/new.go
View File

@@ -11,6 +11,22 @@ import (
"github.com/sirupsen/logrus"
)
// Provider for message queue extensions
type Provider interface {
fmt.Stringer
//Supports indicates if this provider can handle a specific URL scheme
Supports(url *url.URL) bool
//New creates a new message queue from a given URL
New(url *url.URL) (models.MessageQueue, error)
}
var mqProviders []Provider
// AddProvider registers a new global message queue provider
func AddProvider(p Provider) {
mqProviders = append(mqProviders, p)
}
// New will parse the URL and return the correct MQ implementation.
func New(mqURL string) (models.MessageQueue, error) {
mq, err := newmq(mqURL)
@@ -27,15 +43,11 @@ func newmq(mqURL string) (models.MessageQueue, error) {
logrus.WithError(err).WithFields(logrus.Fields{"url": mqURL}).Fatal("bad MQ URL")
}
logrus.WithFields(logrus.Fields{"mq": u.Scheme}).Debug("selecting MQ")
switch u.Scheme {
case "memory":
return NewMemoryMQ(), nil
case "redis":
return NewRedisMQ(u)
case "bolt":
return NewBoltMQ(u)
for _, p := range mqProviders {
if p.Supports(u) {
return p.New(u)
}
}
return nil, fmt.Errorf("mq type not supported %v", u.Scheme)
}

23
api/mqs/redis.go → api/mqs/redis/redis.go
View File

@@ -1,4 +1,4 @@
package mqs
package redis
import (
"context"
@@ -11,6 +11,7 @@ import (
"github.com/fnproject/fn/api/common"
"github.com/fnproject/fn/api/models"
"github.com/fnproject/fn/api/mqs"
"github.com/garyburd/redigo/redis"
"github.com/sirupsen/logrus"
)
@@ -22,7 +23,21 @@ type RedisMQ struct {
prefix string
}
func NewRedisMQ(url *url.URL) (*RedisMQ, error) {
type redisProvider int
func (redisProvider) Supports(url *url.URL) bool {
switch url.Scheme {
case "redis":
return true
}
return false
}
func (redisProvider) String() string {
return "redis"
}
func (redisProvider) New(url *url.URL) (models.MessageQueue, error) {
pool := &redis.Pool{
MaxIdle: 512,
// I'm not sure if allowing the pool to block if more than 16 connections are required is a good idea.
@@ -314,3 +329,7 @@ func (mq *RedisMQ) Close() error {
mq.ticker.Stop()
return mq.pool.Close()
}
func init() {
mqs.AddProvider(redisProvider(0))
}

14
api/server/defaultexts/defaultexts.go Normal file
View File

@@ -0,0 +1,14 @@
// defaultexts are the extensions that are auto-loaded in to the default fnserver binary
// included here as a package to simplify inclusion in testing
package defaultexts
import (
_ "github.com/fnproject/fn/api/datastore/sql"
_ "github.com/fnproject/fn/api/datastore/sql/mysql"
_ "github.com/fnproject/fn/api/datastore/sql/postgres"
_ "github.com/fnproject/fn/api/datastore/sql/sqlite"
_ "github.com/fnproject/fn/api/logs/s3"
_ "github.com/fnproject/fn/api/mqs/bolt"
_ "github.com/fnproject/fn/api/mqs/memory"
_ "github.com/fnproject/fn/api/mqs/redis"
)

1
api/server/server_test.go
View File

@@ -16,6 +16,7 @@ import (
"github.com/fnproject/fn/api/agent"
"github.com/fnproject/fn/api/datastore"
"github.com/fnproject/fn/api/datastore/sql"
_ "github.com/fnproject/fn/api/datastore/sql/sqlite"
"github.com/fnproject/fn/api/id"
"github.com/fnproject/fn/api/logs"
"github.com/fnproject/fn/api/models"