mirror of
https://github.com/fnproject/fn.git
synced 2022-10-28 21:29:17 +03:00
merge datastores into sqlx package
replace default bolt option with sqlite3 option. the story here is that we just need a working out of the box solution, and sqlite3 is just fine for that (actually, likely better than bolt). with sqlite3 supplanting bolt, we mostly have sql databases. so remove redis and then we just have one package that has a `sql` implementation of the `models.Datastore` and lean on sqlx to do query rewriting. this does mean queries have to be formed a certain way and likely have to be ANSI-SQL (no special features) but we weren't using them anyway and our base api is basically done and we can easily extend this api as needed to only implement certain methods in certain backends if we need to get cute. * remove bolt & redis datastores (can still use as mqs) * make sql queries work on all 3 (maybe?) * remove bolt log store and use sqlite3 * shove the FnLog shit into the datastore shit for now (free pg/mysql logs... just for demos, etc, not prod) * fix up the docs to remove bolt references * add sqlite3, sqlx dep * fix up tests & mock stuff, make validator less insane * remove put & get in datastore layer as nobody is using. this passes tests which at least seem like they test all the different backends. if we trust our tests then this seems to work great. (tests `make docker-test-run-with-*` work now too)
This commit is contained in:
Reed Allman
@@ -468,47 +468,6 @@ func Test(t *testing.T, ds models.Datastore) {
|
||||
t.Fatalf("Test UpdateRoute inexistent: expected error to be `%v`, but it was `%v`", models.ErrRoutesNotFound, err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("put-get", func(t *testing.T) {
|
||||
// Testing Put/Get
|
||||
err := ds.Put(ctx, nil, nil)
|
||||
if err != models.ErrDatastoreEmptyKey {
|
||||
t.Log(buf.String())
|
||||
t.Fatalf("Test Put(nil,nil): expected error `%v`, but it was `%v`", models.ErrDatastoreEmptyKey, err)
|
||||
}
|
||||
|
||||
err = ds.Put(ctx, []byte("test"), []byte("success"))
|
||||
if err != nil {
|
||||
t.Log(buf.String())
|
||||
t.Fatalf("Test Put: unexpected error: %v", err)
|
||||
}
|
||||
|
||||
val, err := ds.Get(ctx, []byte("test"))
|
||||
if err != nil {
|
||||
t.Log(buf.String())
|
||||
t.Fatalf("Test Put: unexpected error: %v", err)
|
||||
}
|
||||
if string(val) != "success" {
|
||||
t.Log(buf.String())
|
||||
t.Fatalf("Test Get: expected value to be `%v`, but it was `%v`", "success", string(val))
|
||||
}
|
||||
|
||||
err = ds.Put(ctx, []byte("test"), nil)
|
||||
if err != nil {
|
||||
t.Log(buf.String())
|
||||
t.Fatalf("Test Put: unexpected error: %v", err)
|
||||
}
|
||||
|
||||
val, err = ds.Get(ctx, []byte("test"))
|
||||
if err != nil {
|
||||
t.Log(buf.String())
|
||||
t.Fatalf("Test Put: unexpected error: %v", err)
|
||||
}
|
||||
if string(val) != "" {
|
||||
t.Log(buf.String())
|
||||
t.Fatalf("Test Get: expected value to be `%v`, but it was `%v`", "", string(val))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
var testApp = &models.App{
|
||||
|
||||
@@ -5,12 +5,13 @@ import (
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"github.com/Sirupsen/logrus"
|
||||
"strings"
|
||||
|
||||
"gitlab-odx.oracle.com/odx/functions/api/models"
|
||||
)
|
||||
|
||||
// TODO scrap for sqlx
|
||||
|
||||
type RowScanner interface {
|
||||
Scan(dest ...interface{}) error
|
||||
}
|
||||
@@ -165,213 +166,4 @@ func ScanCall(scanner RowScanner, call *models.FnCall) error {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
|
||||
}
|
||||
|
||||
func SQLGetCall(db *sql.DB, callSelector, callID, whereStm string) (*models.FnCall, error) {
|
||||
var call models.FnCall
|
||||
row := db.QueryRow(fmt.Sprintf(whereStm, callSelector), callID)
|
||||
err := ScanCall(row, &call)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &call, nil
|
||||
}
|
||||
|
||||
func SQLGetCalls(db *sql.DB, cSelector string, filter *models.CallFilter, whereStm, andStm string) (models.FnCalls, error) {
|
||||
res := models.FnCalls{}
|
||||
filterQuery, args := BuildFilterCallQuery(filter, whereStm, andStm)
|
||||
rows, err := db.Query(fmt.Sprintf("%s %s", cSelector, filterQuery), args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
for rows.Next() {
|
||||
var call models.FnCall
|
||||
err := ScanCall(rows, &call)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
res = append(res, &call)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func SQLGetApp(db *sql.DB, queryStr string, queryArgs ...interface{}) (*models.App, error) {
|
||||
row := db.QueryRow(queryStr, queryArgs...)
|
||||
|
||||
var resName string
|
||||
var config string
|
||||
err := row.Scan(&resName, &config)
|
||||
if err != nil {
|
||||
if err == sql.ErrNoRows {
|
||||
return nil, models.ErrAppsNotFound
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
res := &models.App{
|
||||
Name: resName,
|
||||
}
|
||||
|
||||
if len(config) > 0 {
|
||||
err := json.Unmarshal([]byte(config), &res.Config)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func SQLGetApps(db *sql.DB, filter *models.AppFilter, whereStm, selectStm string) ([]*models.App, error) {
|
||||
res := []*models.App{}
|
||||
filterQuery, args := BuildFilterAppQuery(filter, whereStm)
|
||||
rows, err := db.Query(fmt.Sprintf(selectStm, filterQuery), args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
for rows.Next() {
|
||||
var app models.App
|
||||
err := ScanApp(rows, &app)
|
||||
|
||||
if err != nil {
|
||||
if err == sql.ErrNoRows {
|
||||
return res, nil
|
||||
}
|
||||
return res, err
|
||||
}
|
||||
res = append(res, &app)
|
||||
}
|
||||
|
||||
if err := rows.Err(); err != nil {
|
||||
return res, err
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func NewDatastore(dataSourceName, dialect string, tables []string) (*sql.DB, error) {
|
||||
db, err := sql.Open(dialect, dataSourceName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err = db.Ping()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
maxIdleConns := 30 // c.MaxIdleConnections
|
||||
db.SetMaxIdleConns(maxIdleConns)
|
||||
logrus.WithFields(logrus.Fields{"max_idle_connections": maxIdleConns}).Info(
|
||||
fmt.Sprintf("%v datastore dialed", dialect))
|
||||
|
||||
for _, v := range tables {
|
||||
_, err = db.Exec(v)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
return db, nil
|
||||
}
|
||||
|
||||
func SQLGetRoutes(db *sql.DB, filter *models.RouteFilter, rSelect string, whereStm, andStm string) ([]*models.Route, error) {
|
||||
res := []*models.Route{}
|
||||
filterQuery, args := BuildFilterRouteQuery(filter, whereStm, andStm)
|
||||
rows, err := db.Query(fmt.Sprintf("%s %s", rSelect, filterQuery), args...)
|
||||
// todo: check for no rows so we don't respond with a sql 500 err
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
for rows.Next() {
|
||||
var route models.Route
|
||||
err := ScanRoute(rows, &route)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
res = append(res, &route)
|
||||
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return res, nil
|
||||
|
||||
}
|
||||
|
||||
func SQLGetRoutesByApp(db *sql.DB, appName string, filter *models.RouteFilter, rSelect, defaultFilterQuery, whereStm, andStm string) ([]*models.Route, error) {
|
||||
res := []*models.Route{}
|
||||
var filterQuery string
|
||||
var args []interface{}
|
||||
if filter == nil {
|
||||
filterQuery = defaultFilterQuery
|
||||
args = []interface{}{appName}
|
||||
} else {
|
||||
filter.AppName = appName
|
||||
filterQuery, args = BuildFilterRouteQuery(filter, whereStm, andStm)
|
||||
}
|
||||
rows, err := db.Query(fmt.Sprintf("%s %s", rSelect, filterQuery), args...)
|
||||
// todo: check for no rows so we don't respond with a sql 500 err
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
for rows.Next() {
|
||||
var route models.Route
|
||||
err := ScanRoute(rows, &route)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
res = append(res, &route)
|
||||
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func SQLGetRoute(db *sql.DB, appName, routePath, rSelectCondition, routeSelector string) (*models.Route, error) {
|
||||
var route models.Route
|
||||
|
||||
row := db.QueryRow(fmt.Sprintf(rSelectCondition, routeSelector), appName, routePath)
|
||||
err := ScanRoute(row, &route)
|
||||
|
||||
if err == sql.ErrNoRows {
|
||||
return nil, models.ErrRoutesNotFound
|
||||
} else if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &route, nil
|
||||
}
|
||||
|
||||
func SQLRemoveRoute(db *sql.DB, appName, routePath, deleteStm string) error {
|
||||
res, err := db.Exec(deleteStm, routePath, appName)
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
n, err := res.RowsAffected()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if n == 0 {
|
||||
return models.ErrRoutesRemoving
|
||||
}
|
||||
|
||||
return nil
|
||||
|
||||
}
|
||||
|
||||
@@ -6,62 +6,28 @@ import (
|
||||
"gitlab-odx.oracle.com/odx/functions/api/models"
|
||||
)
|
||||
|
||||
// Datastore is a copy of models.Datastore, with additional comments on parameter guarantees.
|
||||
type Datastore interface {
|
||||
// name will never be empty.
|
||||
GetApp(ctx context.Context, name string) (*models.App, error)
|
||||
|
||||
GetApps(ctx context.Context, appFilter *models.AppFilter) ([]*models.App, error)
|
||||
|
||||
// app and app.Name will never be nil/empty.
|
||||
InsertApp(ctx context.Context, app *models.App) (*models.App, error)
|
||||
UpdateApp(ctx context.Context, app *models.App) (*models.App, error)
|
||||
|
||||
// name will never be empty.
|
||||
RemoveApp(ctx context.Context, name string) error
|
||||
|
||||
// appName and routePath will never be empty.
|
||||
GetRoute(ctx context.Context, appName, routePath string) (*models.Route, error)
|
||||
RemoveRoute(ctx context.Context, appName, routePath string) error
|
||||
|
||||
GetRoutes(ctx context.Context, filter *models.RouteFilter) (routes []*models.Route, err error)
|
||||
|
||||
// appName will never be empty
|
||||
GetRoutesByApp(ctx context.Context, appName string, filter *models.RouteFilter) (routes []*models.Route, err error)
|
||||
|
||||
// route will never be nil and route's AppName and Path will never be empty.
|
||||
InsertRoute(ctx context.Context, route *models.Route) (*models.Route, error)
|
||||
UpdateRoute(ctx context.Context, route *models.Route) (*models.Route, error)
|
||||
|
||||
InsertTask(ctx context.Context, task *models.Task) error
|
||||
GetTask(ctx context.Context, callID string) (*models.FnCall, error)
|
||||
GetTasks(ctx context.Context, filter *models.CallFilter) (models.FnCalls, error)
|
||||
|
||||
// key will never be nil/empty
|
||||
Put(ctx context.Context, key, val []byte) error
|
||||
Get(ctx context.Context, key []byte) ([]byte, error)
|
||||
}
|
||||
|
||||
// NewValidator returns a models.Datastore which validates certain arguments before delegating to ds.
|
||||
func NewValidator(ds Datastore) models.Datastore {
|
||||
func NewValidator(ds models.Datastore) models.Datastore {
|
||||
return &validator{ds}
|
||||
}
|
||||
|
||||
type validator struct {
|
||||
ds Datastore
|
||||
models.Datastore
|
||||
}
|
||||
|
||||
// name will never be empty.
|
||||
func (v *validator) GetApp(ctx context.Context, name string) (app *models.App, err error) {
|
||||
if name == "" {
|
||||
return nil, models.ErrDatastoreEmptyAppName
|
||||
}
|
||||
return v.ds.GetApp(ctx, name)
|
||||
return v.Datastore.GetApp(ctx, name)
|
||||
}
|
||||
|
||||
func (v *validator) GetApps(ctx context.Context, appFilter *models.AppFilter) ([]*models.App, error) {
|
||||
return v.ds.GetApps(ctx, appFilter)
|
||||
return v.Datastore.GetApps(ctx, appFilter)
|
||||
}
|
||||
|
||||
// app and app.Name will never be nil/empty.
|
||||
func (v *validator) InsertApp(ctx context.Context, app *models.App) (*models.App, error) {
|
||||
if app == nil {
|
||||
return nil, models.ErrDatastoreEmptyApp
|
||||
@@ -70,9 +36,10 @@ func (v *validator) InsertApp(ctx context.Context, app *models.App) (*models.App
|
||||
return nil, models.ErrDatastoreEmptyAppName
|
||||
}
|
||||
|
||||
return v.ds.InsertApp(ctx, app)
|
||||
return v.Datastore.InsertApp(ctx, app)
|
||||
}
|
||||
|
||||
// app and app.Name will never be nil/empty.
|
||||
func (v *validator) UpdateApp(ctx context.Context, app *models.App) (*models.App, error) {
|
||||
if app == nil {
|
||||
return nil, models.ErrDatastoreEmptyApp
|
||||
@@ -80,17 +47,19 @@ func (v *validator) UpdateApp(ctx context.Context, app *models.App) (*models.App
|
||||
if app.Name == "" {
|
||||
return nil, models.ErrDatastoreEmptyAppName
|
||||
}
|
||||
return v.ds.UpdateApp(ctx, app)
|
||||
return v.Datastore.UpdateApp(ctx, app)
|
||||
}
|
||||
|
||||
// name will never be empty.
|
||||
func (v *validator) RemoveApp(ctx context.Context, name string) error {
|
||||
if name == "" {
|
||||
return models.ErrDatastoreEmptyAppName
|
||||
}
|
||||
|
||||
return v.ds.RemoveApp(ctx, name)
|
||||
return v.Datastore.RemoveApp(ctx, name)
|
||||
}
|
||||
|
||||
// appName and routePath will never be empty.
|
||||
func (v *validator) GetRoute(ctx context.Context, appName, routePath string) (*models.Route, error) {
|
||||
if appName == "" {
|
||||
return nil, models.ErrDatastoreEmptyAppName
|
||||
@@ -99,24 +68,26 @@ func (v *validator) GetRoute(ctx context.Context, appName, routePath string) (*m
|
||||
return nil, models.ErrDatastoreEmptyRoutePath
|
||||
}
|
||||
|
||||
return v.ds.GetRoute(ctx, appName, routePath)
|
||||
return v.Datastore.GetRoute(ctx, appName, routePath)
|
||||
}
|
||||
|
||||
func (v *validator) GetRoutes(ctx context.Context, routeFilter *models.RouteFilter) (routes []*models.Route, err error) {
|
||||
if routeFilter != nil && routeFilter.AppName != "" {
|
||||
return v.ds.GetRoutesByApp(ctx, routeFilter.AppName, routeFilter)
|
||||
return v.Datastore.GetRoutesByApp(ctx, routeFilter.AppName, routeFilter)
|
||||
}
|
||||
|
||||
return v.ds.GetRoutes(ctx, routeFilter)
|
||||
return v.Datastore.GetRoutes(ctx, routeFilter)
|
||||
}
|
||||
|
||||
// appName will never be empty
|
||||
func (v *validator) GetRoutesByApp(ctx context.Context, appName string, routeFilter *models.RouteFilter) (routes []*models.Route, err error) {
|
||||
if appName == "" {
|
||||
return nil, models.ErrDatastoreEmptyAppName
|
||||
}
|
||||
return v.ds.GetRoutesByApp(ctx, appName, routeFilter)
|
||||
return v.Datastore.GetRoutesByApp(ctx, appName, routeFilter)
|
||||
}
|
||||
|
||||
// route will never be nil and route's AppName and Path will never be empty.
|
||||
func (v *validator) InsertRoute(ctx context.Context, route *models.Route) (*models.Route, error) {
|
||||
if route == nil {
|
||||
return nil, models.ErrDatastoreEmptyRoute
|
||||
@@ -128,9 +99,10 @@ func (v *validator) InsertRoute(ctx context.Context, route *models.Route) (*mode
|
||||
return nil, models.ErrDatastoreEmptyRoutePath
|
||||
}
|
||||
|
||||
return v.ds.InsertRoute(ctx, route)
|
||||
return v.Datastore.InsertRoute(ctx, route)
|
||||
}
|
||||
|
||||
// route will never be nil and route's AppName and Path will never be empty.
|
||||
func (v *validator) UpdateRoute(ctx context.Context, newroute *models.Route) (*models.Route, error) {
|
||||
if newroute == nil {
|
||||
return nil, models.ErrDatastoreEmptyRoute
|
||||
@@ -141,9 +113,10 @@ func (v *validator) UpdateRoute(ctx context.Context, newroute *models.Route) (*m
|
||||
if newroute.Path == "" {
|
||||
return nil, models.ErrDatastoreEmptyRoutePath
|
||||
}
|
||||
return v.ds.UpdateRoute(ctx, newroute)
|
||||
return v.Datastore.UpdateRoute(ctx, newroute)
|
||||
}
|
||||
|
||||
// appName and routePath will never be empty.
|
||||
func (v *validator) RemoveRoute(ctx context.Context, appName, routePath string) error {
|
||||
if appName == "" {
|
||||
return models.ErrDatastoreEmptyAppName
|
||||
@@ -152,35 +125,13 @@ func (v *validator) RemoveRoute(ctx context.Context, appName, routePath string)
|
||||
return models.ErrDatastoreEmptyRoutePath
|
||||
}
|
||||
|
||||
return v.ds.RemoveRoute(ctx, appName, routePath)
|
||||
}
|
||||
|
||||
func (v *validator) Put(ctx context.Context, key, value []byte) error {
|
||||
if len(key) == 0 {
|
||||
return models.ErrDatastoreEmptyKey
|
||||
}
|
||||
|
||||
return v.ds.Put(ctx, key, value)
|
||||
}
|
||||
|
||||
func (v *validator) Get(ctx context.Context, key []byte) ([]byte, error) {
|
||||
if len(key) == 0 {
|
||||
return nil, models.ErrDatastoreEmptyKey
|
||||
}
|
||||
return v.ds.Get(ctx, key)
|
||||
}
|
||||
|
||||
func (v *validator) InsertTask(ctx context.Context, task *models.Task) error {
|
||||
return v.ds.InsertTask(ctx, task)
|
||||
return v.Datastore.RemoveRoute(ctx, appName, routePath)
|
||||
}
|
||||
|
||||
// callID will never be empty.
|
||||
func (v *validator) GetTask(ctx context.Context, callID string) (*models.FnCall, error) {
|
||||
if callID == "" {
|
||||
return nil, models.ErrDatastoreEmptyTaskID
|
||||
}
|
||||
return v.ds.GetTask(ctx, callID)
|
||||
}
|
||||
|
||||
func (v *validator) GetTasks(ctx context.Context, filter *models.CallFilter) (models.FnCalls, error) {
|
||||
return v.ds.GetTasks(ctx, filter)
|
||||
return v.Datastore.GetTask(ctx, callID)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user