[Feature] Function status

api/datastore/bolt/bolt.go

@@ -23,6 +23,7 @@ type BoltDatastore struct {
 	appsBucket   []byte
 	logsBucket   []byte
 	extrasBucket []byte
+	callsBucket  []byte
 	db           *bolt.DB
 	log          logrus.FieldLogger
 }
@@ -50,8 +51,9 @@ func New(url *url.URL) (models.Datastore, error) {
 	appsBucketName := []byte(bucketPrefix + "apps")
 	logsBucketName := []byte(bucketPrefix + "logs")
 	extrasBucketName := []byte(bucketPrefix + "extras") // todo: think of a better name
+	callsBucketName := []byte(bucketPrefix + "calls")
 	err = db.Update(func(tx *bolt.Tx) error {
-		for _, name := range [][]byte{routesBucketName, appsBucketName, logsBucketName, extrasBucketName} {
+		for _, name := range [][]byte{routesBucketName, appsBucketName, logsBucketName, extrasBucketName, callsBucketName} {
 			_, err := tx.CreateBucketIfNotExists(name)
 			if err != nil {
 				log.WithError(err).WithFields(logrus.Fields{"name": name}).Error("create bucket")
@@ -70,6 +72,7 @@ func New(url *url.URL) (models.Datastore, error) {
 		appsBucket:   appsBucketName,
 		logsBucket:   logsBucketName,
 		extrasBucket: extrasBucketName,
+		callsBucket:  callsBucketName,
 		db:           db,
 		log:          log,
 	}
@@ -78,6 +81,72 @@ func New(url *url.URL) (models.Datastore, error) {
 	return datastoreutil.NewValidator(ds), nil
 }
 
+
+func (ds *BoltDatastore) InsertTask(ctx context.Context, task *models.Task) error {
+	var fnCall *models.FnCall
+	taskID := []byte(task.ID)
+
+	err := ds.db.Update(
+		func(tx *bolt.Tx) error {
+			bIm := tx.Bucket(ds.callsBucket)
+			buf, err := json.Marshal(fnCall.FromTask(task))
+			if err != nil {
+				return err
+			}
+			err = bIm.Put(taskID, buf)
+			if err != nil {
+				return err
+			}
+			return nil
+		})
+	return err
+}
+
+func (ds *BoltDatastore) GetTasks(ctx context.Context, filter *models.CallFilter) (models.FnCalls, error) {
+	res := models.FnCalls{}
+	err := ds.db.View(func(tx *bolt.Tx) error {
+		b := tx.Bucket(ds.callsBucket)
+		err2 := b.ForEach(func(key, v []byte) error {
+			call := &models.FnCall{}
+			err := json.Unmarshal(v, call)
+			if err != nil {
+				return err
+			}
+			if applyCallFilter(call, filter) {
+				res = append(res, call)
+			}
+			return nil
+		})
+		if err2 != nil {
+			logrus.WithError(err2).Errorln("Couldn't get calls!")
+		}
+		return err2
+	})
+	return res, err
+}
+
+
+func (ds *BoltDatastore) GetTask(ctx context.Context, callID string) (*models.FnCall, error) {
+	var res *models.FnCall
+	err := ds.db.View(func(tx *bolt.Tx) error {
+		b := tx.Bucket(ds.callsBucket)
+		v := b.Get([]byte(callID))
+		if v != nil {
+			fnCall := &models.FnCall{}
+			err := json.Unmarshal(v, fnCall)
+			if err != nil {
+				return nil
+			}
+			res = fnCall
+		} else {
+			return models.ErrCallNotFound
+		}
+		return nil
+	})
+	return res, err
+}
+
+
 func (ds *BoltDatastore) InsertApp(ctx context.Context, app *models.App) (*models.App, error) {
 	appname := []byte(app.Name)
 
@@ -99,7 +168,7 @@ func (ds *BoltDatastore) InsertApp(ctx context.Context, app *models.App) (*model
 			return err
 		}
 		bjParent := tx.Bucket(ds.routesBucket)
-		_, err = bjParent.CreateBucketIfNotExists([]byte(app.Name))
+		_, err = bjParent.CreateBucketIfNotExists(appname)
 		if err != nil {
 			return err
 		}
@@ -435,3 +504,7 @@ func applyRouteFilter(route *models.Route, filter *models.RouteFilter) bool {
 		(filter.AppName == "" || route.AppName == filter.AppName) &&
 		(filter.Image == "" || route.Image == filter.Image)
 }
+
+func applyCallFilter(call *models.FnCall, filter *models.CallFilter) bool {
+	return filter == nil || (filter.AppName == call.AppName) && (filter.Path == call.Path)
+}

api/datastore/internal/datastoretest/test.go

@@ -12,9 +12,12 @@ import (
 	"net/url"
 	"os"
 	"reflect"
+	"time"
 
 	"github.com/Sirupsen/logrus"
 	"github.com/gin-gonic/gin"
+	"github.com/go-openapi/strfmt"
+	"github.com/satori/go.uuid"
 )
 
 func setLogBuffer() *bytes.Buffer {
@@ -41,6 +44,50 @@ func Test(t *testing.T, ds models.Datastore) {
 
 	ctx := context.Background()
 
+	task := &models.Task{}
+	task.CreatedAt = strfmt.DateTime(time.Now())
+	task.Status = "success"
+	task.StartedAt = strfmt.DateTime(time.Now())
+	task.CompletedAt = strfmt.DateTime(time.Now())
+	task.AppName = testApp.Name
+	task.Path = testRoute.Path
+
+	t.Run("call-insert", func(t *testing.T) {
+		task.ID = uuid.NewV4().String()
+		err := ds.InsertTask(ctx, task)
+		if err != nil {
+			t.Log(buf.String())
+			t.Fatalf("Test InsertTask(ctx, &task): unexpected error `%v`", err)
+		}
+	})
+
+	t.Run("call-get", func(t *testing.T) {
+		task.ID = uuid.NewV4().String()
+		ds.InsertTask(ctx, task)
+		newTask, err := ds.GetTask(ctx, task.ID)
+		if err != nil {
+			t.Fatalf("Test GetTask(ctx, task.ID): unexpected error `%v`", err)
+		}
+		if task.ID != newTask.ID {
+			t.Log(buf.String())
+			t.Fatalf("Test GetTask(ctx, task.ID): unexpected error `%v`", err)
+		}
+	})
+
+	t.Run("calls-get", func(t *testing.T) {
+		filter := &models.CallFilter{AppName: task.AppName, Path:task.Path}
+		task.ID = uuid.NewV4().String()
+		ds.InsertTask(ctx, task)
+		calls, err := ds.GetTasks(ctx, filter)
+		if err != nil {
+			t.Fatalf("Test GetTasks(ctx, filter): unexpected error `%v`", err)
+		}
+		if len(calls) == 0 {
+			t.Log(buf.String())
+			t.Fatalf("Test GetTasks(ctx, filter): unexpected error `%v`", err)
+		}
+	})
+
 	t.Run("apps", func(t *testing.T) {
 		// Testing insert app
 		_, err := ds.InsertApp(ctx, nil)

api/datastore/internal/datastoreutil/shared.go

@@ -0,0 +1,375 @@
+package datastoreutil
+
+import (
+	"bytes"
+	"database/sql"
+	"encoding/json"
+	"github.com/Sirupsen/logrus"
+	"fmt"
+	"strings"
+
+	"gitlab-odx.oracle.com/odx/functions/api/models"
+)
+
+type RowScanner interface {
+	Scan(dest ...interface{}) error
+}
+
+
+func ScanRoute(scanner RowScanner, route *models.Route) error {
+	var headerStr string
+	var configStr string
+
+	err := scanner.Scan(
+		&route.AppName,
+		&route.Path,
+		&route.Image,
+		&route.Format,
+		&route.MaxConcurrency,
+		&route.Memory,
+		&route.Type,
+		&route.Timeout,
+		&route.IdleTimeout,
+		&headerStr,
+		&configStr,
+	)
+	if err != nil {
+		return err
+	}
+
+	if len(headerStr) > 0 {
+		err = json.Unmarshal([]byte(headerStr), &route.Headers)
+		if err != nil {
+			return err
+		}
+	}
+
+	if len(configStr) > 0 {
+		err = json.Unmarshal([]byte(configStr), &route.Config)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func ScanApp(scanner RowScanner, app *models.App) error {
+	var configStr string
+
+	err := scanner.Scan(
+		&app.Name,
+		&configStr,
+	)
+	if err != nil {
+		return err
+	}
+	if len(configStr) > 0 {
+		err = json.Unmarshal([]byte(configStr), &app.Config)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+
+}
+
+func BuildFilterRouteQuery(filter *models.RouteFilter, whereStm, andStm string) (string, []interface{}) {
+	if filter == nil {
+		return "", nil
+	}
+	var b bytes.Buffer
+	var args []interface{}
+
+	where := func(colOp, val string) {
+		if val != "" {
+			args = append(args, val)
+			if len(args) == 1 {
+				fmt.Fprintf(&b, whereStm, colOp)
+			} else {
+				//TODO: maybe better way to detect/driver SQL dialect-specific things
+				if strings.Contains(whereStm, "$") {
+					// PgSQL specific
+					fmt.Fprintf(&b, andStm, colOp, len(args))
+				} else {
+					// MySQL specific
+					fmt.Fprintf(&b, andStm, colOp)
+				}
+			}
+		}
+	}
+
+	where("path =", filter.Path)
+	where("app_name =", filter.AppName)
+	where("image =", filter.Image)
+
+	return b.String(), args
+}
+
+
+func BuildFilterAppQuery(filter *models.AppFilter, whereStm string) (string, []interface{}) {
+	if filter == nil {
+		return "", nil
+	}
+
+	if filter.Name != "" {
+		return whereStm, []interface{}{filter.Name}
+	}
+
+	return "", nil
+}
+
+
+func BuildFilterCallQuery(filter *models.CallFilter, whereStm, andStm string) (string, []interface{}) {
+	if filter == nil {
+		return "", nil
+	}
+	var b bytes.Buffer
+	var args []interface{}
+
+	where := func(colOp, val string) {
+		if val != "" {
+			args = append(args, val)
+			if len(args) == 1 {
+				fmt.Fprintf(&b, whereStm, colOp)
+			} else {
+				fmt.Fprintf(&b, andStm, colOp)
+			}
+		}
+	}
+
+	where("path =", filter.Path)
+	where("app_name =", filter.AppName)
+
+	return b.String(), args
+}
+
+func ScanCall(scanner RowScanner, call *models.FnCall) error {
+	err := scanner.Scan(
+		&call.ID,
+		&call.CreatedAt,
+		&call.StartedAt,
+		&call.CompletedAt,
+		&call.Status,
+		&call.AppName,
+		&call.Path,
+	)
+
+	if err == sql.ErrNoRows {
+		return models.ErrCallNotFound
+	} else if err != nil {
+		return err
+	}
+	return nil
+
+}
+
+func SQLGetCall(db *sql.DB, callSelector, callID, whereStm string) (*models.FnCall, error) {
+	var call models.FnCall
+	row := db.QueryRow(fmt.Sprintf(whereStm, callSelector), callID)
+	err := ScanCall(row, &call)
+	if err != nil {
+		return nil, err
+	}
+	return &call, nil
+}
+
+func SQLGetCalls(db *sql.DB, cSelector string, filter *models.CallFilter, whereStm, andStm string) (models.FnCalls, error) {
+	res := models.FnCalls{}
+	filterQuery, args := BuildFilterCallQuery(filter, whereStm, andStm)
+	rows, err := db.Query(fmt.Sprintf("%s %s", cSelector, filterQuery), args...)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+
+	for rows.Next() {
+		var call models.FnCall
+		err := ScanCall(rows, &call)
+		if err != nil {
+			continue
+		}
+		res = append(res, &call)
+	}
+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+	return res, nil
+}
+
+func SQLGetApp(db *sql.DB, queryStr string, queryArgs ...interface{}) (*models.App, error) {
+	row := db.QueryRow(queryStr, queryArgs...)
+
+	var resName string
+	var config string
+	err := row.Scan(&resName, &config)
+	if err != nil {
+		if err == sql.ErrNoRows {
+			return nil, models.ErrAppsNotFound
+		}
+		return nil, err
+	}
+
+	res := &models.App{
+		Name: resName,
+	}
+
+	if len(config) > 0 {
+		err := json.Unmarshal([]byte(config), &res.Config)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return res, nil
+}
+
+func SQLGetApps(db *sql.DB, filter *models.AppFilter, whereStm, selectStm string) ([]*models.App, error) {
+	res := []*models.App{}
+	filterQuery, args := BuildFilterAppQuery(filter, whereStm)
+	rows, err := db.Query(fmt.Sprintf(selectStm, filterQuery), args...)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+
+	for rows.Next() {
+		var app models.App
+		err := ScanApp(rows, &app)
+
+		if err != nil {
+			if err == sql.ErrNoRows {
+				return res, nil
+			}
+			return res, err
+		}
+		res = append(res, &app)
+	}
+
+	if err := rows.Err(); err != nil {
+		return res, err
+	}
+	return res, nil
+}
+
+func NewDatastore(dataSourceName, dialect string, tables []string) (*sql.DB, error) {
+	db, err := sql.Open(dialect, dataSourceName)
+	if err != nil {
+		return nil, err
+	}
+
+	err = db.Ping()
+	if err != nil {
+		return nil, err
+	}
+
+	maxIdleConns := 30 // c.MaxIdleConnections
+	db.SetMaxIdleConns(maxIdleConns)
+	logrus.WithFields(logrus.Fields{"max_idle_connections": maxIdleConns}).Info(
+		fmt.Sprintf("%v datastore dialed", dialect))
+
+
+	for _, v := range tables {
+		_, err = db.Exec(v)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return db, nil
+}
+
+func SQLGetRoutes(db *sql.DB, filter *models.RouteFilter, rSelect string, whereStm, andStm string) ([]*models.Route, error) {
+	res := []*models.Route{}
+	filterQuery, args := BuildFilterRouteQuery(filter, whereStm, andStm)
+	rows, err := db.Query(fmt.Sprintf("%s %s", rSelect, filterQuery), args...)
+	// todo: check for no rows so we don't respond with a sql 500 err
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+
+	for rows.Next() {
+		var route models.Route
+		err := ScanRoute(rows, &route)
+		if err != nil {
+			continue
+		}
+		res = append(res, &route)
+
+	}
+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+	return res, nil
+
+}
+
+func SQLGetRoutesByApp(db *sql.DB, appName string, filter *models.RouteFilter, rSelect, defaultFilterQuery, whereStm, andStm string) ([]*models.Route, error) {
+	res := []*models.Route{}
+	var filterQuery string
+	var args []interface{}
+	if filter == nil {
+		filterQuery = defaultFilterQuery
+		args = []interface{}{appName}
+	} else {
+		filter.AppName = appName
+		filterQuery, args = BuildFilterRouteQuery(filter, whereStm, andStm)
+	}
+	rows, err := db.Query(fmt.Sprintf("%s %s", rSelect, filterQuery), args...)
+	// todo: check for no rows so we don't respond with a sql 500 err
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+
+	for rows.Next() {
+		var route models.Route
+		err := ScanRoute(rows, &route)
+		if err != nil {
+			continue
+		}
+		res = append(res, &route)
+
+	}
+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+
+	return res, nil
+}
+
+func SQLGetRoute(db *sql.DB, appName, routePath, rSelectCondition, routeSelector string) (*models.Route, error) {
+	var route models.Route
+
+	row := db.QueryRow(fmt.Sprintf(rSelectCondition, routeSelector), appName, routePath)
+	err := ScanRoute(row, &route)
+
+	if err == sql.ErrNoRows {
+		return nil, models.ErrRoutesNotFound
+	} else if err != nil {
+		return nil, err
+	}
+	return &route, nil
+}
+
+func SQLRemoveRoute(db *sql.DB, appName, routePath, deleteStm string) error {
+	res, err := db.Exec(deleteStm, routePath, appName)
+
+	if err != nil {
+		return err
+	}
+
+	n, err := res.RowsAffected()
+	if err != nil {
+		return err
+	}
+
+	if n == 0 {
+		return models.ErrRoutesRemoving
+	}
+
+	return nil
+
+}

api/datastore/internal/datastoreutil/validator.go

@@ -33,6 +33,10 @@ type Datastore interface {
 	InsertRoute(ctx context.Context, route *models.Route) (*models.Route, error)
 	UpdateRoute(ctx context.Context, route *models.Route) (*models.Route, error)
 
+	InsertTask(ctx context.Context, task *models.Task) error
+	GetTask(ctx context.Context, callID string) (*models.FnCall, error)
+	GetTasks(ctx context.Context, filter *models.CallFilter) (models.FnCalls, error)
+
 	// key will never be nil/empty
 	Put(ctx context.Context, key, val []byte) error
 	Get(ctx context.Context, key []byte) ([]byte, error)
@@ -164,4 +168,19 @@ func (v *validator) Get(ctx context.Context, key []byte) ([]byte, error) {
 		return nil, models.ErrDatastoreEmptyKey
 	}
 	return v.ds.Get(ctx, key)
-}
+}
+
+func (v *validator) InsertTask(ctx context.Context, task *models.Task) error {
+	return v.ds.InsertTask(ctx, task)
+}
+
+func (v *validator) GetTask(ctx context.Context, callID string) (*models.FnCall, error) {
+	if callID == "" {
+		return nil, models.ErrDatastoreEmptyTaskID
+	}
+	return v.ds.GetTask(ctx, callID)
+}
+
+func (v *validator) GetTasks(ctx context.Context, filter *models.CallFilter) (models.FnCalls, error) {
+	return v.ds.GetTasks(ctx, filter)
+}

api/datastore/mock.go

@@ -8,23 +8,27 @@ import (
 )
 
 type mock struct {
-	Apps   []*models.App
-	Routes []*models.Route
+	Apps   models.Apps
+	Routes models.Routes
+	Calls  models.FnCalls
 	data map[string][]byte
 }
 
 func NewMock() models.Datastore {
-	return NewMockInit(nil, nil)
+	return NewMockInit(nil, nil, nil)
 }
 
-func NewMockInit(apps []*models.App, routes []*models.Route) models.Datastore {
+func NewMockInit(apps models.Apps, routes models.Routes, calls models.FnCalls) models.Datastore {
 	if apps == nil {
-		apps = []*models.App{}
+		apps = models.Apps{}
 	}
 	if routes == nil {
-		routes = []*models.Route{}
+		routes = models.Routes{}
 	}
-	return datastoreutil.NewValidator(&mock{apps, routes, make(map[string][]byte)})
+	if calls == nil {
+		calls = models.FnCalls{}
+	}
+	return datastoreutil.NewValidator(&mock{apps, routes, calls, make(map[string][]byte)})
 }
 
 func (m *mock) GetApp(ctx context.Context, appName string) (app *models.App, err error) {
@@ -137,3 +141,23 @@ func (m *mock) Put(ctx context.Context, key, value []byte) error {
 func (m *mock) Get(ctx context.Context, key []byte) ([]byte, error) {
 	return m.data[string(key)], nil
 }
+
+func (m *mock) InsertTask(ctx context.Context, task *models.Task) error {
+	var call *models.FnCall
+	m.Calls = append(m.Calls, call.FromTask(task))
+	return nil
+}
+
+func (m *mock) GetTask(ctx context.Context, callID string) (*models.FnCall, error) {
+	for _, t := range m.Calls {
+		if t.ID == callID {
+			return t, nil
+		}
+	}
+
+	return nil, models.ErrCallNotFound
+}
+
+func (m *mock) GetTasks(ctx context.Context, filter *models.CallFilter) (models.FnCalls, error) {
+	return m.Calls, nil
+}

api/datastore/mysql/mysql.go

@@ -1,14 +1,12 @@
 package mysql
 
 import (
-	"bytes"
 	"context"
 	"database/sql"
 	"encoding/json"
 	"fmt"
 	"net/url"
 
-	"github.com/Sirupsen/logrus"
 	"github.com/go-sql-driver/mysql"
 	_ "github.com/go-sql-driver/mysql"
 	"gitlab-odx.oracle.com/odx/functions/api/datastore/internal/datastoreutil"
@@ -42,13 +40,19 @@ const extrasTableCreate = `CREATE TABLE IF NOT EXISTS extras (
 
 const routeSelector = `SELECT app_name, path, image, format, maxc, memory, type, timeout, idle_timeout, headers, config FROM routes`
 
-type rowScanner interface {
-	Scan(dest ...interface{}) error
-}
+const callTableCreate = `CREATE TABLE IF NOT EXISTS calls (
+	created_at varchar(256) NOT NULL,
+	started_at varchar(256) NOT NULL,
+	completed_at varchar(256) NOT NULL,
+	status varchar(256) NOT NULL,
+	id varchar(256) NOT NULL,
+	app_name varchar(256) NOT NULL,
+	path varchar(256) NOT NULL,
+	PRIMARY KEY (id)
+);`
+
+const callSelector = `SELECT id, created_at, started_at, completed_at, status, app_name, path FROM calls`
 
-type rowQuerier interface {
-	QueryRow(query string, args ...interface{}) *sql.Row
-}
 
 /*
 MySQLDatastore defines a basic MySQL Datastore struct.
@@ -61,33 +65,19 @@ type MySQLDatastore struct {
 New creates a new MySQL Datastore.
 */
 func New(url *url.URL) (models.Datastore, error) {
-	u := fmt.Sprintf("%s@%s%s", url.User.String(), url.Host, url.Path)
-	db, err := sql.Open("mysql", u)
+	tables := []string{routesTableCreate, appsTableCreate, extrasTableCreate, callTableCreate}
+	dialect := "mysql"
+	sqlDatastore := &MySQLDatastore{}
+	dataSourceName := fmt.Sprintf("%s@%s%s", url.User.String(), url.Host, url.Path)
+
+	db, err := datastoreutil.NewDatastore(dataSourceName, dialect, tables)
 	if err != nil {
 		return nil, err
 	}
 
-	err = db.Ping()
-	if err != nil {
-		return nil, err
-	}
+	sqlDatastore.db = db
+	return datastoreutil.NewValidator(sqlDatastore), nil
 
-	maxIdleConns := 30
-	db.SetMaxIdleConns(maxIdleConns)
-	logrus.WithFields(logrus.Fields{"max_idle_connections": maxIdleConns}).Info("MySQL dialed")
-
-	pg := &MySQLDatastore{
-		db: db,
-	}
-
-	for _, v := range []string{routesTableCreate, appsTableCreate, extrasTableCreate} {
-		_, err = db.Exec(v)
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	return datastoreutil.NewValidator(pg), nil
 }
 
 /*
@@ -188,78 +178,26 @@ func (ds *MySQLDatastore) RemoveApp(ctx context.Context, appName string) error {
 	  WHERE name = ?
 	`, appName)
 
-	if err != nil {
-		return err
-	}
-
-	return nil
+	return err
 }
 
 /*
 GetApp retrieves an app from MySQL.
 */
 func (ds *MySQLDatastore) GetApp(ctx context.Context, name string) (*models.App, error) {
-	row := ds.db.QueryRow(`SELECT name, config FROM apps WHERE name=?`, name)
-
-	var resName string
-	var config string
-	err := row.Scan(&resName, &config)
-
-	res := &models.App{
-		Name: resName,
-	}
-
-	json.Unmarshal([]byte(config), &res.Config)
-
-	if err != nil {
-		if err == sql.ErrNoRows {
-			return nil, models.ErrAppsNotFound
-		}
-		return nil, err
-	}
-
-	return res, nil
-}
-
-func scanApp(scanner rowScanner, app *models.App) error {
-	var configStr string
-
-	err := scanner.Scan(
-		&app.Name,
-		&configStr,
-	)
-
-	json.Unmarshal([]byte(configStr), &app.Config)
-
-	return err
+	queryStr := `SELECT name, config FROM apps WHERE name=?`
+	queryArgs := []interface{}{name}
+	return datastoreutil.SQLGetApp(ds.db, queryStr, queryArgs...)
 }
 
 /*
 GetApps retrieves an array of apps according to a specific filter.
 */
 func (ds *MySQLDatastore) GetApps(ctx context.Context, filter *models.AppFilter) ([]*models.App, error) {
-	res := []*models.App{}
-	filterQuery, args := buildFilterAppQuery(filter)
-	rows, err := ds.db.Query(fmt.Sprintf("SELECT DISTINCT name, config FROM apps %s", filterQuery), args...)
-	if err != nil {
-		return nil, err
-	}
-	defer rows.Close()
+	whereStm := "WHERE name LIKE ?"
+	selectStm := "SELECT DISTINCT name, config FROM apps %s"
 
-	for rows.Next() {
-		var app models.App
-		err := scanApp(rows, &app)
-
-		if err != nil {
-			break
-		}
-		res = append(res, &app)
-	}
-
-	if err := rows.Err(); err != nil {
-		return res, err
-	}
-	return res, nil
+	return datastoreutil.SQLGetApps(ds.db, filter, whereStm, selectStm)
 }
 
 /*
@@ -336,7 +274,7 @@ func (ds *MySQLDatastore) UpdateRoute(ctx context.Context, newroute *models.Rout
 	var route models.Route
 	err := ds.Tx(func(tx *sql.Tx) error {
 		row := ds.db.QueryRow(fmt.Sprintf("%s WHERE app_name=? AND path=?", routeSelector), newroute.AppName, newroute.Path)
-		if err := scanRoute(row, &route); err == sql.ErrNoRows {
+		if err := datastoreutil.ScanRoute(row, &route); err == sql.ErrNoRows {
 			return models.ErrRoutesNotFound
 		} else if err != nil {
 			return err
@@ -402,174 +340,39 @@ func (ds *MySQLDatastore) UpdateRoute(ctx context.Context, newroute *models.Rout
 RemoveRoute removes an existing route on MySQL.
 */
 func (ds *MySQLDatastore) RemoveRoute(ctx context.Context, appName, routePath string) error {
-	res, err := ds.db.Exec(`
-		DELETE FROM routes
-		WHERE path = ? AND app_name = ?
-	`, routePath, appName)
+	deleteStm := `DELETE FROM routes WHERE path = ? AND app_name = ?`
 
-	if err != nil {
-		return err
-	}
-
-	n, err := res.RowsAffected()
-	if err != nil {
-		return err
-	}
-
-	if n == 0 {
-		return models.ErrRoutesRemoving
-	}
-
-	return nil
-}
-
-func scanRoute(scanner rowScanner, route *models.Route) error {
-	var headerStr string
-	var configStr string
-
-	err := scanner.Scan(
-		&route.AppName,
-		&route.Path,
-		&route.Image,
-		&route.Format,
-		&route.MaxConcurrency,
-		&route.Memory,
-		&route.Type,
-		&route.Timeout,
-		&route.IdleTimeout,
-		&headerStr,
-		&configStr,
-	)
-	if err != nil {
-		return err
-	}
-
-	if headerStr == "" {
-		return models.ErrRoutesNotFound
-	}
-
-	if err := json.Unmarshal([]byte(headerStr), &route.Headers); err != nil {
-		return err
-	}
-	return json.Unmarshal([]byte(configStr), &route.Config)
+	return datastoreutil.SQLRemoveRoute(ds.db, appName, routePath, deleteStm)
 }
 
 /*
 GetRoute retrieves a route from MySQL.
 */
 func (ds *MySQLDatastore) GetRoute(ctx context.Context, appName, routePath string) (*models.Route, error) {
-	var route models.Route
+	rSelectCondition := "%s WHERE app_name=? AND path=?"
 
-	row := ds.db.QueryRow(fmt.Sprintf("%s WHERE app_name=? AND path=?", routeSelector), appName, routePath)
-	err := scanRoute(row, &route)
-
-	if err == sql.ErrNoRows {
-		return nil, models.ErrRoutesNotFound
-	} else if err != nil {
-		return nil, err
-	}
-	return &route, nil
+	return datastoreutil.SQLGetRoute(ds.db, appName, routePath, rSelectCondition, routeSelector)
 }
 
 /*
 GetRoutes retrieves an array of routes according to a specific filter.
 */
 func (ds *MySQLDatastore) GetRoutes(ctx context.Context, filter *models.RouteFilter) ([]*models.Route, error) {
-	res := []*models.Route{}
-	filterQuery, args := buildFilterRouteQuery(filter)
-	rows, err := ds.db.Query(fmt.Sprintf("%s %s", routeSelector, filterQuery), args...)
-	if err != nil {
-		return nil, err
-	}
-	defer rows.Close()
+	whereStm := "WHERE %s ?"
+	andStm := " AND %s ?"
 
-	for rows.Next() {
-		var route models.Route
-		err := scanRoute(rows, &route)
-		if err != nil {
-			continue
-		}
-		res = append(res, &route)
-
-	}
-	if err := rows.Err(); err != nil {
-		return nil, err
-	}
-	return res, nil
+	return datastoreutil.SQLGetRoutes(ds.db, filter, routeSelector, whereStm, andStm)
 }
 
 /*
 GetRoutesByApp retrieves a route with a specific app name.
 */
 func (ds *MySQLDatastore) GetRoutesByApp(ctx context.Context, appName string, filter *models.RouteFilter) ([]*models.Route, error) {
-	res := []*models.Route{}
+	whereStm := "WHERE %s ?"
+	andStm := " AND %s ?"
+	defaultFilterQuery := "WHERE app_name = ?"
 
-	var filterQuery string
-	var args []interface{}
-	if filter == nil {
-		filterQuery = "WHERE app_name = ?"
-		args = []interface{}{appName}
-	} else {
-		filter.AppName = appName
-		filterQuery, args = buildFilterRouteQuery(filter)
-	}
-	rows, err := ds.db.Query(fmt.Sprintf("%s %s", routeSelector, filterQuery), args...)
-	if err != nil {
-		return nil, err
-	}
-	defer rows.Close()
-
-	for rows.Next() {
-		var route models.Route
-		err := scanRoute(rows, &route)
-		if err != nil {
-			continue
-		}
-		res = append(res, &route)
-
-	}
-	if err := rows.Err(); err != nil {
-		return nil, err
-	}
-
-	return res, nil
-}
-
-func buildFilterAppQuery(filter *models.AppFilter) (string, []interface{}) {
-	if filter == nil {
-		return "", nil
-	}
-
-	if filter.Name != "" {
-		return "WHERE name LIKE ?", []interface{}{filter.Name}
-	}
-
-	return "", nil
-}
-
-func buildFilterRouteQuery(filter *models.RouteFilter) (string, []interface{}) {
-	if filter == nil {
-		return "", nil
-	}
-	var b bytes.Buffer
-	var args []interface{}
-
-	where := func(colOp, val string) {
-		if val != "" {
-			args = append(args, val)
-			if len(args) == 1 {
-				fmt.Fprintf(&b, "WHERE %s ?", colOp)
-			} else {
-				fmt.Fprintf(&b, " AND %s ?", colOp)
-			}
-		}
-	}
-
-	where("path =", filter.Path)
-	where("app_name =", filter.AppName)
-	where("image =", filter.Image)
-
-	return b.String(), args
+	return datastoreutil.SQLGetRoutesByApp(ds.db, appName, filter, routeSelector, defaultFilterQuery, whereStm, andStm)
 }
 
 /*
@@ -625,3 +428,32 @@ func (ds *MySQLDatastore) Tx(f func(*sql.Tx) error) error {
 	}
 	return tx.Commit()
 }
+
+func (ds * MySQLDatastore) InsertTask(ctx context.Context, task *models.Task) error {
+	stmt, err := ds.db.Prepare("INSERT calls SET id=?,created_at=?,started_at=?,completed_at=?,status=?,app_name=?,path=?")
+	if err != nil {
+		return err
+	}
+	_, err = stmt.Exec(task.ID, task.CreatedAt.String(),
+		task.StartedAt.String(), task.CompletedAt.String(),
+		task.Status, task.AppName, task.Path)
+
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (ds * MySQLDatastore) GetTask(ctx context.Context, callID string) (*models.FnCall, error) {
+	whereStm := "%s WHERE id=?"
+
+	return datastoreutil.SQLGetCall(ds.db, callSelector, callID, whereStm)
+}
+
+func (ds * MySQLDatastore) GetTasks(ctx context.Context, filter *models.CallFilter) (models.FnCalls, error) {
+	whereStm := "WHERE %s ?"
+	andStm := " AND %s ?"
+
+	return datastoreutil.SQLGetCalls(ds.db, callSelector, filter, whereStm, andStm)
+}

api/datastore/postgres/postgres.go

@@ -8,8 +8,6 @@ import (
 
 	"context"
 
-	"bytes"
-	"github.com/Sirupsen/logrus"
 	"gitlab-odx.oracle.com/odx/functions/api/datastore/internal/datastoreutil"
 	"gitlab-odx.oracle.com/odx/functions/api/models"
 	"github.com/lib/pq"
@@ -44,41 +42,35 @@ const extrasTableCreate = `CREATE TABLE IF NOT EXISTS extras (
 
 const routeSelector = `SELECT app_name, path, image, format, maxc, memory, type, timeout, idle_timeout, headers, config FROM routes`
 
-type rowScanner interface {
-	Scan(dest ...interface{}) error
-}
+const callsTableCreate = `CREATE TABLE IF NOT EXISTS calls (
+	created_at character varying(256) NOT NULL,
+	started_at character varying(256) NOT NULL,
+	completed_at character varying(256) NOT NULL,
+	status character varying(256) NOT NULL,
+	id character varying(256) NOT NULL,
+	app_name character varying(256) NOT NULL,
+	path character varying(256) NOT NULL,
+	PRIMARY KEY (id)
+);`
+
+const callSelector = `SELECT id, created_at, started_at, completed_at, status, app_name, path FROM calls`
 
 type PostgresDatastore struct {
 	db *sql.DB
 }
 
 func New(url *url.URL) (models.Datastore, error) {
-	db, err := sql.Open("postgres", url.String())
+	tables := []string{routesTableCreate, appsTableCreate, extrasTableCreate, callsTableCreate}
+	sqlDatastore := &PostgresDatastore{}
+	dialect := "postgres"
+
+	db, err := datastoreutil.NewDatastore(url.String(), dialect, tables)
 	if err != nil {
 		return nil, err
 	}
 
-	err = db.Ping()
-	if err != nil {
-		return nil, err
-	}
-
-	maxIdleConns := 30 // c.MaxIdleConnections
-	db.SetMaxIdleConns(maxIdleConns)
-	logrus.WithFields(logrus.Fields{"max_idle_connections": maxIdleConns}).Info("Postgres dialed")
-
-	pg := &PostgresDatastore{
-		db: db,
-	}
-
-	for _, v := range []string{routesTableCreate, appsTableCreate, extrasTableCreate} {
-		_, err = db.Exec(v)
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	return datastoreutil.NewValidator(pg), nil
+	sqlDatastore.db = db
+	return datastoreutil.NewValidator(sqlDatastore), nil
 }
 
 func (ds *PostgresDatastore) InsertApp(ctx context.Context, app *models.App) (*models.App, error) {
@@ -156,93 +148,23 @@ func (ds *PostgresDatastore) UpdateApp(ctx context.Context, newapp *models.App)
 }
 
 func (ds *PostgresDatastore) RemoveApp(ctx context.Context, appName string) error {
-	_, err := ds.db.Exec(`
-	  DELETE FROM apps
-	  WHERE name = $1
-	`, appName)
+	_, err := ds.db.Exec(`DELETE FROM apps WHERE name = $1`, appName)
 
-	if err != nil {
-		return err
-	}
-
-	return nil
+	return err
 }
 
 func (ds *PostgresDatastore) GetApp(ctx context.Context, name string) (*models.App, error) {
-	row := ds.db.QueryRow("SELECT name, config FROM apps WHERE name=$1", name)
+	queryStr := "SELECT name, config FROM apps WHERE name=$1"
+	queryArgs := []interface{}{name}
 
-	var resName string
-	var config string
-	err := row.Scan(&resName, &config)
-	if err != nil {
-		if err == sql.ErrNoRows {
-			return nil, models.ErrAppsNotFound
-		}
-		return nil, err
-	}
-
-	res := &models.App{
-		Name: resName,
-	}
-
-	if len(config) > 0 {
-		err := json.Unmarshal([]byte(config), &res.Config)
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	return res, nil
-}
-
-func scanApp(scanner rowScanner, app *models.App) error {
-	var configStr string
-
-	err := scanner.Scan(
-		&app.Name,
-		&configStr,
-	)
-	if err != nil {
-		return err
-	}
-
-	if len(configStr) > 0 {
-		err = json.Unmarshal([]byte(configStr), &app.Config)
-		if err != nil {
-			return err
-		}
-	}
-
-	return nil
+	return datastoreutil.SQLGetApp(ds.db, queryStr, queryArgs...)
 }
 
 func (ds *PostgresDatastore) GetApps(ctx context.Context, filter *models.AppFilter) ([]*models.App, error) {
-	res := []*models.App{}
+	whereStm := "WHERE name LIKE $1"
+	selectStm := "SELECT DISTINCT * FROM apps %s"
 
-	filterQuery, args := buildFilterAppQuery(filter)
-	rows, err := ds.db.Query(fmt.Sprintf("SELECT DISTINCT * FROM apps %s", filterQuery), args...)
-	if err != nil {
-		return nil, err
-	}
-	defer rows.Close()
-
-	for rows.Next() {
-		var app models.App
-		err := scanApp(rows, &app)
-
-		if err != nil {
-			if err == sql.ErrNoRows {
-				return res, nil
-			}
-			return res, err
-		}
-		res = append(res, &app)
-	}
-
-	if err := rows.Err(); err != nil {
-		return res, err
-	}
-	return res, nil
+	return datastoreutil.SQLGetApps(ds.db, filter, whereStm, selectStm)
 }
 
 func (ds *PostgresDatastore) InsertRoute(ctx context.Context, route *models.Route) (*models.Route, error) {
@@ -315,7 +237,7 @@ func (ds *PostgresDatastore) UpdateRoute(ctx context.Context, newroute *models.R
 	var route models.Route
 	err := ds.Tx(func(tx *sql.Tx) error {
 		row := ds.db.QueryRow(fmt.Sprintf("%s WHERE app_name=$1 AND path=$2", routeSelector), newroute.AppName, newroute.Path)
-		if err := scanRoute(row, &route); err == sql.ErrNoRows {
+		if err := datastoreutil.ScanRoute(row, &route); err == sql.ErrNoRows {
 			return models.ErrRoutesNotFound
 		} else if err != nil {
 			return err
@@ -378,173 +300,30 @@ func (ds *PostgresDatastore) UpdateRoute(ctx context.Context, newroute *models.R
 }
 
 func (ds *PostgresDatastore) RemoveRoute(ctx context.Context, appName, routePath string) error {
-	res, err := ds.db.Exec(`
-		DELETE FROM routes
-		WHERE path = $1 AND app_name = $2
-	`, routePath, appName)
+	deleteStm := `DELETE FROM routes WHERE path = $1 AND app_name = $2`
 
-	if err != nil {
-		return err
-	}
-
-	n, err := res.RowsAffected()
-	if err != nil {
-		return err
-	}
-
-	if n == 0 {
-		return models.ErrRoutesRemoving
-	}
-
-	return nil
-}
-
-func scanRoute(scanner rowScanner, route *models.Route) error {
-	var headerStr string
-	var configStr string
-
-	err := scanner.Scan(
-		&route.AppName,
-		&route.Path,
-		&route.Image,
-		&route.Format,
-		&route.MaxConcurrency,
-		&route.Memory,
-		&route.Type,
-		&route.Timeout,
-		&route.IdleTimeout,
-		&headerStr,
-		&configStr,
-	)
-	if err != nil {
-		return err
-	}
-
-	if len(headerStr) > 0 {
-		err = json.Unmarshal([]byte(headerStr), &route.Headers)
-		if err != nil {
-			return err
-		}
-	}
-
-	if len(configStr) > 0 {
-		err = json.Unmarshal([]byte(configStr), &route.Config)
-		if err != nil {
-			return err
-		}
-	}
-
-	return nil
+	return datastoreutil.SQLRemoveRoute(ds.db, appName, routePath, deleteStm)
 }
 
 func (ds *PostgresDatastore) GetRoute(ctx context.Context, appName, routePath string) (*models.Route, error) {
-	var route models.Route
+	rSelectCondition := "%s WHERE app_name=$1 AND path=$2"
 
-	row := ds.db.QueryRow(fmt.Sprintf("%s WHERE app_name=$1 AND path=$2", routeSelector), appName, routePath)
-	err := scanRoute(row, &route)
-
-	if err == sql.ErrNoRows {
-		return nil, models.ErrRoutesNotFound
-	} else if err != nil {
-		return nil, err
-	}
-	return &route, nil
+	return datastoreutil.SQLGetRoute(ds.db, appName, routePath, rSelectCondition, routeSelector)
 }
 
 func (ds *PostgresDatastore) GetRoutes(ctx context.Context, filter *models.RouteFilter) ([]*models.Route, error) {
-	res := []*models.Route{}
-	filterQuery, args := buildFilterRouteQuery(filter)
-	rows, err := ds.db.Query(fmt.Sprintf("%s %s", routeSelector, filterQuery), args...)
-	// todo: check for no rows so we don't respond with a sql 500 err
-	if err != nil {
-		return nil, err
-	}
-	defer rows.Close()
+	whereStm := "WHERE %s $1"
+	andStm := " AND %s $%d"
 
-	for rows.Next() {
-		var route models.Route
-		err := scanRoute(rows, &route)
-		if err != nil {
-			continue
-		}
-		res = append(res, &route)
-
-	}
-	if err := rows.Err(); err != nil {
-		return nil, err
-	}
-	return res, nil
+	return datastoreutil.SQLGetRoutes(ds.db, filter, routeSelector, whereStm, andStm)
 }
 
 func (ds *PostgresDatastore) GetRoutesByApp(ctx context.Context, appName string, filter *models.RouteFilter) ([]*models.Route, error) {
-	res := []*models.Route{}
+	defaultFilterQuery := "WHERE app_name = $1"
+	whereStm := "WHERE %s $1"
+	andStm := " AND %s $%d"
 
-	var filterQuery string
-	var args []interface{}
-	if filter == nil {
-		filterQuery = "WHERE app_name = $1"
-		args = []interface{}{appName}
-	} else {
-		filter.AppName = appName
-		filterQuery, args = buildFilterRouteQuery(filter)
-	}
-	rows, err := ds.db.Query(fmt.Sprintf("%s %s", routeSelector, filterQuery), args...)
-	// todo: check for no rows so we don't respond with a sql 500 err
-	if err != nil {
-		return nil, err
-	}
-	defer rows.Close()
-
-	for rows.Next() {
-		var route models.Route
-		err := scanRoute(rows, &route)
-		if err != nil {
-			continue
-		}
-		res = append(res, &route)
-
-	}
-	if err := rows.Err(); err != nil {
-		return nil, err
-	}
-
-	return res, nil
-}
-func buildFilterAppQuery(filter *models.AppFilter) (string, []interface{}) {
-	if filter == nil {
-		return "", nil
-	}
-
-	if filter.Name != "" {
-		return "WHERE name LIKE $1", []interface{}{filter.Name}
-	}
-
-	return "", nil
-}
-
-func buildFilterRouteQuery(filter *models.RouteFilter) (string, []interface{}) {
-	if filter == nil {
-		return "", nil
-	}
-	var b bytes.Buffer
-	var args []interface{}
-
-	where := func(colOp, val string) {
-		if val != "" {
-			args = append(args, val)
-			if len(args) == 1 {
-				fmt.Fprintf(&b, "WHERE %s $1", colOp)
-			} else {
-				fmt.Fprintf(&b, " AND %s $%d", colOp, len(args))
-			}
-		}
-	}
-
-	where("path =", filter.Path)
-	where("app_name =", filter.AppName)
-	where("image =", filter.Image)
-
-	return b.String(), args
+	return datastoreutil.SQLGetRoutesByApp(ds.db, appName, filter, routeSelector, defaultFilterQuery, whereStm, andStm)
 }
 
 func (ds *PostgresDatastore) Put(ctx context.Context, key, value []byte) error {
@@ -591,3 +370,40 @@ func (ds *PostgresDatastore) Tx(f func(*sql.Tx) error) error {
 	}
 	return tx.Commit()
 }
+
+func (ds *PostgresDatastore) InsertTask(ctx context.Context, task *models.Task) error {
+	err := ds.Tx(func(tx *sql.Tx) error {
+		_, err := tx.Exec(
+			`INSERT INTO calls (
+				id,
+				created_at,
+				started_at,
+				completed_at,
+				status,
+				app_name,
+				path) VALUES ($1, $2, $3, $4, $5, $6, $7);`,
+			task.ID,
+			task.CreatedAt.String(),
+			task.StartedAt.String(),
+			task.CompletedAt.String(),
+			task.Status,
+			task.AppName,
+			task.Path,
+		)
+		return err
+	})
+	return err
+}
+
+func (ds *PostgresDatastore) GetTask(ctx context.Context, callID string) (*models.FnCall, error) {
+	whereStm := "%s WHERE id=$1"
+
+	return datastoreutil.SQLGetCall(ds.db, callSelector, callID, whereStm)
+}
+
+func (ds *PostgresDatastore) GetTasks(ctx context.Context, filter *models.CallFilter) (models.FnCalls, error) {
+	whereStm := "WHERE %s $1"
+	andStm := " AND %s $2"
+
+	return datastoreutil.SQLGetCalls(ds.db, callSelector, filter, whereStm, andStm)
+}

api/datastore/redis/redis.go

@@ -310,3 +310,65 @@ func applyRouteFilter(route *models.Route, filter *models.RouteFilter) bool {
 		(filter.AppName == "" || route.AppName == filter.AppName) &&
 		(filter.Image == "" || route.Image == filter.Image)
 }
+
+func applyCallFilter(call *models.FnCall, filter *models.CallFilter) bool {
+	return filter == nil || (filter.Path == "" || call.Path == filter.Path) &&
+		(filter.AppName == "" || call.AppName == filter.AppName)
+}
+
+func (ds *RedisDataStore) InsertTask(ctx context.Context, task *models.Task) error {
+	_, err := ds.conn.Do("HEXISTS", "calls", task.ID)
+	if err != nil {
+		return err
+	}
+
+	taskBytes, err := json.Marshal(task)
+	if err != nil {
+		return err
+	}
+
+	if _, err := ds.conn.Do("HSET", "calls", task.ID, taskBytes); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (ds *RedisDataStore) GetTask(ctx context.Context, callID string) (*models.FnCall, error) {
+	reply, err := ds.conn.Do("HGET", "calls", callID)
+	if err != nil {
+		return nil, err
+	} else if reply == nil {
+		return nil, models.ErrCallNotFound
+	}
+	res := &models.FnCall{}
+	if err := json.Unmarshal(reply.([]byte), res); err != nil {
+		return nil, err
+	}
+	return res, nil
+}
+
+func (ds *RedisDataStore) GetTasks(ctx context.Context, filter *models.CallFilter) (models.FnCalls, error) {
+	res := models.FnCalls{}
+
+	reply, err := ds.conn.Do("HGETALL", "calls")
+	if err != nil {
+		return nil, err
+	}
+
+	calls, err := redis.StringMap(reply, err)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, v := range calls {
+		var call models.FnCall
+		if err := json.Unmarshal([]byte(v), &call); err != nil {
+			return nil, err
+		}
+		if applyCallFilter(&call, filter) {
+			res = append(res, &call)
+		}
+	}
+	return res, nil
+
+}