alihan/fn-serverless
1
0
Fork 0
mirror of https://github.com/fnproject/fn.git synced 2022-10-28 21:29:17 +03:00
Code Issues Packages Projects Releases Wiki Activity

App ID (#641)

Browse Source 
* App ID

* Clean-up

* Use ID or name to reference apps

* Can use app by name or ID

* Get rid of AppName for routes API and model

 routes API is completely backwards-compatible
 routes API accepts both app ID and name

* Get rid of AppName from calls API and model

* Fixing tests

* Get rid of AppName from logs API and model

* Restrict API to work with app names only

* Addressing review comments

* Fix for hybrid mode

* Fix rebase problems

* Addressing review comments

* Addressing review comments pt.2

* Fixing test issue

* Addressing review comments pt.3

* Updated docstring

* Adjust UpdateApp SQL implementation to work with app IDs instead of names

* Fixing tests

* fmt after rebase

* Make tests green again!

* Use GetAppByID wherever it is necessary

 - adding new v2 endpoints to keep hybrid api/runner mode working
 - extract CallBase from Call object to expose that to a user
   (it doesn't include any app reference, as we do for all other API objects)

* Get rid of GetAppByName

* Adjusting server router setup

* Make hybrid work again

* Fix datastore tests

* Fixing tests

* Do not ignore app_id

* Resolve issues after rebase

* Updating test to make it work as it was

* Tabula rasa for migrations

* Adding calls API test

 - we need to ensure we give "App not found" for the missing app and missing call in first place
 - making previous test work (request missing call for the existing app)

* Make datastore tests work fine with correctly applied migrations

* Make CallFunction middleware work again

 had to adjust its implementation to set app ID before proceeding

* The biggest rebase ever made

* Fix 8's migration

* Fix tests

* Fix hybrid client

* Fix tests problem

* Increment app ID migration version

* Fixing TestAppUpdate

* Fix rebase issues

* Addressing review comments

* Renew vendor

* Updated swagger doc per recommendations
This commit is contained in:
Denis Makogon
2018-03-26 21:19:36 +03:00
committed by Reed Allman
parent 4e90844a67
commit 3c15ca6ea6
59 changed files with 1101 additions and 657 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
api/datastore/sql/migratex/migrate.go
View File

@@ -111,7 +111,8 @@ func migrate(ctx context.Context, db *sqlx.DB, migs []Migration, up bool) error
}
for _, m := range migs {
// skip over migrations we have run
if (up && curVersion < m.Version()) || (!up && curVersion >= m.Version()) {
mVersion := m.Version()
if (up && curVersion < mVersion) || (!up && curVersion >= mVersion) {
// do each individually, for large migrations it's better to checkpoint
// than to try to do them all in one big go.

159
api/datastore/sql/migrations/10_add_app_id.go Normal file
View File

@@ -0,0 +1,159 @@
package migrations
import (
"context"
"database/sql"
"fmt"
"github.com/fnproject/fn/api/datastore/sql/migratex"
"github.com/fnproject/fn/api/id"
"github.com/fnproject/fn/api/models"
"github.com/jmoiron/sqlx"
)
func up10(ctx context.Context, tx *sqlx.Tx) error {
addAppIDStatements := []string{
"ALTER TABLE apps ADD id VARCHAR(256);",
"ALTER TABLE calls ADD app_id VARCHAR(256);",
"ALTER TABLE logs ADD app_id VARCHAR(256);",
"ALTER TABLE routes ADD app_id VARCHAR(256);",
}
for _, statement := range addAppIDStatements {
_, err := tx.ExecContext(ctx, statement)
if err != nil {
return err
}
}
rows, err := tx.QueryxContext(ctx, "SELECT DISTINCT name FROM apps;")
if err != nil {
return err
}
res := []*models.App{}
for rows.Next() {
var app models.App
err := rows.StructScan(&app)
if err != nil {
if err == sql.ErrNoRows {
return nil
}
return err
}
app.ID = id.New().String()
res = append(res, &app)
}
err = rows.Close()
if err != nil {
return err
}
if err := rows.Err(); err != nil {
return err
}
// it is required for some reason, can't do this within the rows iteration.
for _, app := range res {
query := tx.Rebind(`UPDATE apps SET id=:id WHERE name=:name`)
_, err = tx.NamedExecContext(ctx, query, app)
if err != nil {
return err
}
for _, t := range []string{"routes", "calls", "logs"} {
q := fmt.Sprintf(`UPDATE %s SET app_id=:id WHERE app_name=:name;`, t)
_, err = tx.NamedExecContext(ctx, tx.Rebind(q), app)
if err != nil {
return err
}
}
}
dropAppNameStatements := []string{
"ALTER TABLE routes DROP COLUMN app_name;",
"ALTER TABLE calls DROP COLUMN app_name;",
"ALTER TABLE logs DROP COLUMN app_name;",
}
for _, statement := range dropAppNameStatements {
_, err := tx.ExecContext(ctx, statement)
if err != nil {
return err
}
}
return nil
}
func down10(ctx context.Context, tx *sqlx.Tx) error {
addAppNameStatements := []string{
"ALTER TABLE calls ADD app_name VARCHAR(256);",
"ALTER TABLE logs ADD app_name VARCHAR(256);",
"ALTER TABLE routes ADD app_name VARCHAR(256);",
}
for _, statement := range addAppNameStatements {
_, err := tx.ExecContext(ctx, statement)
if err != nil {
return err
}
}
rows, err := tx.QueryxContext(ctx, "SELECT DISTINCT id, name FROM apps;")
if err != nil {
return err
}
res := []*models.App{}
for rows.Next() {
var app models.App
err := rows.StructScan(&app)
if err != nil {
if err == sql.ErrNoRows {
return nil
}
return err
}
res = append(res, &app)
}
err = rows.Close()
if err != nil {
return err
}
if err := rows.Err(); err != nil {
return err
}
// it is required for some reason, can't do this within the rows iteration.
for _, app := range res {
for _, t := range []string{"routes", "calls", "logs"} {
q := "UPDATE " + t + " SET app_name=:name WHERE app_id=:id;"
_, err = tx.NamedExecContext(ctx, tx.Rebind(q), app)
if err != nil {
return err
}
}
}
removeAppIDStatements := []string{
"ALTER TABLE logs DROP COLUMN app_id;",
"ALTER TABLE calls DROP COLUMN app_id;",
"ALTER TABLE routes DROP COLUMN app_id;",
"ALTER TABLE apps DROP COLUMN id;",
}
for _, statement := range removeAppIDStatements {
_, err := tx.ExecContext(ctx, statement)
if err != nil {
return err
}
}
return nil
}
func init() {
Migrations = append(Migrations, &migratex.MigFields{
VersionFunc: vfunc(10),
UpFunc: up10,
DownFunc: down10,
})
}

152
api/datastore/sql/sql.go
View File

@@ -39,7 +39,7 @@ import (
// with migrations (sadly, need complex transaction)
var tables = [...]string{`CREATE TABLE IF NOT EXISTS routes (
app_name varchar(256) NOT NULL,
app_id varchar(256) NOT NULL,
path varchar(256) NOT NULL,
image varchar(256) NOT NULL,
format varchar(16) NOT NULL,
@@ -53,10 +53,11 @@ var tables = [...]string{`CREATE TABLE IF NOT EXISTS routes (
annotations text NOT NULL,
created_at text,
updated_at varchar(256),
PRIMARY KEY (app_name, path)
PRIMARY KEY (app_id, path)
);`,
`CREATE TABLE IF NOT EXISTS apps (
id varchar(256),
name varchar(256) NOT NULL PRIMARY KEY,
config text NOT NULL,
annotations text NOT NULL,
@@ -70,7 +71,7 @@ var tables = [...]string{`CREATE TABLE IF NOT EXISTS routes (
completed_at varchar(256) NOT NULL,
status varchar(256) NOT NULL,
id varchar(256) NOT NULL,
app_name varchar(256) NOT NULL,
app_id varchar(256) NOT NULL,
path varchar(256) NOT NULL,
stats text,
error text,
@@ -79,14 +80,16 @@ var tables = [...]string{`CREATE TABLE IF NOT EXISTS routes (
`CREATE TABLE IF NOT EXISTS logs (
id varchar(256) NOT NULL PRIMARY KEY,
app_name varchar(256) NOT NULL,
app_id varchar(256) NOT NULL,
log text NOT NULL
);`,
}
const (
routeSelector = `SELECT app_name, path, image, format, memory, cpus, type, timeout, idle_timeout, headers, config, annotations, created_at, updated_at FROM routes`
callSelector = `SELECT id, created_at, started_at, completed_at, status, app_name, path, stats, error FROM calls`
routeSelector = `SELECT app_id, path, image, format, memory, type, cpus, timeout, idle_timeout, headers, config, annotations, created_at, updated_at FROM routes`
callSelector = `SELECT id, created_at, started_at, completed_at, status, app_id, path, stats, error FROM calls`
appIDSelector = `SELECT id, name, config, annotations, created_at, updated_at FROM apps WHERE id=?`
ensureAppSelector = `SELECT id FROM apps WHERE name=?`
)
type sqlStore struct {
@@ -252,8 +255,25 @@ func (ds *sqlStore) clear() error {
})
}
func (ds *sqlStore) GetAppID(ctx context.Context, appName string) (string, error) {
var app models.App
query := ds.db.Rebind(ensureAppSelector)
row := ds.db.QueryRowxContext(ctx, query, appName)
err := row.StructScan(&app)
if err == sql.ErrNoRows {
return "", models.ErrAppsNotFound
}
if err != nil {
return "", err
}
return app.ID, nil
}
func (ds *sqlStore) InsertApp(ctx context.Context, app *models.App) (*models.App, error) {
query := ds.db.Rebind(`INSERT INTO apps (
id,
name,
config,
annotations,
@@ -261,6 +281,7 @@ func (ds *sqlStore) InsertApp(ctx context.Context, app *models.App) (*models.App
updated_at
)
VALUES (
:id,
:name,
:config,
:annotations,
@@ -290,17 +311,19 @@ func (ds *sqlStore) InsertApp(ctx context.Context, app *models.App) (*models.App
}
func (ds *sqlStore) UpdateApp(ctx context.Context, newapp *models.App) (*models.App, error) {
app := &models.App{Name: newapp.Name}
var app models.App
err := ds.Tx(func(tx *sqlx.Tx) error {
// NOTE: must query whole object since we're returning app, Update logic
// must only modify modifiable fields (as seen here). need to fix brittle..
query := tx.Rebind(`SELECT name, config, annotations, created_at, updated_at FROM apps WHERE name=?`)
row := tx.QueryRowxContext(ctx, query, app.Name)
err := row.StructScan(app)
query := tx.Rebind(appIDSelector)
row := tx.QueryRowxContext(ctx, query, newapp.ID)
err := row.StructScan(&app)
if err == sql.ErrNoRows {
return models.ErrAppsNotFound
} else if err != nil {
}
if err != nil {
return err
}
@@ -329,12 +352,12 @@ func (ds *sqlStore) UpdateApp(ctx context.Context, newapp *models.App) (*models.
return nil, err
}
return app, nil
return &app, nil
}
func (ds *sqlStore) RemoveApp(ctx context.Context, appName string) error {
func (ds *sqlStore) RemoveApp(ctx context.Context, appID string) error {
return ds.Tx(func(tx *sqlx.Tx) error {
res, err := tx.ExecContext(ctx, tx.Rebind(`DELETE FROM apps WHERE name=?`), appName)
res, err := tx.ExecContext(ctx, tx.Rebind(`DELETE FROM apps WHERE id=?`), appID)
if err != nil {
return err
}
@@ -347,33 +370,34 @@ func (ds *sqlStore) RemoveApp(ctx context.Context, appName string) error {
}
deletes := []string{
`DELETE FROM logs WHERE app_name=?`,
`DELETE FROM calls WHERE app_name=?`,
`DELETE FROM routes WHERE app_name=?`,
`DELETE FROM logs WHERE app_id=?`,
`DELETE FROM calls WHERE app_id=?`,
`DELETE FROM routes WHERE app_id=?`,
}
for _, stmt := range deletes {
_, err := tx.ExecContext(ctx, tx.Rebind(stmt), appName)
_, err := tx.ExecContext(ctx, tx.Rebind(stmt), appID)
if err != nil {
return err
}
}
return nil
})
}
func (ds *sqlStore) GetApp(ctx context.Context, name string) (*models.App, error) {
query := ds.db.Rebind(`SELECT name, config, annotations, created_at, updated_at FROM apps WHERE name=?`)
row := ds.db.QueryRowxContext(ctx, query, name)
func (ds *sqlStore) GetAppByID(ctx context.Context, appID string) (*models.App, error) {
var app models.App
query := ds.db.Rebind(appIDSelector)
row := ds.db.QueryRowxContext(ctx, query, appID)
var res models.App
err := row.StructScan(&res)
err := row.StructScan(&app)
if err == sql.ErrNoRows {
return nil, models.ErrAppsNotFound
} else if err != nil {
}
if err != nil {
return nil, err
}
return &res, nil
return &app, err
}
// GetApps retrieves an array of apps according to a specific filter.
@@ -413,15 +437,15 @@ func (ds *sqlStore) GetApps(ctx context.Context, filter *models.AppFilter) ([]*m
func (ds *sqlStore) InsertRoute(ctx context.Context, route *models.Route) (*models.Route, error) {
err := ds.Tx(func(tx *sqlx.Tx) error {
query := tx.Rebind(`SELECT 1 FROM apps WHERE name=?`)
r := tx.QueryRowContext(ctx, query, route.AppName)
query := tx.Rebind(`SELECT 1 FROM apps WHERE id=?`)
r := tx.QueryRowContext(ctx, query, route.AppID)
if err := r.Scan(new(int)); err != nil {
if err == sql.ErrNoRows {
return models.ErrAppsNotFound
}
}
query = tx.Rebind(`SELECT 1 FROM routes WHERE app_name=? AND path=?`)
same, err := tx.QueryContext(ctx, query, route.AppName, route.Path)
query = tx.Rebind(`SELECT 1 FROM routes WHERE app_id=? AND path=?`)
same, err := tx.QueryContext(ctx, query, route.AppID, route.Path)
if err != nil {
return err
}
@@ -431,7 +455,7 @@ func (ds *sqlStore) InsertRoute(ctx context.Context, route *models.Route) (*mode
}
query = tx.Rebind(`INSERT INTO routes (
app_name,
app_id,
path,
image,
format,
@@ -447,7 +471,7 @@ func (ds *sqlStore) InsertRoute(ctx context.Context, route *models.Route) (*mode
updated_at
)
VALUES (
:app_name,
:app_id,
:path,
:image,
:format,
@@ -474,8 +498,8 @@ func (ds *sqlStore) InsertRoute(ctx context.Context, route *models.Route) (*mode
func (ds *sqlStore) UpdateRoute(ctx context.Context, newroute *models.Route) (*models.Route, error) {
var route models.Route
err := ds.Tx(func(tx *sqlx.Tx) error {
query := tx.Rebind(fmt.Sprintf("%s WHERE app_name=? AND path=?", routeSelector))
row := tx.QueryRowxContext(ctx, query, newroute.AppName, newroute.Path)
query := tx.Rebind(fmt.Sprintf("%s WHERE app_id=? AND path=?", routeSelector))
row := tx.QueryRowxContext(ctx, query, newroute.AppID, newroute.Path)
err := row.StructScan(&route)
if err == sql.ErrNoRows {
@@ -502,7 +526,7 @@ func (ds *sqlStore) UpdateRoute(ctx context.Context, newroute *models.Route) (*m
config = :config,
annotations = :annotations,
updated_at = :updated_at
WHERE app_name=:app_name AND path=:path;`)
WHERE app_id=:app_id AND path=:path;`)
res, err := tx.NamedExecContext(ctx, query, &route)
if err != nil {
@@ -525,9 +549,9 @@ func (ds *sqlStore) UpdateRoute(ctx context.Context, newroute *models.Route) (*m
return &route, nil
}
func (ds *sqlStore) RemoveRoute(ctx context.Context, appName, routePath string) error {
query := ds.db.Rebind(`DELETE FROM routes WHERE path = ? AND app_name = ?`)
res, err := ds.db.ExecContext(ctx, query, routePath, appName)
func (ds *sqlStore) RemoveRoute(ctx context.Context, appID string, routePath string) error {
query := ds.db.Rebind(`DELETE FROM routes WHERE path = ? AND app_id = ?`)
res, err := ds.db.ExecContext(ctx, query, routePath, appID)
if err != nil {
return err
}
@@ -544,10 +568,10 @@ func (ds *sqlStore) RemoveRoute(ctx context.Context, appName, routePath string)
return nil
}
func (ds *sqlStore) GetRoute(ctx context.Context, appName, routePath string) (*models.Route, error) {
rSelectCondition := "%s WHERE app_name=? AND path=?"
func (ds *sqlStore) GetRoute(ctx context.Context, appID, routePath string) (*models.Route, error) {
rSelectCondition := "%s WHERE app_id=? AND path=?"
query := ds.db.Rebind(fmt.Sprintf(rSelectCondition, routeSelector))
row := ds.db.QueryRowxContext(ctx, query, appName, routePath)
row := ds.db.QueryRowxContext(ctx, query, appID, routePath)
var route models.Route
err := row.StructScan(&route)
@@ -560,13 +584,13 @@ func (ds *sqlStore) GetRoute(ctx context.Context, appName, routePath string) (*m
}
// GetRoutesByApp retrieves a route with a specific app name.
func (ds *sqlStore) GetRoutesByApp(ctx context.Context, appName string, filter *models.RouteFilter) ([]*models.Route, error) {
func (ds *sqlStore) GetRoutesByApp(ctx context.Context, appID string, filter *models.RouteFilter) ([]*models.Route, error) {
res := []*models.Route{}
if filter == nil {
filter = new(models.RouteFilter)
}
filter.AppName = appName
filter.AppID = appID
filterQuery, args := buildFilterRouteQuery(filter)
query := fmt.Sprintf("%s %s", routeSelector, filterQuery)
@@ -617,7 +641,7 @@ func (ds *sqlStore) InsertCall(ctx context.Context, call *models.Call) error {
started_at,
completed_at,
status,
app_name,
app_id,
path,
stats,
error
@@ -628,7 +652,7 @@ func (ds *sqlStore) InsertCall(ctx context.Context, call *models.Call) error {
:started_at,
:completed_at,
:status,
:app_name,
:app_id,
:path,
:stats,
:error
@@ -646,25 +670,25 @@ func equivalentCalls(expected *models.Call, actual *models.Call) bool {
time.Time(expected.StartedAt).Unix() == time.Time(actual.StartedAt).Unix() &&
time.Time(expected.CompletedAt).Unix() == time.Time(actual.CompletedAt).Unix() &&
expected.Status == actual.Status &&
expected.AppName == actual.AppName &&
expected.Path == actual.Path &&
expected.Error == actual.Error &&
len(expected.Stats) == len(actual.Stats)
len(expected.Stats) == len(actual.Stats) &&
expected.AppID == actual.AppID
// TODO: We don't do comparisons of individual Stats. We probably should.
return equivalentFields
}
func (ds *sqlStore) UpdateCall(ctx context.Context, from *models.Call, to *models.Call) error {
// Assert that from and to are supposed to be the same call
if from.ID != to.ID || from.AppName != to.AppName {
if from.ID != to.ID || from.AppID != to.AppID {
return errors.New("assertion error: 'from' and 'to' calls refer to different app/ID")
}
// Atomic update
err := ds.Tx(func(tx *sqlx.Tx) error {
var call models.Call
query := tx.Rebind(fmt.Sprintf(`%s WHERE id=? AND app_name=?`, callSelector))
row := tx.QueryRowxContext(ctx, query, from.ID, from.AppName)
query := tx.Rebind(fmt.Sprintf(`%s WHERE id=? AND app_id=?`, callSelector))
row := tx.QueryRowxContext(ctx, query, from.ID, from.AppID)
err := row.StructScan(&call)
if err == sql.ErrNoRows {
@@ -686,11 +710,11 @@ func (ds *sqlStore) UpdateCall(ctx context.Context, from *models.Call, to *model
started_at = :started_at,
completed_at = :completed_at,
status = :status,
app_name = :app_name,
app_id = :app_id,
path = :path,
stats = :stats,
error = :error
WHERE id=:id AND app_name=:app_name;`)
WHERE id=:id AND app_id=:app_id;`)
res, err := tx.NamedExecContext(ctx, query, to)
if err != nil {
@@ -713,10 +737,10 @@ func (ds *sqlStore) UpdateCall(ctx context.Context, from *models.Call, to *model
return nil
}
func (ds *sqlStore) GetCall(ctx context.Context, appName, callID string) (*models.Call, error) {
query := fmt.Sprintf(`%s WHERE id=? AND app_name=?`, callSelector)
func (ds *sqlStore) GetCall(ctx context.Context, appID, callID string) (*models.Call, error) {
query := fmt.Sprintf(`%s WHERE id=? AND app_id=?`, callSelector)
query = ds.db.Rebind(query)
row := ds.db.QueryRowxContext(ctx, query, callID, appName)
row := ds.db.QueryRowxContext(ctx, query, callID, appID)
var call models.Call
err := row.StructScan(&call)
@@ -754,7 +778,7 @@ func (ds *sqlStore) GetCalls(ctx context.Context, filter *models.CallFilter) ([]
return res, nil
}
func (ds *sqlStore) InsertLog(ctx context.Context, appName, callID string, logR io.Reader) error {
func (ds *sqlStore) InsertLog(ctx context.Context, appID, callID string, logR io.Reader) error {
// coerce this into a string for sql
var log string
if stringer, ok := logR.(fmt.Stringer); ok {
@@ -767,14 +791,15 @@ func (ds *sqlStore) InsertLog(ctx context.Context, appName, callID string, logR
log = b.String()
}
query := ds.db.Rebind(`INSERT INTO logs (id, app_name, log) VALUES (?, ?, ?);`)
_, err := ds.db.ExecContext(ctx, query, callID, appName, log)
query := ds.db.Rebind(`INSERT INTO logs (id, app_id, log) VALUES (?, ?, ?);`)
_, err := ds.db.ExecContext(ctx, query, callID, appID, log)
return err
}
func (ds *sqlStore) GetLog(ctx context.Context, appName, callID string) (io.Reader, error) {
query := ds.db.Rebind(`SELECT log FROM logs WHERE id=? AND app_name=?`)
row := ds.db.QueryRowContext(ctx, query, callID, appName)
func (ds *sqlStore) GetLog(ctx context.Context, appID, callID string) (io.Reader, error) {
query := ds.db.Rebind(`SELECT log FROM logs WHERE id=? AND app_id=?`)
row := ds.db.QueryRowContext(ctx, query, callID, appID)
var log string
err := row.Scan(&log)
@@ -806,7 +831,7 @@ func buildFilterRouteQuery(filter *models.RouteFilter) (string, []interface{}) {
}
}
where("app_name=? ", filter.AppName)
where("app_id=? ", filter.AppID)
where("image=?", filter.Image)
where("path>?", filter.Cursor)
// where("path LIKE ?%", filter.PathPrefix) TODO needs escaping
@@ -857,7 +882,6 @@ func buildFilterAppQuery(filter *models.AppFilter) (string, []interface{}, error
fmt.Fprintf(&b, ` LIMIT ?`)
args = append(args, filter.PerPage)
if len(filter.NameIn) > 0 {
// fmt.Println("about to sqlx.in:", b.String(), args)
return sqlx.In(b.String(), args...)
}
return b.String(), args, nil
@@ -888,7 +912,7 @@ func buildFilterCallQuery(filter *models.CallFilter) (string, []interface{}) {
if !time.Time(filter.FromTime).IsZero() {
where("created_at>", filter.FromTime.String())
}
where("app_name=", filter.AppName)
where("app_id=", filter.AppID)
where("path=", filter.Path)
fmt.Fprintf(&b, ` ORDER BY id DESC`) // TODO assert this is indexed