List Cursor management moved into datastore layer. (#1102)

* Don't try to delete an app that wasn't successfully created in the case of failure

* Allow datastore implementations to inject additional annotations on objects

* Allow for datastores transparently adding annotations on apps, fns and triggers. Change NameIn filter to Name for apps.

* Move *List types including JSON annotations for App, Fn and Trigger into models

* Change return types for GetApps, GetFns and GetTriggers on datastore to
be models.*List and ove cursor generation into datastore

* Trigger cursor handling fixed into db layer

Also changes the name generation so that it is not in the same order
as the id (well is random), this means we are now testing our name ordering.

* GetFns now respects cursors

* Apps now feeds cursor back

* Mock fixes

* Fixing up api level cursor decoding

* Tidy up treatment of cursors in the db layer

* Adding conditions for non nil items lists

* fix mock test
This commit is contained in:
Tom Coupland
2018-06-29 19:14:13 +01:00
committed by Owen Cliffe
parent fca107c815
commit d7139358ce
20 changed files with 522 additions and 240 deletions

View File

@@ -9,6 +9,8 @@ import (
"context"
"fmt"
"log"
"math/rand"
"sort"
"sync/atomic"
"testing"
"time"
@@ -50,7 +52,7 @@ type ResourceProvider interface {
// BasicResourceProvider supplies simple objects and can be used as a base for custom resource providers
type BasicResourceProvider struct {
idCount int32
idCount uint32
}
// DataStoreFunc provides an instance of a data store
@@ -60,8 +62,8 @@ func NewBasicResourceProvider() ResourceProvider {
return &BasicResourceProvider{}
}
func (brp *BasicResourceProvider) NextID() int32 {
return atomic.AddInt32(&brp.idCount, 1)
func (brp *BasicResourceProvider) NextID() uint32 {
return atomic.AddUint32(&brp.idCount, rand.Uint32())
}
func (brp *BasicResourceProvider) DefaultCtx() context.Context {
@@ -173,6 +175,7 @@ func (h *Harness) GivenTriggerInDb(validTrigger *models.Trigger) *models.Trigger
return trigger
}
func (h *Harness) AppForDeletion(app *models.App) {
h.appIds = append(h.appIds, app.ID)
}
@@ -185,6 +188,30 @@ func NewHarness(t *testing.T, ctx context.Context, ds models.Datastore) *Harness
}
}
type AppByName []*models.App
func (a AppByName) Len() int { return len(a) }
func (a AppByName) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a AppByName) Less(i, j int) bool { return a[i].Name < a[j].Name }
type FnByName []*models.Fn
func (f FnByName) Len() int { return len(f) }
func (f FnByName) Swap(i, j int) { f[i], f[j] = f[j], f[i] }
func (f FnByName) Less(i, j int) bool { return f[i].Name < f[j].Name }
type TriggerByName []*models.Trigger
func (f TriggerByName) Len() int { return len(f) }
func (f TriggerByName) Swap(i, j int) { f[i], f[j] = f[j], f[i] }
func (f TriggerByName) Less(i, j int) bool { return f[i].Name < f[j].Name }
type RouteByPath []*models.Route
func (f RouteByPath) Len() int { return len(f) }
func (f RouteByPath) Swap(i, j int) { f[i], f[j] = f[j], f[i] }
func (f RouteByPath) Less(i, j int) bool { return f[i].Path < f[j].Path }
func RunAppsTest(t *testing.T, dsf DataStoreFunc, rp ResourceProvider) {
ds := dsf(t)
@@ -206,11 +233,10 @@ func RunAppsTest(t *testing.T, dsf DataStoreFunc, rp ResourceProvider) {
start := time.Now()
returnedApp, err := ds.InsertApp(ctx, rp.ValidApp())
h.AppForDeletion(returnedApp)
if err != nil {
t.Fatalf("Expected succcess, got %s", err)
}
h.AppForDeletion(returnedApp)
if !time.Time(returnedApp.CreatedAt).After(start) {
t.Fatalf("expected created to be set %s", returnedApp.CreatedAt)
@@ -274,7 +300,7 @@ func RunAppsTest(t *testing.T, dsf DataStoreFunc, rp ResourceProvider) {
t.Fatalf("error when updating app: %v", err)
}
expected := &models.App{ID: testApp.ID, Name: testApp.Name, Config: map[string]string{"TEST": "1"}}
if !updated.Equals(expected) {
if !expected.EqualsWithAnnotationSubset(updated) {
t.Fatalf("expected updated `%v` but got `%v`", expected, updated)
}
})
@@ -297,7 +323,7 @@ func RunAppsTest(t *testing.T, dsf DataStoreFunc, rp ResourceProvider) {
t.Fatalf("error when updating app: %v", err)
}
expected := &models.App{Name: testApp.Name, ID: testApp.ID, Config: map[string]string{"TEST": "1", "OTHER": "TEST"}}
if !updated.Equals(expected) {
if !expected.EqualsWithAnnotationSubset(updated) {
t.Fatalf("expected updated `%v` but got `%v`", expected, updated)
}
})
@@ -345,7 +371,7 @@ func RunAppsTest(t *testing.T, dsf DataStoreFunc, rp ResourceProvider) {
t.Fatalf("error when updating app: %v", err)
}
expected := &models.App{Name: testApp.Name, ID: testApp.ID, Config: map[string]string{"OTHER": "TEST"}}
if !updated.Equals(expected) {
if !expected.EqualsWithAnnotationSubset(updated) {
t.Fatalf("expected updated `%#v` but got `%#v`", expected, updated)
}
})
@@ -376,22 +402,43 @@ func RunAppsTest(t *testing.T, dsf DataStoreFunc, rp ResourceProvider) {
t.Run("List apps", func(t *testing.T) {
h := NewHarness(t, ctx, ds)
defer h.Cleanup()
testApp := h.GivenAppInDb(rp.ValidApp())
apps, err := ds.GetApps(ctx, &models.AppFilter{PerPage: 100})
if err != nil {
t.Fatalf("not expecting err %s", err)
}
if len(apps.Items) != 0 {
t.Fatalf("expecting 0 results, got %d", len(apps.Items))
}
if apps.Items == nil {
t.Fatalf("response items must not be nil")
}
a1 := h.GivenAppInDb(rp.ValidApp())
h.GivenAppInDb(rp.ValidApp())
// Testing list apps
apps, err := ds.GetApps(ctx, &models.AppFilter{PerPage: 100})
apps, err = ds.GetApps(ctx, &models.AppFilter{PerPage: 100})
if err != nil {
t.Fatalf("unexpected error %v", err)
}
if len(apps) == 0 {
t.Fatal("expected result count to be greater than 0")
if len(apps.Items) != 2 {
t.Fatalf("expected result count to be 2, got %d", len(apps.Items))
}
for _, app := range apps {
if app.Name == testApp.Name {
apps, err = ds.GetApps(ctx, &models.AppFilter{PerPage: 100, Name: a1.Name})
if err != nil {
t.Fatalf("unexpected error %v", err)
}
if len(apps.Items) != 1 {
t.Fatalf("expected result count to be 1, got %d", len(apps.Items))
}
for _, app := range apps.Items {
if app.Name == a1.Name {
return
}
}
t.Fatalf("expected app list to contain app %s, got %#v", testApp.Name, apps)
t.Fatalf("expected app list to contain app %s, got %#v", a1.Name, apps)
})
t.Run("Simple Pagination", func(t *testing.T) {
@@ -402,38 +449,43 @@ func RunAppsTest(t *testing.T, dsf DataStoreFunc, rp ResourceProvider) {
a2 := h.GivenAppInDb(rp.ValidApp())
a3 := h.GivenAppInDb(rp.ValidApp())
gendApps := []*models.App{a1, a2, a3}
sort.Sort(AppByName(gendApps))
apps, err := ds.GetApps(ctx, &models.AppFilter{PerPage: 1})
if err != nil {
t.Fatalf(" error: %s", err)
}
if len(apps) != 1 {
t.Fatalf(" expected result count to be 1 but got %d", len(apps))
} else if apps[0].Name != a1.Name {
t.Fatalf(" expected `app.Name` to be `%s` but it was `%s`", a1.Name, apps[0].Name)
if len(apps.Items) != 1 {
t.Fatalf(" expected result count to be 1 but got %d", len(apps.Items))
} else if apps.Items[0].Name != gendApps[0].Name {
t.Fatalf(" expected `app.Name` to be `%s` but it was `%s`", gendApps[0].Name, apps.Items[0].Name)
}
apps, err = ds.GetApps(ctx, &models.AppFilter{PerPage: 100, Cursor: apps[0].Name})
apps, err = ds.GetApps(ctx, &models.AppFilter{PerPage: 100, Cursor: apps.NextCursor})
if err != nil {
t.Fatalf(" error: %s", err)
}
if len(apps) != 2 {
t.Fatalf(" expected result count to be 2 but got %d", len(apps))
} else if apps[0].Name != a2.Name {
t.Fatalf(" expected `app.Name` to be `%s` but it was `%s`", a2.Name, apps[0].Name)
} else if apps[1].Name != a3.Name {
t.Fatalf(" expected `app.Name` to be `%s` but it was `%s`", a3.Name, apps[1].Name)
if len(apps.Items) != 2 {
t.Fatalf(" expected result count to be 2 but got %d", len(apps.Items))
} else if apps.Items[0].Name != gendApps[1].Name {
t.Fatalf(" expected `app.Name` to be `%s` but it was `%s`", gendApps[1].Name, apps.Items[0].Name)
} else if apps.Items[1].Name != gendApps[2].Name {
t.Fatalf(" expected `app.Name` to be `%s` but it was `%s`", gendApps[2].Name, apps.Items[1].Name)
}
a4 := h.GivenAppInDb(rp.ValidApp())
gendApps = append(gendApps, a4)
sort.Sort(AppByName(gendApps))
apps, err = ds.GetApps(ctx, &models.AppFilter{PerPage: 100})
if err != nil {
t.Fatalf(" error: %s", err)
}
if len(apps) != 4 {
t.Fatalf(" expected result count to be 4 but got %d", len(apps))
} else if apps[3].Name != a4.Name {
t.Fatalf(" expected `app.Name` to be `%s` but it was `%s`", a4.Name, apps[0].Name)
if len(apps.Items) != 4 {
t.Fatalf(" expected result count to be 4 but got %d", len(apps.Items))
} else if apps.Items[3].Name != gendApps[3].Name {
t.Fatalf(" expected `app.Name` to be `%s` but it was `%s`", gendApps[4].Name, apps.Items[0].Name)
}
})
@@ -714,7 +766,7 @@ func RunRoutesTest(t *testing.T, dsf DataStoreFunc, rp ResourceProvider) {
}
})
t.Run("pagination on routes return rotues in order ", func(t *testing.T) {
t.Run("pagination on routes return routes in order ", func(t *testing.T) {
h := NewHarness(t, ctx, ds)
defer h.Cleanup()
testApp := h.GivenAppInDb(rp.ValidApp())
@@ -723,14 +775,17 @@ func RunRoutesTest(t *testing.T, dsf DataStoreFunc, rp ResourceProvider) {
r2 := h.GivenRouteInDb(rp.ValidRoute(testApp.ID))
r3 := h.GivenRouteInDb(rp.ValidRoute(testApp.ID))
gendRoutes := []*models.Route{r1, r2, r3}
sort.Sort(RouteByPath(gendRoutes))
routes, err := ds.GetRoutesByApp(ctx, testApp.ID, &models.RouteFilter{PerPage: 1})
if err != nil {
t.Fatalf("error: %s", err)
}
if len(routes) != 1 {
t.Fatalf("expected result count to be 1 but got %d", len(routes))
} else if routes[0].Path != r1.Path {
t.Fatalf("expected `route.Path` to be `%s` but it was `%s`", r1.Path, routes[0].Path)
} else if routes[0].Path != gendRoutes[0].Path {
t.Fatalf("expected `route.Path` to be `%s` but it was `%s`", gendRoutes[0].Path, routes[0].Path)
}
routes, err = ds.GetRoutesByApp(ctx, testApp.ID, &models.RouteFilter{PerPage: 2, Cursor: routes[0].Path})
@@ -740,10 +795,10 @@ func RunRoutesTest(t *testing.T, dsf DataStoreFunc, rp ResourceProvider) {
if len(routes) != 2 {
t.Fatalf("expected result count to be 2 but got %d", len(routes))
} else if routes[0].Path != r2.Path {
t.Fatalf("expected `route.Path` to be `%s` but it was `%s`", r2.Path, routes[0].Path)
} else if routes[1].Path != r3.Path {
t.Fatalf("expected `route.Path` to be `%s` but it was `%s`", r3.Path, routes[1].Path)
} else if routes[0].Path != gendRoutes[1].Path {
t.Fatalf("expected `route.Path` to be `%s` but it was `%s`", gendRoutes[1].Path, routes[0].Path)
} else if routes[1].Path != gendRoutes[2].Path {
t.Fatalf("expected `route.Path` to be `%s` but it was `%s`", gendRoutes[2].Path, routes[1].Path)
}
r4 := rp.ValidRoute(testApp.ID)
@@ -882,7 +937,7 @@ func RunFnsTest(t *testing.T, dsf DataStoreFunc, rp ResourceProvider) {
if err != nil {
t.Fatalf("unexpected error %v : %s", err, testFn.ID)
}
if !fn.Equals(testFn) {
if !testFn.EqualsWithAnnotationSubset(fn) {
t.Fatalf("expected to get the right func:\n%v\nbut got:\n%v", testFn, fn)
}
})
@@ -927,7 +982,7 @@ func RunFnsTest(t *testing.T, dsf DataStoreFunc, rp ResourceProvider) {
"THIRD": "3",
},
}
if !updated.Equals(expected) {
if !expected.EqualsWithAnnotationSubset(updated) {
t.Fatalf("expected updated `%#v` but got `%#v`", expected, updated)
}
@@ -979,7 +1034,7 @@ func RunFnsTest(t *testing.T, dsf DataStoreFunc, rp ResourceProvider) {
"THIRD": "3",
},
}
if !updated.Equals(expected) {
if !expected.EqualsWithAnnotationSubset(updated) {
t.Fatalf("expected updated:\n`%v`\nbut got:\n`%v`", expected, updated)
}
})
@@ -993,9 +1048,12 @@ func RunFnsTest(t *testing.T, dsf DataStoreFunc, rp ResourceProvider) {
if err != nil {
t.Fatalf("unexpected error %v", err)
}
if len(fns) != 0 {
if len(fns.Items) != 0 {
t.Fatal("expected result count to be 0")
}
if fns.Items == nil {
t.Fatal("response items must not be nil")
}
})
t.Run("basic pagination with funcs", func(t *testing.T) {
@@ -1006,36 +1064,39 @@ func RunFnsTest(t *testing.T, dsf DataStoreFunc, rp ResourceProvider) {
f2 := h.GivenFnInDb(rp.ValidFn(testApp.ID))
f3 := h.GivenFnInDb(rp.ValidFn(testApp.ID))
gendFns := []*models.Fn{f1, f2, f3}
sort.Sort(FnByName(gendFns))
// Testing list fns
fns, err := ds.GetFns(ctx, &models.FnFilter{AppID: testApp.ID})
if err != nil {
t.Fatalf("unexpected error %v", err)
}
if len(fns) != 3 {
t.Fatalf("expected result count to be 3, but was %d", len(fns))
if len(fns.Items) != 3 {
t.Fatalf("expected result count to be 3, but was %d", len(fns.Items))
}
fns, err = ds.GetFns(ctx, &models.FnFilter{AppID: testApp.ID, PerPage: 1})
if err != nil {
t.Fatalf("unexpected error %v", err)
}
if len(fns) != 1 {
t.Fatalf("expected result count to be 1, but was %d", len(fns))
if len(fns.Items) != 1 {
t.Fatalf("expected result count to be 1, but was %d", len(fns.Items))
}
if !f1.Equals(fns[0]) {
t.Fatalf("Expecting function to be %#v but was %#v", f1, fns[0])
if !gendFns[0].EqualsWithAnnotationSubset(fns.Items[0]) {
t.Fatalf("Expecting function to be %#v but was %#v", gendFns[0], fns.Items[0])
}
fns, err = ds.GetFns(ctx, &models.FnFilter{AppID: testApp.ID, PerPage: 2, Cursor: fns[0].Name})
fns, err = ds.GetFns(ctx, &models.FnFilter{AppID: testApp.ID, PerPage: 2, Cursor: fns.NextCursor})
if err != nil {
t.Fatalf("error: %s", err)
}
if len(fns) != 2 {
t.Fatalf("expected result count to be 2 but got %d", len(fns))
} else if !fns[0].Equals(f2) {
t.Fatalf("expected `func.Name` to be `%#v` but it was `%#v`", f2, fns[0])
} else if !fns[1].Equals(f3) {
t.Fatalf("expected `func.Name` to be `%#v` but it was `%#v`", f3, fns[1])
if len(fns.Items) != 2 {
t.Fatalf("expected result count to be 2 but got %d", len(fns.Items))
} else if !gendFns[1].EqualsWithAnnotationSubset(fns.Items[0]) {
t.Fatalf("expected `func.Name` to be `%#v` but it was `%#v`", gendFns[1].Name, fns.Items[0].Name)
} else if !gendFns[2].EqualsWithAnnotationSubset(fns.Items[1]) {
t.Fatalf("expected `func.Name` to be `%#v` but it was `%#v`", gendFns[2], fns.Items[1])
}
})
@@ -1105,7 +1166,7 @@ func RunTriggersTest(t *testing.T, dsf DataStoreFunc, rp ResourceProvider) {
t.Fatalf("error when storing new trigger: %s", err)
}
newTrigger.ID = insertedTrigger.ID
if !insertedTrigger.Equals(newTrigger) {
if !newTrigger.EqualsWithAnnotationSubset(insertedTrigger) {
t.Errorf("Expecting returned trigger %#v to equal %#v", insertedTrigger, newTrigger)
}
@@ -1131,7 +1192,7 @@ func RunTriggersTest(t *testing.T, dsf DataStoreFunc, rp ResourceProvider) {
t.Fatalf("No ID ")
}
newTrigger.ID = insertedTrigger.ID
if !insertedTrigger.Equals(newTrigger) {
if !newTrigger.EqualsWithAnnotationSubset(insertedTrigger) {
t.Errorf("Expecting returned trigger %#v to equal %#v", insertedTrigger, newTrigger)
}
})
@@ -1157,7 +1218,7 @@ func RunTriggersTest(t *testing.T, dsf DataStoreFunc, rp ResourceProvider) {
}
newTrigger.ID = insertedTrigger.ID
if !gotTrigger.Equals(newTrigger) {
if !newTrigger.EqualsWithAnnotationSubset(gotTrigger) {
t.Errorf("Expecting returned trigger %#v to equal %#v", gotTrigger, newTrigger)
}
})
@@ -1173,8 +1234,8 @@ func RunTriggersTest(t *testing.T, dsf DataStoreFunc, rp ResourceProvider) {
t.Run("non-existant app", func(t *testing.T) {
nonMatchingFilter := &models.TriggerFilter{AppID: "notexist"}
triggers, err := ds.GetTriggers(ctx, nonMatchingFilter)
if len(triggers) != 0 && err == nil {
t.Fatalf("expected empty trigger list and no error, but got list [%v] and err %s", triggers, err)
if len(triggers.Items) != 0 && err == nil {
t.Fatalf("expected empty trigger list and no error, but got list [%v] and err %s", triggers.Items, err)
}
})
@@ -1193,6 +1254,116 @@ func RunTriggersTest(t *testing.T, dsf DataStoreFunc, rp ResourceProvider) {
}
})
t.Run("page triggers", func(t *testing.T) {
h := NewHarness(t, ctx, ds)
defer h.Cleanup()
testApp := h.GivenAppInDb(rp.ValidApp())
testFn := h.GivenFnInDb(rp.ValidFn(testApp.ID))
var storedTriggers []*models.Trigger
for i := 0; i < 10; i++ {
trigger := rp.ValidTrigger(testApp.ID, testFn.ID)
trigger.Source = fmt.Sprintf("src_%v", i)
storedTriggers = append(storedTriggers, h.GivenTriggerInDb(trigger))
}
sort.Sort(TriggerByName(storedTriggers))
appIDFilter := &models.TriggerFilter{AppID: testApp.ID}
triggers, err := ds.GetTriggers(ctx, appIDFilter)
if err != nil {
t.Fatalf("Test GetTriggers(page triggers), not expecting err %s", err)
}
if len(triggers.Items) != 10 {
t.Fatalf("Test GetTriggers(page triggers), expecting 10 results, got %d", len(triggers.Items))
}
for i := 1; i < 10; i++ {
if triggers.Items[i-1].Name > triggers.Items[i].Name {
t.Fatalf("Test GetTriggers(page triggers), names out of order, %s, %s", triggers.Items[i-1], triggers.Items[i])
}
}
fiveFilter := &models.TriggerFilter{AppID: testApp.ID, PerPage: 5}
triggers, err = ds.GetTriggers(ctx, fiveFilter)
if err != nil {
t.Fatalf("Test GetTriggers(page triggers), not expecting err %s", err)
}
if len(triggers.Items) != 5 {
t.Fatalf("Test GetTriggers(page triggers), expecting 5 results, got %d", len(triggers.Items))
}
for i := 0; i < 5; i++ {
if !triggers.Items[i].EqualsWithAnnotationSubset(storedTriggers[i]) {
t.Fatalf("Test GetTriggers(first five page triggers), expect equal, %s, %s", triggers.Items[i], storedTriggers[i])
}
}
if triggers.NextCursor == "" {
t.Fatalf("Test GetTriggers(first five page triggers), expected Cursor but got nothing")
}
secondFiveFilter := &models.TriggerFilter{AppID: testApp.ID, PerPage: 5, Cursor: triggers.NextCursor}
triggers, err = ds.GetTriggers(ctx, secondFiveFilter)
if err != nil {
t.Fatalf("Test GetTriggers(second five page triggers), not expecting err %s", err)
}
if len(triggers.Items) != 5 {
t.Fatalf("Test GetTriggers(second five page triggers), expecting 5 results, got %d", len(triggers.Items))
}
for i := 0; i < 5; i++ {
if !triggers.Items[i].EqualsWithAnnotationSubset(storedTriggers[i+5]) {
t.Fatalf("Test GetTriggers(second five page triggers), expect equal, %s, %s", triggers.Items[i], storedTriggers[i+5])
}
}
zeroFilter := &models.TriggerFilter{AppID: testApp.ID, PerPage: 0}
triggers, err = ds.GetTriggers(ctx, zeroFilter)
if err != nil {
t.Fatalf("Test GetTriggers(zero page triggers), not expecting err %s", err)
}
if len(triggers.Items) != 10 {
t.Fatalf("Test GetTriggers(zero page triggers), expecting 10 results, got %d", len(triggers.Items))
}
if triggers.NextCursor != "" {
t.Fatalf("Test GetTriggers(zero page triggers), expected no NextCursor, got %s", triggers.NextCursor)
}
negativeFilter := &models.TriggerFilter{AppID: testApp.ID, PerPage: -10}
triggers, err = ds.GetTriggers(ctx, negativeFilter)
if err != nil {
t.Fatalf("Test GetTriggers(negative page triggers), not expecting err %s", err)
}
if len(triggers.Items) != 10 {
t.Fatalf("Test GetTriggers(negative page triggers), expecting 10 results, got %d", len(triggers.Items))
}
if triggers.NextCursor != "" {
t.Fatalf("Test GetTriggers(negative page triggers), expected no NextCursor, got %s", triggers.NextCursor)
}
emptyListFilter := &models.TriggerFilter{AppID: "notexist"}
triggers, err = ds.GetTriggers(ctx, emptyListFilter)
if err != nil {
t.Fatalf("Test GetTriggers(notexist page triggers), not expecting err %s", err)
}
if len(triggers.Items) != 0 {
t.Fatalf("Test GetTriggers(notexist page triggers), expecting 0 results, got %d", len(triggers.Items))
}
if triggers.Items == nil {
t.Fatalf("Test GetTriggers(notexist page triggers), response items must not be nil")
}
})
t.Run("filter triggers", func(t *testing.T) {
h := NewHarness(t, ctx, ds)
defer h.Cleanup()
@@ -1210,7 +1381,10 @@ func RunTriggersTest(t *testing.T, dsf DataStoreFunc, rp ResourceProvider) {
trigger := rp.ValidTrigger(testApp.ID, testFn2.ID)
trigger.Source = fmt.Sprintf("src_%v", 11)
h.GivenTriggerInDb(trigger)
trigger = h.GivenTriggerInDb(trigger)
storedTriggers = append(storedTriggers, trigger)
sort.Sort(TriggerByName(storedTriggers))
appIDFilter := &models.TriggerFilter{AppID: testApp.ID}
triggers, err := ds.GetTriggers(ctx, appIDFilter)
@@ -1218,14 +1392,13 @@ func RunTriggersTest(t *testing.T, dsf DataStoreFunc, rp ResourceProvider) {
t.Fatalf("Test GetTriggers(get all triggers for app), not expecting err %s", err)
}
if len(triggers) != 11 {
t.Fatalf("Test GetTriggers(get all triggers for app), expecting 10 results, got %d", len(triggers))
if len(triggers.Items) != 11 {
t.Fatalf("Test GetTriggers(get all triggers for app), expecting 10 results, got %d", len(triggers.Items))
}
for i := 1; i < 10; i++ {
if !storedTriggers[i].Equals(triggers[i]) {
t.Fatalf("expecting ordered by names, but aren't: %s, %s", storedTriggers[i].Name, triggers[i].Name)
for i := 0; i < 11; i++ {
if !storedTriggers[i].EqualsWithAnnotationSubset(triggers.Items[i]) {
t.Fatalf("Test GetTriggers(get all triggers for app), expecting ordered by names, but aren't: %+v, %+v", storedTriggers[i], triggers.Items[i])
}
}
@@ -1235,41 +1408,12 @@ func RunTriggersTest(t *testing.T, dsf DataStoreFunc, rp ResourceProvider) {
t.Fatalf("Test GetTriggers(filter by name), not expecting err %s", err)
}
if len(triggers) != 1 {
t.Fatalf("Test GetTriggers(filter by name), expecting 1 results, got %d", len(triggers))
if len(triggers.Items) != 1 {
t.Fatalf("Test GetTriggers(filter by name), expecting 1 results, got %d", len(triggers.Items))
}
if !triggers[0].Equals(storedTriggers[0]) {
t.Fatalf("expect single result to equal first stored result : %#v != %#v", triggers[4], storedTriggers[4])
}
appIDPagedFilter := &models.TriggerFilter{AppID: testApp.ID, PerPage: 5}
triggers, err = ds.GetTriggers(ctx, appIDPagedFilter)
if err != nil {
t.Fatalf("Test GetTriggers(page triggers for app), not expecting err %s", err)
}
if len(triggers) != 5 {
t.Fatalf("Test GetTriggers(get all triggers for app), expecting 5 results, got %d", len(triggers))
}
if !triggers[4].Equals(storedTriggers[4]) {
t.Fatalf("expect 5th result to equal 5th stored result : %#v != %#v", triggers[4], storedTriggers[4])
}
appIDPagedFilter.Cursor = triggers[4].ID
triggers, err = ds.GetTriggers(ctx, appIDPagedFilter)
if err != nil {
t.Fatalf("Test GetTriggers(page triggers for app), not expecting err %s", err)
}
if len(triggers) != 5 {
t.Fatalf("Test GetTriggers(get all triggers for app), expecting 5 results, got %d", len(triggers))
}
if !triggers[4].Equals(storedTriggers[9]) {
t.Fatalf("expect 5th result to equal 9th stored result : %#v != %#v", triggers[4], storedTriggers[9])
if !storedTriggers[0].EqualsWithAnnotationSubset(triggers.Items[0]) {
t.Fatalf("expect single result to equal first stored result : %#v != %#v", triggers.Items[4], storedTriggers[4])
}
// components are AND'd
@@ -1278,8 +1422,8 @@ func RunTriggersTest(t *testing.T, dsf DataStoreFunc, rp ResourceProvider) {
if err != nil {
t.Fatalf("Test GetTriggers(AND filtering), not expecting err %s", err)
}
if len(triggers) != 10 {
t.Fatalf("Test GetTriggers(AND filtering), expecting 10 results, got %d", len(triggers))
if len(triggers.Items) != 10 {
t.Fatalf("Test GetTriggers(AND filtering), expecting 10 results, got %d", len(triggers.Items))
}
})
@@ -1300,7 +1444,7 @@ func RunTriggersTest(t *testing.T, dsf DataStoreFunc, rp ResourceProvider) {
t.Fatalf("error when updating trigger: %s", err)
}
if !gotTrigger.Equals(testTrigger) {
if !testTrigger.EqualsWithAnnotationSubset(gotTrigger) {
t.Fatalf("expecting returned triggers equal, got : %#v : %#v", testTrigger, gotTrigger)
}
@@ -1308,7 +1452,7 @@ func RunTriggersTest(t *testing.T, dsf DataStoreFunc, rp ResourceProvider) {
if err != nil {
t.Fatalf("wasn't expecting an error : %s", err)
}
if !gotTrigger.Equals(testTrigger) {
if !testTrigger.EqualsWithAnnotationSubset(gotTrigger) {
t.Fatalf("expecting fetch trigger to be updated got : %v : %v", testTrigger, gotTrigger)
}

View File

@@ -28,7 +28,7 @@ func (m *metricds) GetAppByID(ctx context.Context, appID string) (*models.App, e
return m.ds.GetAppByID(ctx, appID)
}
func (m *metricds) GetApps(ctx context.Context, filter *models.AppFilter) ([]*models.App, error) {
func (m *metricds) GetApps(ctx context.Context, filter *models.AppFilter) (*models.AppList, error) {
ctx, span := trace.StartSpan(ctx, "ds_get_apps")
defer span.End()
return m.ds.GetApps(ctx, filter)
@@ -107,7 +107,7 @@ func (m *metricds) GetTriggerByID(ctx context.Context, triggerID string) (*model
return m.ds.GetTriggerByID(ctx, triggerID)
}
func (m *metricds) GetTriggers(ctx context.Context, filter *models.TriggerFilter) ([]*models.Trigger, error) {
func (m *metricds) GetTriggers(ctx context.Context, filter *models.TriggerFilter) (*models.TriggerList, error) {
ctx, span := trace.StartSpan(ctx, "ds_get_triggers")
defer span.End()
return m.ds.GetTriggers(ctx, filter)
@@ -125,7 +125,7 @@ func (m *metricds) UpdateFn(ctx context.Context, fn *models.Fn) (*models.Fn, err
return m.ds.UpdateFn(ctx, fn)
}
func (m *metricds) GetFns(ctx context.Context, filter *models.FnFilter) ([]*models.Fn, error) {
func (m *metricds) GetFns(ctx context.Context, filter *models.FnFilter) (*models.FnList, error) {
ctx, span := trace.StartSpan(ctx, "ds_get_funcs")
defer span.End()
return m.ds.GetFns(ctx, filter)

View File

@@ -31,7 +31,7 @@ func (v *validator) GetAppByID(ctx context.Context, appID string) (*models.App,
return v.Datastore.GetAppByID(ctx, appID)
}
func (v *validator) GetApps(ctx context.Context, appFilter *models.AppFilter) ([]*models.App, error) {
func (v *validator) GetApps(ctx context.Context, appFilter *models.AppFilter) (*models.AppList, error) {
return v.Datastore.GetApps(ctx, appFilter)
}
@@ -151,7 +151,7 @@ func (v *validator) UpdateTrigger(ctx context.Context, trigger *models.Trigger)
return v.Datastore.UpdateTrigger(ctx, trigger)
}
func (v *validator) GetTriggers(ctx context.Context, filter *models.TriggerFilter) ([]*models.Trigger, error) {
func (v *validator) GetTriggers(ctx context.Context, filter *models.TriggerFilter) (*models.TriggerList, error) {
if filter.AppID == "" {
return nil, models.ErrTriggerMissingAppID
@@ -195,7 +195,7 @@ func (v *validator) GetFnByID(ctx context.Context, fnID string) (*models.Fn, err
return v.Datastore.GetFnByID(ctx, fnID)
}
func (v *validator) GetFns(ctx context.Context, filter *models.FnFilter) ([]*models.Fn, error) {
func (v *validator) GetFns(ctx context.Context, filter *models.FnFilter) (*models.FnList, error) {
if filter.AppID == "" {
return nil, models.ErrFnsMissingAppID

View File

@@ -2,9 +2,9 @@ package datastore
import (
"context"
"encoding/base64"
"sort"
"strings"
"time"
"github.com/fnproject/fn/api/common"
@@ -12,6 +12,7 @@ import (
"github.com/fnproject/fn/api/id"
"github.com/fnproject/fn/api/logs"
"github.com/fnproject/fn/api/models"
"github.com/sirupsen/logrus"
)
type mock struct {
@@ -75,33 +76,43 @@ func (s sortA) Len() int { return len(s) }
func (s sortA) Less(i, j int) bool { return strings.Compare(s[i].Name, s[j].Name) < 0 }
func (s sortA) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
func (m *mock) GetApps(ctx context.Context, appFilter *models.AppFilter) ([]*models.App, error) {
func (m *mock) GetApps(ctx context.Context, filter *models.AppFilter) (*models.AppList, error) {
// sort them all first for cursoring (this is for testing, n is small & mock is not concurrent..)
sort.Sort(sortA(m.Apps))
var apps []*models.App
var cursor string
if filter.Cursor != "" {
s, err := base64.RawURLEncoding.DecodeString(filter.Cursor)
if err != nil {
return nil, err
}
logrus.Error(s)
cursor = string(s)
}
apps := []*models.App{}
for _, a := range m.Apps {
if len(apps) == appFilter.PerPage {
if len(apps) == filter.PerPage {
break
}
if len(appFilter.NameIn) > 0 {
var found bool
for _, fn := range appFilter.NameIn {
if fn == a.Name {
found = true
break
}
}
if !found {
if strings.Compare(cursor, a.Name) < 0 {
if filter.Name != "" && filter.Name != a.Name {
continue
}
}
if strings.Compare(appFilter.Cursor, a.Name) < 0 {
apps = append(apps, a.Clone())
}
}
return apps, nil
var nextCursor string
if len(apps) > 0 && len(apps) == filter.PerPage {
last := []byte(apps[len(apps)-1].Name)
nextCursor = base64.RawURLEncoding.EncodeToString(last)
}
return &models.AppList{
NextCursor: nextCursor,
Items: apps,
}, nil
}
func (m *mock) InsertApp(ctx context.Context, newApp *models.App) (*models.App, error) {
@@ -315,25 +326,44 @@ func (s sortF) Len() int { return len(s) }
func (s sortF) Less(i, j int) bool { return strings.Compare(s[i].Name, s[j].Name) < 0 }
func (s sortF) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
func (m *mock) GetFns(ctx context.Context, filter *models.FnFilter) ([]*models.Fn, error) {
func (m *mock) GetFns(ctx context.Context, filter *models.FnFilter) (*models.FnList, error) {
// sort them all first for cursoring (this is for testing, n is small & mock is not concurrent..)
sort.Sort(sortF(m.Fns))
funcs := []*models.Fn{}
var cursor string
if filter.Cursor != "" {
s, err := base64.RawURLEncoding.DecodeString(filter.Cursor)
if err != nil {
return nil, err
}
cursor = string(s)
}
for _, f := range m.Fns {
if filter.PerPage > 0 && len(funcs) == filter.PerPage {
break
}
if strings.Compare(filter.Cursor, f.Name) < 0 &&
if strings.Compare(cursor, f.Name) < 0 &&
(filter.AppID == "" || filter.AppID == f.AppID) &&
(filter.Name == "" || filter.Name == f.Name) {
funcs = append(funcs, f)
}
}
return funcs, nil
var nextCursor string
if len(funcs) > 0 && len(funcs) == filter.PerPage {
last := []byte(funcs[len(funcs)-1].Name)
nextCursor = base64.RawURLEncoding.EncodeToString(last)
}
return &models.FnList{
NextCursor: nextCursor,
Items: funcs,
}, nil
}
func (m *mock) GetFnByID(ctx context.Context, fnID string) (*models.Fn, error) {
@@ -438,12 +468,21 @@ func (m *mock) GetTriggerByID(ctx context.Context, triggerId string) (*models.Tr
type sortT []*models.Trigger
func (s sortT) Len() int { return len(s) }
func (s sortT) Less(i, j int) bool { return strings.Compare(s[i].ID, s[j].ID) < 0 }
func (s sortT) Less(i, j int) bool { return strings.Compare(s[i].Name, s[j].Name) < 0 }
func (s sortT) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
func (m *mock) GetTriggers(ctx context.Context, filter *models.TriggerFilter) ([]*models.Trigger, error) {
func (m *mock) GetTriggers(ctx context.Context, filter *models.TriggerFilter) (*models.TriggerList, error) {
sort.Sort(sortT(m.Triggers))
var cursor string
if filter.Cursor != "" {
s, err := base64.RawURLEncoding.DecodeString(filter.Cursor)
if err != nil {
return nil, err
}
cursor = string(s)
}
res := []*models.Trigger{}
for _, t := range m.Triggers {
if filter.PerPage > 0 && len(res) == filter.PerPage {
@@ -451,7 +490,7 @@ func (m *mock) GetTriggers(ctx context.Context, filter *models.TriggerFilter) ([
}
matched := true
if filter.Cursor != "" && t.ID <= filter.Cursor {
if filter.Cursor != "" && t.Name <= cursor {
matched = false
}
@@ -470,7 +509,17 @@ func (m *mock) GetTriggers(ctx context.Context, filter *models.TriggerFilter) ([
res = append(res, t)
}
}
return res, nil
var nextCursor string
if len(res) > 0 && len(res) == filter.PerPage {
last := []byte(res[len(res)-1].Name)
nextCursor = base64.RawURLEncoding.EncodeToString(last)
}
return &models.TriggerList{
NextCursor: nextCursor,
Items: res,
}, nil
}
func (m *mock) RemoveTrigger(ctx context.Context, triggerID string) error {

View File

@@ -4,6 +4,7 @@ import (
"bytes"
"context"
"database/sql"
"encoding/base64"
"fmt"
"io"
"net/url"
@@ -508,12 +509,8 @@ func (ds *SQLStore) GetAppByID(ctx context.Context, appID string) (*models.App,
}
// GetApps retrieves an array of apps according to a specific filter.
func (ds *SQLStore) GetApps(ctx context.Context, filter *models.AppFilter) ([]*models.App, error) {
res := []*models.App{} // for JSON empty list
if filter.NameIn != nil && len(filter.NameIn) == 0 { // this basically makes sure it doesn't return ALL apps
return res, nil
}
func (ds *SQLStore) GetApps(ctx context.Context, filter *models.AppFilter) (*models.AppList, error) {
res := &models.AppList{Items: []*models.App{}}
query, args, err := buildFilterAppQuery(filter)
if err != nil {
@@ -535,7 +532,12 @@ func (ds *SQLStore) GetApps(ctx context.Context, filter *models.AppFilter) ([]*m
}
return res, err
}
res = append(res, &app)
res.Items = append(res.Items, &app)
}
if len(res.Items) > 0 && len(res.Items) == filter.PerPage {
last := []byte(res.Items[len(res.Items)-1].Name)
res.NextCursor = base64.RawURLEncoding.EncodeToString(last)
}
if err := rows.Err(); err != nil {
@@ -844,13 +846,16 @@ func (ds *SQLStore) UpdateFn(ctx context.Context, fn *models.Fn) (*models.Fn, er
return fn, nil
}
func (ds *SQLStore) GetFns(ctx context.Context, filter *models.FnFilter) ([]*models.Fn, error) {
res := []*models.Fn{} // for JSON empty list
func (ds *SQLStore) GetFns(ctx context.Context, filter *models.FnFilter) (*models.FnList, error) {
res := &models.FnList{Items: []*models.Fn{}}
if filter == nil {
filter = new(models.FnFilter)
}
filterQuery, args := buildFilterFnQuery(filter)
filterQuery, args, err := buildFilterFnQuery(filter)
if err != nil {
return res, err
}
query := fmt.Sprintf("%s %s", fnSelector, filterQuery)
query = ds.db.Rebind(query)
@@ -869,14 +874,19 @@ func (ds *SQLStore) GetFns(ctx context.Context, filter *models.FnFilter) ([]*mod
if err != nil {
continue
}
res = append(res, &fn)
res.Items = append(res.Items, &fn)
}
if len(res.Items) > 0 && len(res.Items) == filter.PerPage {
last := []byte(res.Items[len(res.Items)-1].Name)
res.NextCursor = base64.RawURLEncoding.EncodeToString(last)
}
if err := rows.Err(); err != nil {
if err == sql.ErrNoRows {
return res, nil // no error for empty list
}
}
return res, nil
}
@@ -1066,16 +1076,20 @@ func buildFilterAppQuery(filter *models.AppFilter) (string, []interface{}, error
var b bytes.Buffer
// where("name LIKE ?%", filter.Name) // TODO needs escaping?
args = where(&b, args, "name>?", filter.Cursor)
args = where(&b, args, "name IN (?)", filter.NameIn)
if filter.Cursor != "" {
s, err := base64.RawURLEncoding.DecodeString(filter.Cursor)
if err != nil {
return "", args, err
}
args = where(&b, args, "name>?", string(s))
}
if filter.Name != "" {
args = where(&b, args, "name=?", filter.Name)
}
fmt.Fprintf(&b, ` ORDER BY name ASC`) // TODO assert this is indexed
fmt.Fprintf(&b, ` LIMIT ?`)
args = append(args, filter.PerPage)
if len(filter.NameIn) > 0 {
return sqlx.In(b.String(), args...)
}
return b.String(), args, nil
}
@@ -1103,23 +1117,30 @@ func buildFilterCallQuery(filter *models.CallFilter) (string, []interface{}) {
return b.String(), args
}
func buildFilterFnQuery(filter *models.FnFilter) (string, []interface{}) {
func buildFilterFnQuery(filter *models.FnFilter) (string, []interface{}, error) {
if filter == nil {
return "", nil
return "", nil, nil
}
var b bytes.Buffer
var args []interface{}
// where(fmt.Sprintf("image LIKE '%s%%'"), filter.Image) // TODO needs escaping, prob we want prefix query to ignore tags
args = where(&b, args, "app_id=? ", filter.AppID)
args = where(&b, args, "name>?", filter.Cursor)
if filter.Cursor != "" {
s, err := base64.RawURLEncoding.DecodeString(filter.Cursor)
if err != nil {
return "", args, err
}
args = where(&b, args, "name>?", string(s))
}
fmt.Fprintf(&b, ` ORDER BY name ASC`)
if filter.PerPage > 0 {
fmt.Fprintf(&b, ` LIMIT ?`)
args = append(args, filter.PerPage)
}
return b.String(), args
return b.String(), args, nil
}
func where(b *bytes.Buffer, args []interface{}, colOp string, val interface{}) []interface{} {
@@ -1305,7 +1326,7 @@ func (ds *SQLStore) GetTriggerByID(ctx context.Context, triggerID string) (*mode
return &trigger, nil
}
func buildFilterTriggerQuery(filter *models.TriggerFilter) (string, []interface{}) {
func buildFilterTriggerQuery(filter *models.TriggerFilter) (string, []interface{}, error) {
var b bytes.Buffer
var args []interface{}
@@ -1323,29 +1344,35 @@ func buildFilterTriggerQuery(filter *models.TriggerFilter) (string, []interface{
}
if filter.Cursor != "" {
fmt.Fprintf(&b, ` AND id > ?`)
args = append(args, filter.Cursor)
s, err := base64.RawURLEncoding.DecodeString(filter.Cursor)
if err != nil {
return "", nil, err
}
fmt.Fprintf(&b, ` AND name > ?`)
args = append(args, string(s))
}
fmt.Fprintf(&b, ` ORDER BY name ASC`)
if filter.PerPage != 0 {
if filter.PerPage > 0 {
fmt.Fprintf(&b, ` LIMIT ?`)
args = append(args, filter.PerPage)
}
return b.String(), args
return b.String(), args, nil
}
func (ds *SQLStore) GetTriggers(ctx context.Context, filter *models.TriggerFilter) ([]*models.Trigger, error) {
res := []*models.Trigger{} // for JSON empty list
func (ds *SQLStore) GetTriggers(ctx context.Context, filter *models.TriggerFilter) (*models.TriggerList, error) {
res := &models.TriggerList{Items: []*models.Trigger{}}
if filter == nil {
filter = new(models.TriggerFilter)
}
filterQuery, args := buildFilterTriggerQuery(filter)
logrus.Error(filterQuery, args)
filterQuery, args, err := buildFilterTriggerQuery(filter)
if err != nil {
return res, err
}
query := fmt.Sprintf("%s WHERE %s", triggerSelector, filterQuery)
query = ds.db.Rebind(query)
@@ -1364,14 +1391,19 @@ func (ds *SQLStore) GetTriggers(ctx context.Context, filter *models.TriggerFilte
if err != nil {
continue
}
res = append(res, &trigger)
res.Items = append(res.Items, &trigger)
}
if len(res.Items) > 0 && len(res.Items) == filter.PerPage {
last := []byte(res.Items[len(res.Items)-1].Name)
res.NextCursor = base64.RawURLEncoding.EncodeToString(last)
}
if err := rows.Err(); err != nil {
if err == sql.ErrNoRows {
return res, nil // no error for empty list
}
}
return res, nil
}