List Cursor management moved into datastore layer. (#1102)

* Don't try to delete an app that wasn't successfully created in the case of failure

* Allow datastore implementations to inject additional annotations on objects

* Allow for datastores transparently adding annotations on apps, fns and triggers. Change NameIn filter to Name for apps.

* Move *List types including JSON annotations for App, Fn and Trigger into models

* Change return types for GetApps, GetFns and GetTriggers on datastore to
be models.*List and ove cursor generation into datastore

* Trigger cursor handling fixed into db layer

Also changes the name generation so that it is not in the same order
as the id (well is random), this means we are now testing our name ordering.

* GetFns now respects cursors

* Apps now feeds cursor back

* Mock fixes

* Fixing up api level cursor decoding

* Tidy up treatment of cursors in the db layer

* Adding conditions for non nil items lists

* fix mock test

api/models/annotations.go

@@ -32,6 +32,10 @@ func (m Annotations) Equals(other Annotations) bool {
 	if len(m) != len(other) {
 		return false
 	}
+	return m.Subset(other)
+}
+
+func (m Annotations) Subset(other Annotations) bool {
 	for k1, v1 := range m {
 		v2, _ := other[k1]
 		if v2 == nil {

api/models/app.go

@@ -135,6 +135,22 @@ func (a1 *App) Equals(a2 *App) bool {
 	return eq
 }
 
+func (a1 *App) EqualsWithAnnotationSubset(a2 *App) bool {
+	// start off equal, check equivalence of each field.
+	// the RHS of && won't eval if eq==false so config checking is lazy
+
+	eq := true
+	eq = eq && a1.ID == a2.ID
+	eq = eq && a1.Name == a2.Name
+	eq = eq && a1.Config.Equals(a2.Config)
+	eq = eq && a1.Annotations.Subset(a2.Annotations)
+	// NOTE: datastore tests are not very fun to write with timestamp checks,
+	// and these are not values the user may set so we kind of don't care.
+	//eq = eq && time.Time(a1.CreatedAt).Equal(time.Time(a2.CreatedAt))
+	//eq = eq && time.Time(a1.UpdatedAt).Equal(time.Time(a2.UpdatedAt))
+	return eq
+}
+
 // Update adds entries from patch to a.Config and a.Annotations, and removes entries with empty values.
 func (a *App) Update(patch *App) {
 	original := a.Clone()
@@ -176,8 +192,12 @@ func (e ErrInvalidSyslog) Error() string { return string(e) }
 
 // AppFilter is the filter used for querying apps
 type AppFilter struct {
-	// NameIn will filter by all names in the list (IN query)
-	NameIn  []string
+	Name    string
 	PerPage int
 	Cursor  string
 }
+
+type AppList struct {
+	NextCursor string `json:"next_cursor,omitempty"`
+	Items      []*App `json:"items"`
+}

api/models/datastore.go

@@ -15,9 +15,9 @@ type Datastore interface {
 	// Returns ErrAppsNotFound if no app is found.
 	GetAppID(ctx context.Context, appName string) (string, error)
 
-	// GetApps gets a slice of Apps, optionally filtered by name.
+	// GetApps gets a slice of Apps, optionally filtered by name, and a cursor.
 	// Missing filter or empty name will match all Apps.
-	GetApps(ctx context.Context, filter *AppFilter) ([]*App, error)
+	GetApps(ctx context.Context, filter *AppFilter) (*AppList, error)
 
 	// InsertApp inserts an App. Returns ErrDatastoreEmptyApp when app is nil, and
 	// ErrDatastoreEmptyAppName when app.Name is empty.
@@ -63,8 +63,8 @@ type Datastore interface {
 	// ErrMissingName is func.Name is empty.
 	UpdateFn(ctx context.Context, fn *Fn) (*Fn, error)
 
-	// GetFns returns a list of funcs, applying any additional filters provided.
-	GetFns(ctx context.Context, filter *FnFilter) ([]*Fn, error)
+	// GetFns returns a list of funcs, and a cursor, applying any additional filters provided.
+	GetFns(ctx context.Context, filter *FnFilter) (*FnList, error)
 
 	// GetFnByID returns a function by ID. Returns ErrDatastoreEmptyFnID if fnID is empty.
 	// Returns ErrFnsNotFound if a fn is not found.
@@ -91,7 +91,7 @@ type Datastore interface {
 
 	// GetTriggers gets a list of triggers that match the specified filter
 	// Return ErrDatastoreEmptyAppId if no AppID set in the filter
-	GetTriggers(ctx context.Context, filter *TriggerFilter) ([]*Trigger, error)
+	GetTriggers(ctx context.Context, filter *TriggerFilter) (*TriggerList, error)
 
 	// implements io.Closer to shutdown
 	io.Closer

api/models/fn.go

@@ -231,6 +231,28 @@ func (f1 *Fn) Equals(f2 *Fn) bool {
 	return eq
 }
 
+func (f1 *Fn) EqualsWithAnnotationSubset(f2 *Fn) bool {
+	// start off equal, check equivalence of each field.
+	// the RHS of && won't eval if eq==false so config/headers checking is lazy
+
+	eq := true
+	eq = eq && f1.ID == f2.ID
+	eq = eq && f1.Name == f2.Name
+	eq = eq && f1.AppID == f2.AppID
+	eq = eq && f1.Image == f2.Image
+	eq = eq && f1.Memory == f2.Memory
+	eq = eq && f1.Format == f2.Format
+	eq = eq && f1.Timeout == f2.Timeout
+	eq = eq && f1.IdleTimeout == f2.IdleTimeout
+	eq = eq && f1.Config.Equals(f2.Config)
+	eq = eq && f1.Annotations.Subset(f2.Annotations)
+	// NOTE: datastore tests are not very fun to write with timestamp checks,
+	// and these are not values the user may set so we kind of don't care.
+	//eq = eq && time.Time(f1.CreatedAt).Equal(time.Time(f2.CreatedAt))
+	//eq = eq && time.Time(f2.UpdatedAt).Equal(time.Time(f2.UpdatedAt))
+	return eq
+}
+
 // Update updates fields in f with non-zero field values from new, and sets
 // updated_at if any of the fields change. 0-length slice Header values, and
 // empty-string Config values trigger removal of map entry.
@@ -279,3 +301,8 @@ type FnFilter struct {
 	Cursor  string
 	PerPage int
 }
+
+type FnList struct {
+	NextCursor string `json:"next_cursor,omitempty"`
+	Items      []*Fn  `json:"items"`
+}

api/models/trigger.go

@@ -36,6 +36,20 @@ func (t *Trigger) Equals(t2 *Trigger) bool {
 	return eq
 }
 
+func (t *Trigger) EqualsWithAnnotationSubset(t2 *Trigger) bool {
+	eq := true
+	eq = eq && t.ID == t2.ID
+	eq = eq && t.Name == t2.Name
+	eq = eq && t.AppID == t2.AppID
+	eq = eq && t.FnID == t2.FnID
+
+	eq = eq && t.Type == t2.Type
+	eq = eq && t.Source == t2.Source
+	eq = eq && t.Annotations.Subset(t2.Annotations)
+
+	return eq
+}
+
 var triggerTypes = []string{"http"}
 
 func ValidTriggerTypes() []string {
@@ -178,3 +192,8 @@ type TriggerFilter struct {
 	Cursor  string
 	PerPage int
 }
+
+type TriggerList struct {
+	NextCursor string     `json:"next_cursor,omitempty"`
+	Items      []*Trigger `json:"items"`
+}

api/models/trigger_test.go

@@ -26,6 +26,19 @@ func TestTriggerJsonMarshalling(t *testing.T) {
 	}
 }
 
+func TestTriggerListJsonMarshalling(t *testing.T) {
+	emptyList := &TriggerList{Items: []*Trigger{}}
+	expected := "{\"items\":[]}"
+
+	v, err := json.Marshal(emptyList)
+	if err != nil {
+		t.Fatalf("Failed to marshal json into %s: %v", expected, err)
+	}
+	if string(v) != expected {
+		t.Errorf("Invalid trigger value, expected %s, got %s", expected, string(v))
+	}
+}
+
 var httpTrigger = &Trigger{Name: "name", AppID: "foo", FnID: "bar", Type: "http", Source: "baz"}
 var invalidTrigger = &Trigger{Name: "name", AppID: "foo", FnID: "bar", Type: "error", Source: "baz"}