Merge pull request #349 from fnproject/pagination

add pagination to all list endpoints

api/datastore/internal/datastoretest/test.go

@@ -4,15 +4,13 @@ import (
 	"bytes"
 	"context"
 	"log"
+	"net/http"
+	"reflect"
 	"testing"
+	"time"
 
 	"github.com/fnproject/fn/api/id"
 	"github.com/fnproject/fn/api/models"
-
-	"net/http"
-	"reflect"
-	"time"
-
 	"github.com/gin-gonic/gin"
 	"github.com/go-openapi/strfmt"
 	"github.com/sirupsen/logrus"
@@ -28,7 +26,7 @@ func setLogBuffer() *bytes.Buffer {
 	return &buf
 }
 
-func Test(t *testing.T, ds models.Datastore) {
+func Test(t *testing.T, dsf func() models.Datastore) {
 	buf := setLogBuffer()
 
 	ctx := context.Background()
@@ -42,6 +40,7 @@ func Test(t *testing.T, ds models.Datastore) {
 	call.Path = testRoute.Path
 
 	t.Run("call-insert", func(t *testing.T) {
+		ds := dsf()
 		call.ID = id.New().String()
 		err := ds.InsertCall(ctx, call)
 		if err != nil {
@@ -51,6 +50,7 @@ func Test(t *testing.T, ds models.Datastore) {
 	})
 
 	t.Run("call-get", func(t *testing.T) {
+		ds := dsf()
 		call.ID = id.New().String()
 		ds.InsertCall(ctx, call)
 		newCall, err := ds.GetCall(ctx, call.AppName, call.ID)
@@ -64,20 +64,120 @@ func Test(t *testing.T, ds models.Datastore) {
 	})
 
 	t.Run("calls-get", func(t *testing.T) {
-		filter := &models.CallFilter{AppName: call.AppName, Path: call.Path}
+		ds := dsf()
+		filter := &models.CallFilter{AppName: call.AppName, Path: call.Path, PerPage: 100}
 		call.ID = id.New().String()
-		ds.InsertCall(ctx, call)
+		call.CreatedAt = strfmt.DateTime(time.Now())
+		err := ds.InsertCall(ctx, call)
+		if err != nil {
+			t.Fatal(err)
+		}
 		calls, err := ds.GetCalls(ctx, filter)
 		if err != nil {
 			t.Fatalf("Test GetCalls(ctx, filter): unexpected error `%v`", err)
 		}
-		if len(calls) == 0 {
+		if len(calls) != 1 {
 			t.Log(buf.String())
+			t.Fatalf("Test GetCalls(ctx, filter): unexpected length `%v`", len(calls))
+		}
+
+		c2 := *call
+		c3 := *call
+		c2.ID = id.New().String()
+		c2.CreatedAt = strfmt.DateTime(time.Now().Add(100 * time.Millisecond)) // add ms cuz db uses it for sort
+		c3.ID = id.New().String()
+		c3.CreatedAt = strfmt.DateTime(time.Now().Add(200 * time.Millisecond))
+
+		err = ds.InsertCall(ctx, &c2)
+		if err != nil {
+			t.Fatal(err)
+		}
+		err = ds.InsertCall(ctx, &c3)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		// test that no filter works too
+		calls, err = ds.GetCalls(ctx, &models.CallFilter{PerPage: 100})
+		if err != nil {
 			t.Fatalf("Test GetCalls(ctx, filter): unexpected error `%v`", err)
 		}
+		if len(calls) != 3 {
+			t.Log(buf.String())
+			t.Fatalf("Test GetCalls(ctx, filter): unexpected length `%v`", len(calls))
+		}
+
+		// test that pagination stuff works. id, descending
+		filter.PerPage = 1
+		calls, err = ds.GetCalls(ctx, filter)
+		if err != nil {
+			t.Fatalf("Test GetCalls(ctx, filter): unexpected error `%v`", err)
+		}
+		if len(calls) != 1 {
+			t.Log(buf.String())
+			t.Fatalf("Test GetCalls(ctx, filter): unexpected length `%v`", len(calls))
+		} else if calls[0].ID != c3.ID {
+			t.Log(buf.String())
+			t.Fatalf("Test GetCalls: call ids not in expected order: %v %v", calls[0].ID, c3.ID)
+		}
+
+		filter.PerPage = 100
+		filter.Cursor = calls[0].ID
+		calls, err = ds.GetCalls(ctx, filter)
+		if err != nil {
+			t.Fatalf("Test GetCalls(ctx, filter): unexpected error `%v`", err)
+		}
+		if len(calls) != 2 {
+			t.Log(buf.String())
+			t.Fatalf("Test GetCalls(ctx, filter): unexpected length `%v`", len(calls))
+		} else if calls[0].ID != c2.ID {
+			t.Log(buf.String())
+			t.Fatalf("Test GetCalls: call ids not in expected order: %v %v", calls[0].ID, c2.ID)
+		} else if calls[1].ID != call.ID {
+			t.Log(buf.String())
+			t.Fatalf("Test GetCalls: call ids not in expected order: %v %v", calls[1].ID, call.ID)
+		}
+
+		// test that filters actually applied
+		calls, err = ds.GetCalls(ctx, &models.CallFilter{AppName: "wrongappname", PerPage: 100})
+		if err != nil {
+			t.Fatalf("Test GetCalls(ctx, filter): unexpected error `%v`", err)
+		}
+		if len(calls) != 0 {
+			t.Log(buf.String())
+			t.Fatalf("Test GetCalls(ctx, filter): unexpected length `%v`", len(calls))
+		}
+
+		calls, err = ds.GetCalls(ctx, &models.CallFilter{Path: "wrongpath", PerPage: 100})
+		if err != nil {
+			t.Fatalf("Test GetCalls(ctx, filter): unexpected error `%v`", err)
+		}
+		if len(calls) != 0 {
+			t.Log(buf.String())
+			t.Fatalf("Test GetCalls(ctx, filter): unexpected length `%v`", len(calls))
+		}
+
+		// make sure from_time and to_time work
+		filter = &models.CallFilter{
+			PerPage:  100,
+			FromTime: call.CreatedAt,
+			ToTime:   c3.CreatedAt,
+		}
+		calls, err = ds.GetCalls(ctx, filter)
+		if err != nil {
+			t.Fatalf("Test GetCalls(ctx, filter): unexpected error `%v`", err)
+		}
+		if len(calls) != 1 {
+			t.Log(buf.String())
+			t.Fatalf("Test GetCalls(ctx, filter): unexpected length `%v`", len(calls))
+		} else if calls[0].ID != c2.ID {
+			t.Log(buf.String())
+			t.Fatalf("Test GetCalls: call id not expected", calls[0].ID, c2.ID)
+		}
 	})
 
 	t.Run("apps", func(t *testing.T) {
+		ds := dsf()
 		// Testing insert app
 		_, err := ds.InsertApp(ctx, nil)
 		if err != models.ErrDatastoreEmptyApp {
@@ -166,7 +266,7 @@ func Test(t *testing.T, ds models.Datastore) {
 		}
 
 		// Testing list apps
-		apps, err := ds.GetApps(ctx, &models.AppFilter{})
+		apps, err := ds.GetApps(ctx, &models.AppFilter{PerPage: 100})
 		if err != nil {
 			t.Log(buf.String())
 			t.Fatalf("Test GetApps: unexpected error %v", err)
@@ -179,15 +279,74 @@ func Test(t *testing.T, ds models.Datastore) {
 			t.Fatalf("Test GetApps: expected `app.Name` to be `%s` but it was `%s`", app.Name, testApp.Name)
 		}
 
-		apps, err = ds.GetApps(ctx, &models.AppFilter{Name: "Tes%"})
+		// test pagination stuff (ordering / limits / cursoring)
+		a2 := *testApp
+		a3 := *testApp
+		a2.Name = "Testa"
+		a3.Name = "Testb"
+		if _, err = ds.InsertApp(ctx, &a2); err != nil {
+			t.Fatal(err)
+		}
+		if _, err = ds.InsertApp(ctx, &a3); err != nil {
+			t.Fatal(err)
+		}
+
+		apps, err = ds.GetApps(ctx, &models.AppFilter{PerPage: 1})
 		if err != nil {
 			t.Log(buf.String())
-			t.Fatalf("Test GetApps(filter): unexpected error %v", err)
+			t.Fatalf("Test GetApps: error: %s", err)
 		}
-		if len(apps) == 0 {
-			t.Fatal("Test GetApps(filter): expected result count to be greater than 0")
+		if len(apps) != 1 {
+			t.Fatalf("Test GetApps: expected result count to be 1 but got %d", len(apps))
+		} else if apps[0].Name != testApp.Name {
+			t.Log(buf.String())
+			t.Fatalf("Test GetApps: expected `app.Name` to be `%s` but it was `%s`", testApp.Name, apps[0].Name)
 		}
 
+		apps, err = ds.GetApps(ctx, &models.AppFilter{PerPage: 100, Cursor: apps[0].Name})
+		if err != nil {
+			t.Log(buf.String())
+			t.Fatalf("Test GetApps: error: %s", err)
+		}
+		if len(apps) != 2 {
+			t.Fatalf("Test GetApps: expected result count to be 2 but got %d", len(apps))
+		} else if apps[0].Name != a2.Name {
+			t.Log(buf.String())
+			t.Fatalf("Test GetApps: expected `app.Name` to be `%s` but it was `%s`", a2.Name, apps[0].Name)
+		} else if apps[1].Name != a3.Name {
+			t.Log(buf.String())
+			t.Fatalf("Test GetApps: expected `app.Name` to be `%s` but it was `%s`", a3.Name, apps[1].Name)
+		}
+
+		a4 := *testApp
+		a4.Name = "Abcdefg" // < /test lexicographically, but not in length
+
+		if _, err = ds.InsertApp(ctx, &a4); err != nil {
+			t.Fatal(err)
+		}
+
+		apps, err = ds.GetApps(ctx, &models.AppFilter{PerPage: 100})
+		if err != nil {
+			t.Log(buf.String())
+			t.Fatalf("Test GetApps: error: %s", err)
+		}
+		if len(apps) != 4 {
+			t.Fatalf("Test GetApps: expected result count to be 4 but got %d", len(apps))
+		} else if apps[0].Name != a4.Name {
+			t.Log(buf.String())
+			t.Fatalf("Test GetApps: expected `app.Name` to be `%s` but it was `%s`", a4.Name, apps[0].Name)
+		}
+
+		// TODO fix up prefix stuff
+		//apps, err = ds.GetApps(ctx, &models.AppFilter{Name: "Tes"})
+		//if err != nil {
+		//t.Log(buf.String())
+		//t.Fatalf("Test GetApps(filter): unexpected error %v", err)
+		//}
+		//if len(apps) != 3 {
+		//t.Fatal("Test GetApps(filter): expected result count to be 3, got", len(apps))
+		//}
+
 		// Testing app delete
 		err = ds.RemoveApp(ctx, "")
 		if err != models.ErrDatastoreEmptyAppName {
@@ -224,6 +383,7 @@ func Test(t *testing.T, ds models.Datastore) {
 	})
 
 	t.Run("routes", func(t *testing.T) {
+		ds := dsf()
 		// Insert app again to test routes
 		_, err := ds.InsertApp(ctx, testApp)
 		if err != nil && err != models.ErrAppsAlreadyExists {
@@ -374,7 +534,7 @@ func Test(t *testing.T, ds models.Datastore) {
 		}
 
 		// Testing list routes
-		routes, err := ds.GetRoutesByApp(ctx, testApp.Name, &models.RouteFilter{})
+		routes, err := ds.GetRoutesByApp(ctx, testApp.Name, &models.RouteFilter{PerPage: 1})
 		if err != nil {
 			t.Log(buf.String())
 			t.Fatalf("Test GetRoutesByApp: unexpected error %v", err)
@@ -390,7 +550,7 @@ func Test(t *testing.T, ds models.Datastore) {
 			t.Fatalf("Test GetRoutes: expected `app.Name` to be `%s` but it was `%s`", testRoute.Path, routes[0].Path)
 		}
 
-		routes, err = ds.GetRoutesByApp(ctx, testApp.Name, &models.RouteFilter{Image: testRoute.Image})
+		routes, err = ds.GetRoutesByApp(ctx, testApp.Name, &models.RouteFilter{Image: testRoute.Image, PerPage: 1})
 		if err != nil {
 			t.Log(buf.String())
 			t.Fatalf("Test GetRoutesByApp: unexpected error %v", err)
@@ -400,13 +560,13 @@ func Test(t *testing.T, ds models.Datastore) {
 		}
 		if routes[0] == nil {
 			t.Log(buf.String())
-			t.Fatalf("Test GetRoutes: expected non-nil route")
+			t.Fatalf("Test GetRoutesByApp: expected non-nil route")
 		} else if routes[0].Path != testRoute.Path {
 			t.Log(buf.String())
-			t.Fatalf("Test GetRoutes: expected `app.Name` to be `%s` but it was `%s`", testRoute.Path, routes[0].Path)
+			t.Fatalf("Test GetRoutesByApp: expected `route.Path` to be `%s` but it was `%s`", testRoute.Path, routes[0].Path)
 		}
 
-		routes, err = ds.GetRoutesByApp(ctx, "notreal", nil)
+		routes, err = ds.GetRoutesByApp(ctx, "notreal", &models.RouteFilter{PerPage: 1})
 		if err != nil {
 			t.Log(buf.String())
 			t.Fatalf("Test GetRoutesByApp: error: %s", err)
@@ -415,20 +575,68 @@ func Test(t *testing.T, ds models.Datastore) {
 			t.Fatalf("Test GetRoutesByApp: expected result count to be 0 but got %d", len(routes))
 		}
 
-		// Testing list routes
-		routes, err = ds.GetRoutes(ctx, &models.RouteFilter{Image: testRoute.Image})
+		// test pagination stuff
+		r2 := *testRoute
+		r3 := *testRoute
+		r2.Path = "/testa"
+		r3.Path = "/testb"
+
+		if _, err = ds.InsertRoute(ctx, &r2); err != nil {
+			t.Fatal(err)
+		}
+		if _, err = ds.InsertRoute(ctx, &r3); err != nil {
+			t.Fatal(err)
+		}
+
+		routes, err = ds.GetRoutesByApp(ctx, testApp.Name, &models.RouteFilter{PerPage: 1})
 		if err != nil {
 			t.Log(buf.String())
-			t.Fatalf("Test GetRoutes: error: %s", err)
+			t.Fatalf("Test GetRoutesByApp: error: %s", err)
 		}
-		if len(routes) == 0 {
-			t.Fatal("Test GetRoutes: expected result count to be greater than 0")
-		}
-		if routes[0].Path != testRoute.Path {
+		if len(routes) != 1 {
+			t.Fatalf("Test GetRoutesByApp: expected result count to be 1 but got %d", len(routes))
+		} else if routes[0].Path != testRoute.Path {
 			t.Log(buf.String())
-			t.Fatalf("Test GetRoutes: expected `app.Name` to be `%s` but it was `%s`", testRoute.Path, routes[0].Path)
+			t.Fatalf("Test GetRoutesByApp: expected `route.Path` to be `%s` but it was `%s`", testRoute.Path, routes[0].Path)
 		}
 
+		routes, err = ds.GetRoutesByApp(ctx, testApp.Name, &models.RouteFilter{PerPage: 2, Cursor: routes[0].Path})
+		if err != nil {
+			t.Log(buf.String())
+			t.Fatalf("Test GetRoutesByApp: error: %s", err)
+		}
+		if len(routes) != 2 {
+			t.Fatalf("Test GetRoutesByApp: expected result count to be 2 but got %d", len(routes))
+		} else if routes[0].Path != r2.Path {
+			t.Log(buf.String())
+			t.Fatalf("Test GetRoutesByApp: expected `route.Path` to be `%s` but it was `%s`", r2.Path, routes[0].Path)
+		} else if routes[1].Path != r3.Path {
+			t.Log(buf.String())
+			t.Fatalf("Test GetRoutesByApp: expected `route.Path` to be `%s` but it was `%s`", r3.Path, routes[1].Path)
+		}
+
+		r4 := *testRoute
+		r4.Path = "/abcdefg" // < /test lexicographically, but not in length
+
+		if _, err = ds.InsertRoute(ctx, &r4); err != nil {
+			t.Fatal(err)
+		}
+
+		routes, err = ds.GetRoutesByApp(ctx, testApp.Name, &models.RouteFilter{PerPage: 100})
+		if err != nil {
+			t.Log(buf.String())
+			t.Fatalf("Test GetRoutesByApp: error: %s", err)
+		}
+		if len(routes) != 4 {
+			t.Fatalf("Test GetRoutesByApp: expected result count to be 4 but got %d", len(routes))
+		} else if routes[0].Path != r4.Path {
+			t.Log(buf.String())
+			t.Fatalf("Test GetRoutesByApp: expected `route.Path` to be `%s` but it was `%s`", r4.Path, routes[0].Path)
+		}
+
+		// TODO test weird ordering possibilities ?
+		// TODO test prefix filtering
+
 		// Testing route delete
 		err = ds.RemoveRoute(ctx, "", "")
 		if err != models.ErrDatastoreEmptyAppName {

api/datastore/internal/datastoreutil/metrics.go

@@ -53,12 +53,6 @@ func (m *metricds) GetRoute(ctx context.Context, appName, routePath string) (*mo
 	return m.ds.GetRoute(ctx, appName, routePath)
 }
 
-func (m *metricds) GetRoutes(ctx context.Context, filter *models.RouteFilter) (routes []*models.Route, err error) {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "ds_get_routes")
-	defer span.Finish()
-	return m.ds.GetRoutes(ctx, filter)
-}
-
 func (m *metricds) GetRoutesByApp(ctx context.Context, appName string, filter *models.RouteFilter) (routes []*models.Route, err error) {
 	span, ctx := opentracing.StartSpanFromContext(ctx, "ds_get_routes_by_app")
 	defer span.Finish()

api/datastore/internal/datastoreutil/validator.go

@@ -73,14 +73,6 @@ func (v *validator) GetRoute(ctx context.Context, appName, routePath string) (*m
 	return v.Datastore.GetRoute(ctx, appName, routePath)
 }
 
-func (v *validator) GetRoutes(ctx context.Context, routeFilter *models.RouteFilter) (routes []*models.Route, err error) {
-	if routeFilter != nil && routeFilter.AppName != "" {
-		return v.Datastore.GetRoutesByApp(ctx, routeFilter.AppName, routeFilter)
-	}
-
-	return v.Datastore.GetRoutes(ctx, routeFilter)
-}
-
 // appName will never be empty
 func (v *validator) GetRoutesByApp(ctx context.Context, appName string, routeFilter *models.RouteFilter) (routes []*models.Route, err error) {
 	if appName == "" {

api/datastore/mock.go

@@ -2,12 +2,14 @@ package datastore
 
 import (
 	"context"
-
-	"github.com/jmoiron/sqlx"
+	"sort"
+	"strings"
+	"time"
 
 	"github.com/fnproject/fn/api/datastore/internal/datastoreutil"
 	"github.com/fnproject/fn/api/logs"
 	"github.com/fnproject/fn/api/models"
+	"github.com/jmoiron/sqlx"
 )
 
 type mock struct {
@@ -37,8 +39,27 @@ func (m *mock) GetApp(ctx context.Context, appName string) (app *models.App, err
 	return nil, models.ErrAppsNotFound
 }
 
+type sortA []*models.App
+
+func (s sortA) Len() int           { return len(s) }
+func (s sortA) Less(i, j int) bool { return strings.Compare(s[i].Name, s[j].Name) < 0 }
+func (s sortA) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }
+
 func (m *mock) GetApps(ctx context.Context, appFilter *models.AppFilter) ([]*models.App, error) {
-	return m.Apps, nil
+	// sort them all first for cursoring (this is for testing, n is small & mock is not concurrent..)
+	sort.Sort(sortA(m.Apps))
+
+	var apps []*models.App
+	for _, a := range m.Apps {
+		if len(apps) == appFilter.PerPage {
+			break
+		}
+		if strings.Compare(appFilter.Cursor, a.Name) < 0 {
+			apps = append(apps, a)
+		}
+	}
+
+	return apps, nil
 }
 
 func (m *mock) InsertApp(ctx context.Context, app *models.App) (*models.App, error) {
@@ -80,16 +101,26 @@ func (m *mock) GetRoute(ctx context.Context, appName, routePath string) (*models
 	return nil, models.ErrRoutesNotFound
 }
 
-func (m *mock) GetRoutes(ctx context.Context, routeFilter *models.RouteFilter) (routes []*models.Route, err error) {
-	for _, r := range m.Routes {
-		routes = append(routes, r)
-	}
-	return
-}
+type sortR []*models.Route
+
+func (s sortR) Len() int           { return len(s) }
+func (s sortR) Less(i, j int) bool { return strings.Compare(s[i].Path, s[j].Path) < 0 }
+func (s sortR) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }
 
 func (m *mock) GetRoutesByApp(ctx context.Context, appName string, routeFilter *models.RouteFilter) (routes []*models.Route, err error) {
+	// sort them all first for cursoring (this is for testing, n is small & mock is not concurrent..)
+	sort.Sort(sortR(m.Routes))
+
 	for _, r := range m.Routes {
-		if r.AppName == appName && (routeFilter.Path == "" || r.Path == routeFilter.Path) && (routeFilter.AppName == "" || r.AppName == routeFilter.AppName) {
+		if len(routes) == routeFilter.PerPage {
+			break
+		}
+
+		if r.AppName == appName &&
+			//strings.HasPrefix(r.Path, routeFilter.PathPrefix) && // TODO
+			(routeFilter.Image == "" || routeFilter.Image == r.Image) &&
+			strings.Compare(routeFilter.Cursor, r.Path) < 0 {
+
 			routes = append(routes, r)
 		}
 	}
@@ -147,7 +178,7 @@ func (m *mock) InsertCall(ctx context.Context, call *models.Call) error {
 
 func (m *mock) GetCall(ctx context.Context, appName, callID string) (*models.Call, error) {
 	for _, t := range m.Calls {
-		if t.ID == callID {
+		if t.ID == callID && t.AppName == appName {
 			return t, nil
 		}
 	}
@@ -155,8 +186,34 @@ func (m *mock) GetCall(ctx context.Context, appName, callID string) (*models.Cal
 	return nil, models.ErrCallNotFound
 }
 
+type sortC []*models.Call
+
+func (s sortC) Len() int           { return len(s) }
+func (s sortC) Less(i, j int) bool { return strings.Compare(s[i].ID, s[j].ID) < 0 }
+func (s sortC) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }
+
 func (m *mock) GetCalls(ctx context.Context, filter *models.CallFilter) ([]*models.Call, error) {
-	return m.Calls, nil
+	// sort them all first for cursoring (this is for testing, n is small & mock is not concurrent..)
+	// calls are in DESC order so use sort.Reverse
+	sort.Sort(sort.Reverse(sortC(m.Calls)))
+
+	var calls []*models.Call
+	for _, c := range m.Calls {
+		if len(calls) == filter.PerPage {
+			break
+		}
+
+		if (filter.AppName == "" || c.AppName == filter.AppName) &&
+			(filter.Path == "" || filter.Path == c.Path) &&
+			(time.Time(filter.FromTime).IsZero() || time.Time(filter.FromTime).Before(time.Time(c.CreatedAt))) &&
+			(time.Time(filter.ToTime).IsZero() || time.Time(c.CreatedAt).Before(time.Time(filter.ToTime))) &&
+			(filter.Cursor == "" || strings.Compare(filter.Cursor, c.ID) > 0) {
+
+			calls = append(calls, c)
+		}
+	}
+
+	return calls, nil
 }
 
 func (m *mock) batchDeleteCalls(ctx context.Context, appName string) error {

api/datastore/mock_test.go

@@ -7,5 +7,5 @@ import (
 )
 
 func TestDatastore(t *testing.T) {
-	datastoretest.Test(t, NewMock())
+	datastoretest.Test(t, NewMock)
 }

api/datastore/sql/sql.go

@@ -12,6 +12,7 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+	"time"
 
 	"github.com/fnproject/fn/api/models"
 	"github.com/go-sql-driver/mysql"
@@ -471,54 +472,23 @@ func (ds *sqlStore) GetRoute(ctx context.Context, appName, routePath string) (*m
 	return &route, nil
 }
 
-// GetRoutes retrieves an array of routes according to a specific filter.
-func (ds *sqlStore) GetRoutes(ctx context.Context, filter *models.RouteFilter) ([]*models.Route, error) {
-	res := []*models.Route{}
-	query, args := buildFilterRouteQuery(filter)
-	query = fmt.Sprintf("%s %s", routeSelector, query)
-	query = ds.db.Rebind(query)
-	rows, err := ds.db.QueryContext(ctx, query, args...)
-	// todo: check for no rows so we don't respond with a sql 500 err
-	if err != nil {
-		return nil, err
-	}
-	defer rows.Close()
-
-	for rows.Next() {
-		var route models.Route
-		err := scanRoute(rows, &route)
-		if err != nil {
-			continue
-		}
-		res = append(res, &route)
-
-	}
-	if err := rows.Err(); err != nil {
-		return nil, err
-	}
-	return res, nil
-}
-
-/*
-GetRoutesByApp retrieves a route with a specific app name.
-*/
+// GetRoutesByApp retrieves a route with a specific app name.
 func (ds *sqlStore) GetRoutesByApp(ctx context.Context, appName string, filter *models.RouteFilter) ([]*models.Route, error) {
 	res := []*models.Route{}
-	var filterQuery string
-	var args []interface{}
 	if filter == nil {
-		filterQuery = "WHERE app_name = ?"
-		args = []interface{}{appName}
-	} else {
-		filter.AppName = appName
-		filterQuery, args = buildFilterRouteQuery(filter)
+		filter = new(models.RouteFilter)
 	}
 
+	filter.AppName = appName
+	filterQuery, args := buildFilterRouteQuery(filter)
+
 	query := fmt.Sprintf("%s %s", routeSelector, filterQuery)
 	query = ds.db.Rebind(query)
 	rows, err := ds.db.QueryContext(ctx, query, args...)
-	// todo: check for no rows so we don't respond with a sql 500 err
 	if err != nil {
+		if err == sql.ErrNoRows {
+			return res, nil // no error for empty list
+		}
 		return nil, err
 	}
 	defer rows.Close()
@@ -533,7 +503,9 @@ func (ds *sqlStore) GetRoutesByApp(ctx context.Context, appName string, filter *
 
 	}
 	if err := rows.Err(); err != nil {
-		return nil, err
+		if err == sql.ErrNoRows {
+			return res, nil // no error for empty list
+		}
 	}
 
 	return res, nil
@@ -739,26 +711,52 @@ func buildFilterRouteQuery(filter *models.RouteFilter) (string, []interface{}) {
 		if val != "" {
 			args = append(args, val)
 			if len(args) == 1 {
-				fmt.Fprintf(&b, `WHERE %s?`, colOp)
+				fmt.Fprintf(&b, `WHERE %s`, colOp)
 			} else {
-				fmt.Fprintf(&b, ` AND %s?`, colOp)
+				fmt.Fprintf(&b, ` AND %s`, colOp)
 			}
 		}
 	}
 
-	where("path=", filter.Path)
-	where("app_name=", filter.AppName)
-	where("image=", filter.Image)
+	where("app_name=? ", filter.AppName)
+	where("image=?", filter.Image)
+	where("path>?", filter.Cursor)
+	// where("path LIKE ?%", filter.PathPrefix) TODO needs escaping
+
+	fmt.Fprintf(&b, ` ORDER BY path ASC`) // TODO assert this is indexed
+	fmt.Fprintf(&b, ` LIMIT ?`)
+	args = append(args, filter.PerPage)
 
 	return b.String(), args
 }
 
 func buildFilterAppQuery(filter *models.AppFilter) (string, []interface{}) {
-	if filter == nil || filter.Name == "" {
+	if filter == nil {
 		return "", nil
 	}
 
-	return "WHERE name LIKE ?", []interface{}{filter.Name}
+	var b bytes.Buffer
+	var args []interface{}
+
+	where := func(colOp, val string) {
+		if val != "" {
+			args = append(args, val)
+			if len(args) == 1 {
+				fmt.Fprintf(&b, `WHERE %s`, colOp)
+			} else {
+				fmt.Fprintf(&b, ` AND %s`, colOp)
+			}
+		}
+	}
+
+	// where("name LIKE ?%", filter.Name) // TODO needs escaping?
+	where("name>?", filter.Cursor)
+
+	fmt.Fprintf(&b, ` ORDER BY name ASC`) // TODO assert this is indexed
+	fmt.Fprintf(&b, ` LIMIT ?`)
+	args = append(args, filter.PerPage)
+
+	return b.String(), args
 }
 
 func buildFilterCallQuery(filter *models.CallFilter) (string, []interface{}) {
@@ -779,11 +777,19 @@ func buildFilterCallQuery(filter *models.CallFilter) (string, []interface{}) {
 		}
 	}
 
-	where("app_name=", filter.AppName)
-
-	if filter.Path != "" {
-		where("path=", filter.Path)
+	where("id<", filter.Cursor)
+	if !time.Time(filter.ToTime).IsZero() {
+		where("created_at<", filter.ToTime.String())
 	}
+	if !time.Time(filter.FromTime).IsZero() {
+		where("created_at>", filter.FromTime.String())
+	}
+	where("app_name=", filter.AppName)
+	where("path=", filter.Path)
+
+	fmt.Fprintf(&b, ` ORDER BY id DESC`) // TODO assert this is indexed
+	fmt.Fprintf(&b, ` LIMIT ?`)
+	args = append(args, filter.PerPage)
 
 	return b.String(), args
 }

api/datastore/sql/sql_test.go

@@ -0,0 +1,29 @@
+package sql
+
+import (
+	"net/url"
+	"os"
+	"testing"
+
+	"github.com/fnproject/fn/api/datastore/internal/datastoretest"
+	"github.com/fnproject/fn/api/datastore/internal/datastoreutil"
+	"github.com/fnproject/fn/api/models"
+)
+
+func TestDatastore(t *testing.T) {
+	defer os.RemoveAll("sqlite_test_dir")
+	u, err := url.Parse("sqlite3://sqlite_test_dir")
+	if err != nil {
+		t.Fatal(err)
+	}
+	f := func() models.Datastore {
+		os.RemoveAll("sqlite_test_dir")
+		ds, err := New(u)
+		if err != nil {
+			t.Fatal(err)
+		}
+		// we don't want to test the validator, really
+		return datastoreutil.NewValidator(ds)
+	}
+	datastoretest.Test(t, f)
+}