alihan/fn-serverless
1
0
Fork 0
mirror of https://github.com/fnproject/fn.git synced 2022-10-28 21:29:17 +03:00
Code Issues Packages Projects Releases Wiki Activity

Adds before/after app get/list. And some bug fixes/cleanup. (#610)

Browse Source 
* Adds before/after app get/list. And some bug fixes/cleanup.

* Fix test
This commit is contained in:
Travis Reeder
2017-12-21 09:32:03 -08:00
committed by GitHub
parent 0a1b180158
commit fdb4188146
26 changed files with 212 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
api/datastore/sql/sql.go
View File

@@ -362,7 +362,16 @@ func (ds *sqlStore) GetApp(ctx context.Context, name string) (*models.App, error
// GetApps retrieves an array of apps according to a specific filter.
func (ds *sqlStore) GetApps(ctx context.Context, filter *models.AppFilter) ([]*models.App, error) {
res := []*models.App{}
query, args := buildFilterAppQuery(filter)
if filter.NameIn != nil && len(filter.NameIn) == 0 { // this basically makes sure it doesn't return ALL apps
return res, nil
}
query, args, err := buildFilterAppQuery(filter)
if err != nil {
return nil, err
}
// fmt.Printf("QUERY: %v\n", query)
// fmt.Printf("ARGS: %v\n", args)
// hmm, should this have DISTINCT in it? shouldn't be possible to have two apps with same name
query = ds.db.Rebind(fmt.Sprintf("SELECT DISTINCT name, config FROM apps %s", query))
rows, err := ds.db.QueryxContext(ctx, query, args...)
if err != nil {
@@ -787,16 +796,29 @@ func buildFilterRouteQuery(filter *models.RouteFilter) (string, []interface{}) {
return b.String(), args
}
func buildFilterAppQuery(filter *models.AppFilter) (string, []interface{}) {
func buildFilterAppQuery(filter *models.AppFilter) (string, []interface{}, error) {
var args []interface{}
if filter == nil {
return "", nil
return "", args, nil
}
var b bytes.Buffer
var args []interface{}
where := func(colOp, val string) {
if val != "" {
// todo: this same thing is in several places in here, DRY it up across this file
where := func(colOp, val interface{}) {
if val == nil {
return
}
switch v := val.(type) {
case string:
if v == "" {
return
}
case []string:
if len(v) == 0 {
return
}
}
args = append(args, val)
if len(args) == 1 {
fmt.Fprintf(&b, `WHERE %s`, colOp)
@@ -804,16 +826,19 @@ func buildFilterAppQuery(filter *models.AppFilter) (string, []interface{}) {
fmt.Fprintf(&b, ` AND %s`, colOp)
}
}
}
// where("name LIKE ?%", filter.Name) // TODO needs escaping?
where("name>?", filter.Cursor)
where("name IN (?)", filter.NameIn)
fmt.Fprintf(&b, ` ORDER BY name ASC`) // TODO assert this is indexed
fmt.Fprintf(&b, ` LIMIT ?`)
args = append(args, filter.PerPage)
return b.String(), args
if len(filter.NameIn) > 0 {
// fmt.Println("about to sqlx.in:", b.String(), args)
return sqlx.In(b.String(), args...)
}
return b.String(), args, nil
}
func buildFilterCallQuery(filter *models.CallFilter) (string, []interface{}) {

5
api/models/app.go
View File

@@ -48,8 +48,11 @@ func (a *App) UpdateConfig(patch Config) {
}
}
// AppFilter is the filter used for querying apps
type AppFilter struct {
Name string // prefix query TODO implemented
Name string
// NameIn will filter by all names in the list (IN query)
NameIn []string
PerPage int
Cursor string
}

47
api/server/app_listeners.go
View File

@@ -77,3 +77,50 @@ func (s *Server) FireAfterAppDelete(ctx context.Context, app *models.App) error
}
return nil
}
// FireBeforeAppGet runs AppListener's BeforeAppGet method.
// todo: All of these listener methods could/should return the 2nd param rather than modifying in place. For instance,
// if a listener were to change the appName here (maybe prefix it or something for the database), it wouldn't be reflected anywhere else.
// If this returned appName, then keep passing along the returned appName, it would work.
func (s *Server) FireBeforeAppGet(ctx context.Context, appName string) error {
for _, l := range s.appListeners {
err := l.BeforeAppGet(ctx, appName)
if err != nil {
return err
}
}
return nil
}
// FireAfterAppGet runs AppListener's AfterAppGet method.
func (s *Server) FireAfterAppGet(ctx context.Context, app *models.App) error {
for _, l := range s.appListeners {
err := l.AfterAppGet(ctx, app)
if err != nil {
return err
}
}
return nil
}
// FireBeforeAppsList runs AppListener's BeforeAppsList method.
func (s *Server) FireBeforeAppsList(ctx context.Context, filter *models.AppFilter) error {
for _, l := range s.appListeners {
err := l.BeforeAppsList(ctx, filter)
if err != nil {
return err
}
}
return nil
}
// FireAfterAppsList runs AppListener's AfterAppsList method.
func (s *Server) FireAfterAppsList(ctx context.Context, apps []*models.App) error {
for _, l := range s.appListeners {
err := l.AfterAppsList(ctx, apps)
if err != nil {
return err
}
}
return nil
}

2
api/server/apps_create.go
View File

@@ -34,7 +34,7 @@ func (s *Server) handleAppCreate(c *gin.Context) {
return
}
app, err := s.Datastore.InsertApp(ctx, wapp.App)
app, err := s.Datastore().InsertApp(ctx, wapp.App)
if err != nil {
handleErrorResponse(c, err)
return

2
api/server/apps_delete.go
View File

@@ -17,7 +17,7 @@ func (s *Server) handleAppDelete(c *gin.Context) {
err := s.FireBeforeAppDelete(ctx, app)
err = s.Datastore.RemoveApp(ctx, app.Name)
err = s.Datastore().RemoveApp(ctx, app.Name)
if err != nil {
handleErrorResponse(c, err)
return

14
api/server/apps_get.go
View File

@@ -11,7 +11,19 @@ func (s *Server) handleAppGet(c *gin.Context) {
ctx := c.Request.Context()
appName := c.MustGet(api.AppName).(string)
app, err := s.Datastore.GetApp(ctx, appName)
err := s.FireBeforeAppGet(ctx, appName)
if err != nil {
handleErrorResponse(c, err)
return
}
app, err := s.Datastore().GetApp(ctx, appName)
if err != nil {
handleErrorResponse(c, err)
return
}
err = s.FireAfterAppGet(ctx, app)
if err != nil {
handleErrorResponse(c, err)
return

15
api/server/apps_list.go
View File

@@ -11,10 +11,21 @@ import (
func (s *Server) handleAppList(c *gin.Context) {
ctx := c.Request.Context()
var filter models.AppFilter
filter := &models.AppFilter{}
filter.Cursor, filter.PerPage = pageParams(c, true)
apps, err := s.Datastore.GetApps(ctx, &filter)
err := s.FireBeforeAppsList(ctx, filter)
if err != nil {
handleErrorResponse(c, err)
return
}
apps, err := s.Datastore().GetApps(ctx, filter)
if err != nil {
handleErrorResponse(c, err)
return
}
err = s.FireAfterAppsList(ctx, apps)
if err != nil {
handleErrorResponse(c, err)
return

4
api/server/apps_update.go
View File

@@ -31,13 +31,13 @@ func (s *Server) handleAppUpdate(c *gin.Context) {
wapp.App.Name = c.MustGet(api.AppName).(string)
err = s.FireAfterAppUpdate(ctx, wapp.App)
err = s.FireBeforeAppUpdate(ctx, wapp.App)
if err != nil {
handleErrorResponse(c, err)
return
}
app, err := s.Datastore.UpdateApp(ctx, wapp.App)
app, err := s.Datastore().UpdateApp(ctx, wapp.App)
if err != nil {
handleErrorResponse(c, err)
return

2
api/server/call_get.go
View File

@@ -12,7 +12,7 @@ func (s *Server) handleCallGet(c *gin.Context) {
appName := c.MustGet(api.AppName).(string)
callID := c.Param(api.Call)
callObj, err := s.Datastore.GetCall(ctx, appName, callID)
callObj, err := s.Datastore().GetCall(ctx, appName, callID)
if err != nil {
handleErrorResponse(c, err)
return

4
api/server/call_list.go
View File

@@ -27,11 +27,11 @@ func (s *Server) handleCallList(c *gin.Context) {
return
}
calls, err := s.Datastore.GetCalls(ctx, &filter)
calls, err := s.Datastore().GetCalls(ctx, &filter)
if len(calls) == 0 {
// TODO this should be done in front of this handler to even get here...
_, err = s.Datastore.GetApp(c, appName)
_, err = s.Datastore().GetApp(c, appName)
}
if err != nil {

6
api/server/extension_points.go
View File

@@ -20,7 +20,7 @@ func (s *Server) apiAppHandlerWrapperFunc(apiHandler fnext.ApiAppHandler) gin.Ha
return func(c *gin.Context) {
// get the app
appName := c.Param(api.CApp)
app, err := s.Datastore.GetApp(c.Request.Context(), appName)
app, err := s.Datastore().GetApp(c.Request.Context(), appName)
if err != nil {
handleErrorResponse(c, err)
c.Abort()
@@ -41,7 +41,7 @@ func (s *Server) apiRouteHandlerWrapperFunc(apiHandler fnext.ApiRouteHandler) gi
context := c.Request.Context()
// get the app
appName := c.Param(api.CApp)
app, err := s.Datastore.GetApp(context, appName)
app, err := s.Datastore().GetApp(context, appName)
if err != nil {
handleErrorResponse(c, err)
c.Abort()
@@ -54,7 +54,7 @@ func (s *Server) apiRouteHandlerWrapperFunc(apiHandler fnext.ApiRouteHandler) gi
}
// get the route TODO
routePath := "/" + c.Param(api.CRoute)
route, err := s.Datastore.GetRoute(context, appName, routePath)
route, err := s.Datastore().GetRoute(context, appName, routePath)
if err != nil {
handleErrorResponse(c, err)
c.Abort()

12
api/server/gin_middlewares.go
View File

@@ -10,6 +10,7 @@ import (
"github.com/fnproject/fn/api"
"github.com/fnproject/fn/api/common"
"github.com/fnproject/fn/api/models"
"github.com/fnproject/fn/fnext"
"github.com/gin-contrib/cors"
"github.com/gin-gonic/gin"
opentracing "github.com/opentracing/opentracing-go"
@@ -85,7 +86,16 @@ func loggerWrap(c *gin.Context) {
c.Next()
}
func appWrap(c *gin.Context) {
func setAppNameInCtx(c *gin.Context) {
// add appName to context
appName := c.GetString(api.AppName)
if appName != "" {
c.Request = c.Request.WithContext(context.WithValue(c.Request.Context(), fnext.AppNameKey, appName))
}
c.Next()
}
func appNameCheck(c *gin.Context) {
appName := c.GetString(api.AppName)
if appName == "" {
handleErrorResponse(c, models.ErrAppsMissingName)

6
api/server/hybrid.go
View File

@@ -44,7 +44,7 @@ func (s *Server) handleRunnerEnqueue(c *gin.Context) {
// runner and enter into 'running' state before we can insert it in the db as
// 'queued' state. we can ignore any error inserting into db here and Start
// will ensure the call exists in the db in 'running' state there.
// s.Datastore.InsertCall(ctx, &call)
// s.Datastore().InsertCall(ctx, &call)
c.JSON(200, struct {
M string `json:"msg"`
@@ -112,7 +112,7 @@ func (s *Server) handleRunnerStart(c *gin.Context) {
// TODO do this client side and validate it here?
//call.Status = "running"
//call.StartedAt = strfmt.DateTime(time.Now())
//err := s.Datastore.UpdateCall(c.Request.Context(), &call)
//err := s.Datastore().UpdateCall(c.Request.Context(), &call)
//if err != nil {
//if err == InvalidStatusChange {
//// TODO we could either let UpdateCall handle setting to error or do it
@@ -153,7 +153,7 @@ func (s *Server) handleRunnerFinish(c *gin.Context) {
// TODO this needs UpdateCall functionality to work for async and should only work if:
// running->error|timeout|success
// TODO all async will fail here :( all sync will work fine :) -- *feeling conflicted*
if err := s.Datastore.InsertCall(ctx, &call); err != nil {
if err := s.Datastore().InsertCall(ctx, &call); err != nil {
common.Logger(ctx).WithError(err).Error("error inserting call into datastore")
// note: Not returning err here since the job could have already finished successfully.
}

4
api/server/middleware.go
View File

@@ -69,13 +69,15 @@ func (s *Server) runMiddleware(c *gin.Context, ms []fnext.Middleware) {
ctx := context.WithValue(c.Request.Context(), fnext.MiddlewareControllerKey, s.newMiddlewareController(c))
last := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// fmt.Println("final handler called")
ctx := r.Context()
mctx := fnext.GetMiddlewareController(ctx)
// check for bypass
mctx := fnext.GetMiddlewareController(r.Context())
if mctx.FunctionCalled() {
// fmt.Println("func already called, skipping")
c.Abort()
return
}
c.Request = c.Request.WithContext(ctx)
c.Next()
})

10
api/server/routes_create_update.go
View File

@@ -57,7 +57,7 @@ func (s *Server) submitRoute(ctx context.Context, wroute *models.RouteWrapper) e
if err != nil {
return err
}
r, err := s.Datastore.InsertRoute(ctx, wroute.Route)
r, err := s.Datastore().InsertRoute(ctx, wroute.Route)
if err != nil {
return err
}
@@ -66,7 +66,7 @@ func (s *Server) submitRoute(ctx context.Context, wroute *models.RouteWrapper) e
}
func (s *Server) changeRoute(ctx context.Context, wroute *models.RouteWrapper) error {
r, err := s.Datastore.UpdateRoute(ctx, wroute.Route)
r, err := s.Datastore().UpdateRoute(ctx, wroute.Route)
if err != nil {
return err
}
@@ -86,7 +86,7 @@ func (s *Server) ensureRoute(ctx context.Context, method string, wroute *models.
}
return routeResponse{"Route successfully created", wroute.Route}, nil
case http.MethodPut:
_, err := s.Datastore.GetRoute(ctx, wroute.Route.AppName, wroute.Route.Path)
_, err := s.Datastore().GetRoute(ctx, wroute.Route.AppName, wroute.Route.Path)
if err != nil && err == models.ErrRoutesNotFound {
err := s.submitRoute(ctx, wroute)
if err != nil {
@@ -111,7 +111,7 @@ func (s *Server) ensureRoute(ctx context.Context, method string, wroute *models.
// ensureApp will only execute if it is on post or put. Patch is not allowed to create apps.
func (s *Server) ensureApp(ctx context.Context, wroute *models.RouteWrapper, method string) error {
app, err := s.Datastore.GetApp(ctx, wroute.Route.AppName)
app, err := s.Datastore().GetApp(ctx, wroute.Route.AppName)
if err != nil && err != models.ErrAppsNotFound {
return err
} else if app == nil {
@@ -126,7 +126,7 @@ func (s *Server) ensureApp(ctx context.Context, wroute *models.RouteWrapper, met
return err
}
_, err = s.Datastore.InsertApp(ctx, newapp)
_, err = s.Datastore().InsertApp(ctx, newapp)
if err != nil {
return err
}

4
api/server/routes_delete.go
View File

@@ -14,12 +14,12 @@ func (s *Server) handleRouteDelete(c *gin.Context) {
appName := c.MustGet(api.AppName).(string)
routePath := path.Clean(c.MustGet(api.Path).(string))
if _, err := s.Datastore.GetRoute(ctx, appName, routePath); err != nil {
if _, err := s.Datastore().GetRoute(ctx, appName, routePath); err != nil {
handleErrorResponse(c, err)
return
}
if err := s.Datastore.RemoveRoute(ctx, appName, routePath); err != nil {
if err := s.Datastore().RemoveRoute(ctx, appName, routePath); err != nil {
handleErrorResponse(c, err)
return
}

2
api/server/routes_get.go
View File

@@ -13,7 +13,7 @@ func (s *Server) handleRouteGet(c *gin.Context) {
appName := c.MustGet(api.AppName).(string)
routePath := path.Clean("/" + c.MustGet(api.Path).(string))
route, err := s.Datastore.GetRoute(ctx, appName, routePath)
route, err := s.Datastore().GetRoute(ctx, appName, routePath)
if err != nil {
handleErrorResponse(c, err)
return

4
api/server/routes_list.go
View File

@@ -19,13 +19,13 @@ func (s *Server) handleRouteList(c *gin.Context) {
// filter.PathPrefix = c.Query("path_prefix") TODO not hooked up
filter.Cursor, filter.PerPage = pageParams(c, true)
routes, err := s.Datastore.GetRoutesByApp(ctx, appName, &filter)
routes, err := s.Datastore().GetRoutesByApp(ctx, appName, &filter)
// if there are no routes for the app, check if the app exists to return
// 404 if it does not
// TODO this should be done in front of this handler to even get here...
if err == nil && len(routes) == 0 {
_, err = s.Datastore.GetApp(ctx, appName)
_, err = s.Datastore().GetApp(ctx, appName)
}
if err != nil {

2
api/server/runner_async_test.go
View File

@@ -19,7 +19,7 @@ func testRouterAsync(ds models.Datastore, mq models.MessageQueue, rnr agent.Agen
s := &Server{
Agent: rnr,
Router: gin.New(),
Datastore: ds,
datastore: ds,
MQ: mq,
nodeType: ServerTypeFull,
}

19
api/server/server.go
View File

@@ -66,7 +66,7 @@ func (s ServerNodeType) String() string {
type Server struct {
Router *gin.Engine
Agent agent.Agent
Datastore models.Datastore
datastore models.Datastore
MQ models.MessageQueue
LogDB models.LogStore
nodeType ServerNodeType
@@ -164,7 +164,7 @@ func WithType(t ServerNodeType) ServerOption {
}
func WithDatastore(ds models.Datastore) ServerOption {
return func(s *Server) { s.Datastore = ds }
return func(s *Server) { s.datastore = ds }
}
func WithMQ(mq models.MessageQueue) ServerOption {
@@ -195,7 +195,7 @@ func New(ctx context.Context, opts ...ServerOption) *Server {
}
if s.LogDB == nil { // TODO seems weird?
s.LogDB = s.Datastore
s.LogDB = s.Datastore()
}
// TODO we maybe should use the agent.DirectDataAccess in the /runner endpoints server side?
@@ -209,13 +209,13 @@ func New(ctx context.Context, opts ...ServerOption) *Server {
}
default:
s.nodeType = ServerTypeFull
if s.Datastore == nil || s.LogDB == nil || s.MQ == nil {
if s.Datastore() == nil || s.LogDB == nil || s.MQ == nil {
logrus.Fatal("Full nodes must configure FN_DB_URL, FN_LOG_URL, FN_MQ_URL.")
}
// TODO force caller to use WithAgent option ?
// TODO for tests we don't want cache, really, if we force WithAgent this can be fixed. cache needs to be moved anyway so that runner nodes can use it...
s.Agent = agent.New(agent.NewCachedDataAccess(agent.NewDirectDataAccess(s.Datastore, s.LogDB, s.MQ)))
s.Agent = agent.New(agent.NewCachedDataAccess(agent.NewDirectDataAccess(s.Datastore(), s.LogDB, s.MQ)))
}
setMachineID()
@@ -376,13 +376,14 @@ func (s *Server) bindHandlers(ctx context.Context) {
if s.nodeType != ServerTypeRunner {
v1 := engine.Group("/v1")
v1.Use(setAppNameInCtx)
v1.Use(s.apiMiddlewareWrapper())
v1.GET("/apps", s.handleAppList)
v1.POST("/apps", s.handleAppCreate)
{
apps := v1.Group("/apps/:app")
apps.Use(appWrap)
apps.Use(appNameCheck)
apps.GET("", s.handleAppGet)
apps.PATCH("", s.handleAppUpdate)
@@ -413,7 +414,7 @@ func (s *Server) bindHandlers(ctx context.Context) {
if s.nodeType != ServerTypeAPI {
runner := engine.Group("/r")
runner.Use(appWrap)
runner.Use(appNameCheck)
runner.Any("/:app", s.handleFunctionCall)
runner.Any("/:app/*route", s.handleFunctionCall)
}
@@ -433,6 +434,10 @@ func (s *Server) bindHandlers(ctx context.Context) {
})
}
func (s *Server) Datastore() models.Datastore {
return s.datastore
}
// returns the unescaped ?cursor and ?perPage values
// pageParams clamps 0 < ?perPage <= 100 and defaults to 30 if 0
// ignores parsing errors and falls back to defaults.

2
docs/developers/apps.md
View File

@@ -60,4 +60,4 @@ fn deploy hello
## Example app
See https://github.com/fnproject/fn/tree/master/examples/apps/hellos for a simple example.
See https://github.com/fnproject/fn/tree/master/examples/apps/hellos for a simple example. Just clone it and run `fn deploy --all` to see it in action.

8
examples/apps/hellos/README.md
View File

@@ -2,4 +2,10 @@
This shows you how to organize functions into a full application and deploy them easily with one command.
See [apps documentation](/docs/developers/apps.md) for details on how to use this.
```sh
fn deploy --all
```
Then surf to http://localhost:8080/r/helloapp
See [apps documentation](/docs/developers/apps.md) for more details on applications.

17
fnext/context.go Normal file
View File

@@ -0,0 +1,17 @@
package fnext
// good reading on this: https://twitter.com/sajma/status/757217773852487680
type contextKey string
// func (c contextKey) String() string {
// return "fnext context key " + string(c)
// }
// Keys for extensions to get things out of the context
var (
// MiddlewareControllerKey is a context key. It can be used in handlers with context.WithValue to
// access the MiddlewareContext.
MiddlewareControllerKey = contextKey("middleware_controller")
// AppNameKey
AppNameKey = contextKey("app_name")
)

19
fnext/listeners.go
View File

@@ -20,6 +20,25 @@ type AppListener interface {
BeforeAppDelete(ctx context.Context, app *models.App) error
// AfterAppDelete called after deleting App in the database
AfterAppDelete(ctx context.Context, app *models.App) error
// BeforeAppGet called right before getting an app
BeforeAppGet(ctx context.Context, appName string) error
// AfterAppGet called after getting app from database
AfterAppGet(ctx context.Context, app *models.App) error
// BeforeAppsList called right before getting a list of all user's apps. Modify the filter to adjust what gets returned.
BeforeAppsList(ctx context.Context, filter *models.AppFilter) error
// AfterAppsList called after deleting getting a list of user's apps. apps is the result after applying AppFilter.
AfterAppsList(ctx context.Context, apps []*models.App) error
// TODO: WHAT IF THESE WERE CHANGE TO WRAPPERS INSTEAD OF LISTENERS, SORT OF LIKE MIDDLEWARE, EG
// AppCreate(ctx, app, next func(ctx, app) or next.AppCreate(ctx, app)) <- where func is the InsertApp function (ie: the corresponding Datastore function)
// Then instead of two two functions and modifying objects in the params, they get modified and then passed on. Eg:
// AppCreate(ctx, app, next) (app *models.App, err error) {
// // do stuff before
// app.Name = app.Name + "-12345"
// app, err = next.AppCreate(ctx, app)
// // do stuff after if you want
// return app, err
// }
}
// CallListener enables callbacks around Call events

13
fnext/middleware.go
View File

@@ -5,12 +5,6 @@ import (
"net/http"
)
var (
// MiddlewareControllerKey is a context key. It can be used in handlers with context.WithValue to
// access the MiddlewareContext.
MiddlewareControllerKey = contextKey("middleware-controller")
)
// MiddlewareController allows a bit more flow control to the middleware, since we multiple paths a request can go down.
// 1) Could be routed towards the API
// 2) Could be routed towards a function
@@ -43,10 +37,3 @@ type MiddlewareFunc func(next http.Handler) http.Handler
func (m MiddlewareFunc) Handle(next http.Handler) http.Handler {
return m(next)
}
// good reading on this: https://twitter.com/sajma/status/757217773852487680
type contextKey string
// func (c contextKey) String() string {
// return "fnext context key " + string(c)
// }

3
fnext/setup.go
View File

@@ -37,4 +37,7 @@ type ExtServer interface {
AddRouteEndpoint(method, path string, handler ApiRouteHandler)
// AddRouteEndpoint adds an endpoints to /v1/apps/:app/routes/:route/x
AddRouteEndpointFunc(method, path string, handler func(w http.ResponseWriter, r *http.Request, app *models.App, route *models.Route))
// Datastore returns the Datastore Fn is using
Datastore() models.Datastore
}