From 5a13e2c0cc432efe052811e1b8d80dd30d98aef2 Mon Sep 17 00:00:00 2001 From: Pedro Nasser Date: Thu, 21 Jul 2016 21:18:02 -0300 Subject: [PATCH] improv api, datastore, postgres, runner --- api/models/app.go | 28 ++- api/models/route.go | 35 +++- api/runner/runner.go | 4 + api/server/datastore/bolt/bolt.go | 2 +- api/server/datastore/postgres/postgres.go | 228 ++++++++++++---------- api/server/router/apps_create.go | 6 + api/server/router/apps_get.go | 13 ++ api/server/router/apps_update.go | 18 +- api/server/router/routes_create.go | 6 + api/server/server.go | 2 +- 10 files changed, 215 insertions(+), 127 deletions(-) diff --git a/api/models/app.go b/api/models/app.go index 137182876..b203cc94a 100644 --- a/api/models/app.go +++ b/api/models/app.go @@ -2,20 +2,32 @@ package models import "errors" -type Apps []App +type Apps []*App var ( - ErrAppsCreate = errors.New("Could not create app") - ErrAppsUpdate = errors.New("Could not update app") - ErrAppsRemoving = errors.New("Could not remove app from datastore") - ErrAppsGet = errors.New("Could not get app from datastore") - ErrAppsList = errors.New("Could not list apps from datastore") - ErrAppsNotFound = errors.New("App not found") + ErrAppsCreate = errors.New("Could not create app") + ErrAppsUpdate = errors.New("Could not update app") + ErrAppsRemoving = errors.New("Could not remove app from datastore") + ErrAppsGet = errors.New("Could not get app from datastore") + ErrAppsList = errors.New("Could not list apps from datastore") + ErrAppsNotFound = errors.New("App not found") + ErrAppNothingToUpdate = errors.New("Nothing to update") ) type App struct { Name string `json:"name"` - Routes Routes `json:"routes"` + Routes Routes `json:"routes,omitempty"` +} + +var ( + ErrAppsValidationName = errors.New("Missing app name") +) + +func (a *App) Validate() error { + if a.Name == "" { + return ErrAppsValidationName + } + return nil } type AppFilter struct { diff --git a/api/models/route.go b/api/models/route.go index 6dbe01323..46b2b1020 100644 --- a/api/models/route.go +++ b/api/models/route.go @@ -14,16 +14,43 @@ var ( ErrRoutesNotFound = errors.New("Route not found") ) -type Routes []Route +type Routes []*Route type Route struct { Name string `json:"name"` AppName string `json:"appname"` Path string `json:"path"` Image string `json:"image"` - Type string `json:"type"` - ContainerPath string `json:"container_path"` - Headers http.Header `json:"headers"` + Type string `json:"type,omitempty"` + ContainerPath string `json:"container_path,omitempty"` + Headers http.Header `json:"headers,omitempty"` +} + +var ( + ErrRoutesValidationName = errors.New("Missing route Name") + ErrRoutesValidationImage = errors.New("Missing route Image") + ErrRoutesValidationAppName = errors.New("Missing route AppName") + ErrRoutesValidationPath = errors.New("Missing route Path") +) + +func (r *Route) Validate() error { + if r.Name == "" { + return ErrRoutesValidationName + } + + if r.Image == "" { + return ErrRoutesValidationImage + } + + if r.AppName == "" { + return ErrRoutesValidationAppName + } + + if r.Path == "" { + return ErrRoutesValidationPath + } + + return nil } type RouteFilter struct { diff --git a/api/runner/runner.go b/api/runner/runner.go index d7faa263e..c72741dfe 100644 --- a/api/runner/runner.go +++ b/api/runner/runner.go @@ -87,6 +87,10 @@ func DockerRun(route *models.Route, c *gin.Context) error { // log.WithField("payload", "---"+string(payload)+"---").Infoln("incoming request") // log.WithField("image", image).Infoln("About to run using this image") + for k, v := range route.Headers { + c.Header(k, v[0]) + } + // TODO: swap all this out with Titan's running via API cmd := exec.Command("docker", "run", "--rm", "-i", "-e", fmt.Sprintf("PAYLOAD=%v", string(payload)), image) stdout, err := cmd.StdoutPipe() diff --git a/api/server/datastore/bolt/bolt.go b/api/server/datastore/bolt/bolt.go index 700157bb0..d9ae94b38 100644 --- a/api/server/datastore/bolt/bolt.go +++ b/api/server/datastore/bolt/bolt.go @@ -33,7 +33,7 @@ func New(url *url.URL) (models.Datastore, error) { log.WithError(err).Errorln("Error on bolt.Open") return nil, err } - bucketPrefix := "fns-" + bucketPrefix := "funcs-" if url.Query()["bucket"] != nil { bucketPrefix = url.Query()["bucket"][0] } diff --git a/api/server/datastore/postgres/postgres.go b/api/server/datastore/postgres/postgres.go index 95bf6e384..b87637608 100644 --- a/api/server/datastore/postgres/postgres.go +++ b/api/server/datastore/postgres/postgres.go @@ -11,46 +11,31 @@ import ( _ "github.com/lib/pq" ) -const routesTableCreate = `CREATE TABLE IF NOT EXISTS routes ( - name character varying(256) NOT NULL PRIMARY KEY, +const routesTableCreate = ` +CREATE TABLE IF NOT EXISTS routes ( + name character varying(256) NOT NULL PRIMARY KEY, path text NOT NULL, - app_name character varying(256) NOT NULL, + app_name character varying(256) NOT NULL, image character varying(256) NOT NULL, - type character varying(256) NOT NULL, + type character varying(256) NOT NULL, container_path text NOT NULL, - headers text NOT NULL, + headers text NOT NULL );` const appsTableCreate = `CREATE TABLE IF NOT EXISTS apps ( - name character varying(256) NOT NULL PRIMARY KEY, + name character varying(256) NOT NULL PRIMARY KEY );` -const routeSelector = `SELECT path, app_name, image, type, container_path, headers FROM routes` +const routeSelector = `SELECT name, path, app_name, image, type, container_path, headers FROM routes` -// Tries to read in properties into `route` from `scanner`. Bubbles up errors. -// Capture sql.Row and sql.Rows type rowScanner interface { Scan(dest ...interface{}) error } -// Capture sql.DB and sql.Tx type rowQuerier interface { QueryRow(query string, args ...interface{}) *sql.Row } -func scanRoute(scanner rowScanner, route *models.Route) error { - err := scanner.Scan( - &route.Name, - &route.Path, - &route.AppName, - &route.Image, - &route.Type, - &route.ContainerPath, - &route.Headers, - ) - return err -} - type PostgresDatastore struct { db *sql.DB } @@ -84,6 +69,86 @@ func New(url *url.URL) (models.Datastore, error) { return pg, nil } +func (ds *PostgresDatastore) StoreApp(app *models.App) (*models.App, error) { + _, err := ds.db.Exec(` + INSERT INTO apps (name) + VALUES ($1) + RETURNING name; + `, app.Name) + + if err != nil { + return nil, err + } + + return app, nil +} + +func (ds *PostgresDatastore) RemoveApp(appName string) error { + _, err := ds.db.Exec(` + DELETE FROM apps + WHERE name = $1 + `, appName) + + if err != nil { + return err + } + + return nil +} + +func (ds *PostgresDatastore) GetApp(name string) (*models.App, error) { + row := ds.db.QueryRow("SELECT name FROM apps WHERE name=$1", name) + + var resName string + err := row.Scan(&resName) + + res := &models.App{ + Name: resName, + } + + if err != nil { + return nil, err + } + + return res, nil +} + +func scanApp(scanner rowScanner, app *models.App) error { + err := scanner.Scan( + &app.Name, + ) + + return err +} + +func (ds *PostgresDatastore) GetApps(filter *models.AppFilter) ([]*models.App, error) { + res := []*models.App{} + + rows, err := ds.db.Query(` + SELECT DISTINCT * + FROM apps`, + ) + if err != nil { + return nil, err + } + defer rows.Close() + + for rows.Next() { + var app models.App + err := scanApp(rows, &app) + + if err != nil { + return nil, err + } + res = append(res, &app) + } + + if err := rows.Err(); err != nil { + return nil, err + } + return res, nil +} + func (ds *PostgresDatastore) StoreRoute(route *models.Route) (*models.Route, error) { var headers string @@ -94,12 +159,19 @@ func (ds *PostgresDatastore) StoreRoute(route *models.Route) (*models.Route, err headers = string(hbyte) - err = ds.db.QueryRow(` + _, err = ds.db.Exec(` INSERT INTO routes ( name, app_name, path, image, type, container_path, headers ) - VALUES ($1, $2, $3, $4, $5, $6, $7)`, + VALUES ($1, $2, $3, $4, $5, $6, $7) + ON CONFLICT (name) DO UPDATE SET + path = $3, + image = $4, + type = $5, + container_path = $6, + headers = $7; + `, route.Name, route.AppName, route.Path, @@ -107,7 +179,8 @@ func (ds *PostgresDatastore) StoreRoute(route *models.Route) (*models.Route, err route.Type, route.ContainerPath, headers, - ).Scan(nil) + ) + if err != nil { return nil, err } @@ -115,17 +188,38 @@ func (ds *PostgresDatastore) StoreRoute(route *models.Route) (*models.Route, err } func (ds *PostgresDatastore) RemoveRoute(appName, routeName string) error { - err := ds.db.QueryRow(` + _, err := ds.db.Exec(` DELETE FROM routes - WHERE name = $1`, - routeName, - ).Scan(nil) + WHERE name = $1 + `, routeName) + if err != nil { return err } return nil } +func scanRoute(scanner rowScanner, route *models.Route) error { + var headerStr string + err := scanner.Scan( + &route.Name, + &route.Path, + &route.AppName, + &route.Image, + &route.Type, + &route.ContainerPath, + &headerStr, + ) + + if headerStr == "" { + return models.ErrRoutesNotFound + } + + err = json.Unmarshal([]byte(headerStr), &route.Headers) + + return err +} + func getRoute(qr rowQuerier, routeName string) (*models.Route, error) { var route models.Route @@ -144,7 +238,6 @@ func (ds *PostgresDatastore) GetRoute(appName, routeName string) (*models.Route, return getRoute(ds.db, routeName) } -// TODO: Add pagination func (ds *PostgresDatastore) GetRoutes(filter *models.RouteFilter) ([]*models.Route, error) { res := []*models.Route{} filterQuery := buildFilterQuery(filter) @@ -169,79 +262,6 @@ func (ds *PostgresDatastore) GetRoutes(filter *models.RouteFilter) ([]*models.Ro return res, nil } -func (ds *PostgresDatastore) GetApp(name string) (*models.App, error) { - row := ds.db.QueryRow("SELECT * FROM groups WHERE name=$1", name) - - var resName string - err := row.Scan(&resName) - - res := &models.App{ - Name: resName, - } - - if err != nil { - return nil, err - } - - return res, nil -} - -func (ds *PostgresDatastore) GetApps(filter *models.AppFilter) ([]*models.App, error) { - res := []*models.App{} - - rows, err := ds.db.Query(` - SELECT DISTINCT * - FROM apps - ORDER BY name`, - ) - if err != nil { - return nil, err - } - defer rows.Close() - - for rows.Next() { - var app models.App - err := rows.Scan(&app) - - if err != nil { - return nil, err - } - res = append(res, &app) - } - - if err := rows.Err(); err != nil { - return nil, err - } - return res, nil -} - -func (ds *PostgresDatastore) StoreApp(app *models.App) (*models.App, error) { - err := ds.db.QueryRow(` - INSERT INTO apps (name) - VALUES ($1) - `, app.Name).Scan(&app.Name) - // ON CONFLICT (name) DO UPDATE SET created_at = $2; - - if err != nil { - return nil, err - } - - return app, nil -} - -func (ds *PostgresDatastore) RemoveApp(appName string) error { - err := ds.db.QueryRow(` - DELETE FROM apps - WHERE name = $1 - `, appName).Scan(nil) - - if err != nil { - return err - } - - return nil -} - func buildFilterQuery(filter *models.RouteFilter) string { filterQuery := "" diff --git a/api/server/router/apps_create.go b/api/server/router/apps_create.go index 6d3f0196b..86b7d702b 100644 --- a/api/server/router/apps_create.go +++ b/api/server/router/apps_create.go @@ -21,6 +21,12 @@ func handleAppCreate(c *gin.Context) { return } + if err := app.Validate(); err != nil { + log.Error(err) + c.JSON(http.StatusInternalServerError, simpleError(err)) + return + } + app, err = store.StoreApp(app) if err != nil { log.WithError(err).Debug(models.ErrAppsCreate) diff --git a/api/server/router/apps_get.go b/api/server/router/apps_get.go index c077d9b57..47f5be75f 100644 --- a/api/server/router/apps_get.go +++ b/api/server/router/apps_get.go @@ -27,5 +27,18 @@ func handleAppGet(c *gin.Context) { return } + filter := &models.RouteFilter{ + AppName: appName, + } + + routes, err := store.GetRoutes(filter) + if err != nil { + log.WithError(err).Error(models.ErrRoutesGet) + c.JSON(http.StatusInternalServerError, simpleError(models.ErrRoutesGet)) + return + } + + app.Routes = routes + c.JSON(http.StatusOK, app) } diff --git a/api/server/router/apps_update.go b/api/server/router/apps_update.go index 7eb1dc4c0..dc7c3db7c 100644 --- a/api/server/router/apps_update.go +++ b/api/server/router/apps_update.go @@ -9,7 +9,7 @@ import ( ) func handleAppUpdate(c *gin.Context) { - store := c.MustGet("store").(models.Datastore) + // store := c.MustGet("store").(models.Datastore) log := c.MustGet("log").(logrus.FieldLogger) app := &models.App{} @@ -21,14 +21,14 @@ func handleAppUpdate(c *gin.Context) { return } - app.Name = c.Param("app") + // app.Name = c.Param("app") - app, err = store.StoreApp(app) - if err != nil { - log.WithError(err).Debug(models.ErrAppsUpdate) - c.JSON(http.StatusInternalServerError, simpleError(models.ErrAppsUpdate)) - return - } + // app, err = store.StoreApp(app) + // if err != nil { + // log.WithError(err).Debug(models.ErrAppsUpdate) + // c.JSON(http.StatusInternalServerError, simpleError(models.ErrAppsUpdate)) + // return + // } - c.JSON(http.StatusOK, app) + c.JSON(http.StatusOK, simpleError(models.ErrAppNothingToUpdate)) } diff --git a/api/server/router/routes_create.go b/api/server/router/routes_create.go index 5dcb57680..7448c26df 100644 --- a/api/server/router/routes_create.go +++ b/api/server/router/routes_create.go @@ -23,6 +23,12 @@ func handleRouteCreate(c *gin.Context) { route.AppName = c.Param("app") + if err := route.Validate(); err != nil { + log.Error(err) + c.JSON(http.StatusInternalServerError, simpleError(err)) + return + } + route, err = store.StoreRoute(route) if err != nil { log.WithError(err).Debug(models.ErrRoutesCreate) diff --git a/api/server/server.go b/api/server/server.go index 7615a9458..dcd75e02b 100644 --- a/api/server/server.go +++ b/api/server/server.go @@ -34,7 +34,7 @@ func extractFields(c *gin.Context) logrus.Fields { func (s *Server) Start() { if s.cfg.DatabaseURL == "" { cwd, _ := os.Getwd() - s.cfg.DatabaseURL = fmt.Sprintf("bolt://%s/bolt.db?bucket=fns", cwd) + s.cfg.DatabaseURL = fmt.Sprintf("bolt://%s/bolt.db?bucket=funcs", cwd) } ds, err := datastore.New(s.cfg.DatabaseURL)