From 2e12e2c7007e7dbfaf462d391aaf1fa0ac921004 Mon Sep 17 00:00:00 2001 From: Pedro Nasser Date: Wed, 12 Oct 2016 17:23:34 -0300 Subject: [PATCH] Fix input async tasks + tests (#137) --- api/models/route.go | 1 + api/runner/runner_test.go | 1 - api/server/helpers.go | 92 ---------------------------- api/server/runner.go | 31 ++++++---- api/server/runner_async_test.go | 105 ++++++++++++++++++++++++++++++++ api/server/runner_test.go | 10 ++- api/server/server.go | 29 ++++++++- api/server/server_test.go | 62 +++++++++++++++++++ glide.lock | 2 +- scripts/test.sh | 2 +- 10 files changed, 227 insertions(+), 108 deletions(-) delete mode 100644 api/server/helpers.go create mode 100644 api/server/runner_async_test.go diff --git a/api/models/route.go b/api/models/route.go index b6b776062..d090354f4 100644 --- a/api/models/route.go +++ b/api/models/route.go @@ -16,6 +16,7 @@ var ( ErrRoutesList = errors.New("Could not list routes from datastore") ErrRoutesNotFound = errors.New("Route not found") ErrRoutesMissingNew = errors.New("Missing new route") + ErrInvalidPayload = errors.New("Invalid payload") ) type Routes []*Route diff --git a/api/runner/runner_test.go b/api/runner/runner_test.go index 1c83c4463..1310484f5 100644 --- a/api/runner/runner_test.go +++ b/api/runner/runner_test.go @@ -12,7 +12,6 @@ import ( ) func TestRunnerHello(t *testing.T) { - t.Skip() runner, err := New(NewMetricLogger()) if err != nil { t.Fatalf("Test error during New() - %s", err) diff --git a/api/server/helpers.go b/api/server/helpers.go deleted file mode 100644 index dc3481cf4..000000000 --- a/api/server/helpers.go +++ /dev/null @@ -1,92 +0,0 @@ -package server - -import ( - "context" - "encoding/json" - "io" - "io/ioutil" - "net/http" - "net/http/httptest" - "testing" - - "github.com/gin-gonic/gin" - "github.com/iron-io/functions/api/models" - "github.com/iron-io/functions/api/runner" - "github.com/iron-io/runner/common" -) - -type appResponse struct { - Message string `json:"message"` - App *models.App `json:"app"` -} - -type appsResponse struct { - Message string `json:"message"` - Apps models.Apps `json:"apps"` -} - -type routeResponse struct { - Message string `json:"message"` - Route *models.Route `json:"route"` -} - -type routesResponse struct { - Message string `json:"message"` - Routes models.Routes `json:"routes"` -} - -type tasksResponse struct { - Message string `json:"message"` - Task models.Task `json:"tasksResponse"` -} - -func testRouter() *gin.Engine { - r := gin.Default() - ctx := context.Background() - r.Use(func(c *gin.Context) { - ctx, _ := common.LoggerWithFields(ctx, extractFields(c)) - c.Set("ctx", ctx) - c.Next() - }) - bindHandlers(r, - func(ctx *gin.Context) { - handleRequest(ctx, nil) - }, - func(ctx *gin.Context) {}) - return r -} - -func testRunner(t *testing.T) *runner.Runner { - r, err := runner.New(runner.NewMetricLogger()) - if err != nil { - t.Fatal("Test: failed to create new runner") - } - return r -} - -func routerRequest(t *testing.T, router *gin.Engine, method, path string, body io.Reader) (*http.Request, *httptest.ResponseRecorder) { - req, err := http.NewRequest(method, "http://localhost:8080"+path, body) - if err != nil { - t.Fatalf("Test: Could not create %s request to %s: %v", method, path, err) - } - - rec := httptest.NewRecorder() - router.ServeHTTP(rec, req) - - return req, rec -} - -func getErrorResponse(t *testing.T, rec *httptest.ResponseRecorder) models.Error { - respBody, err := ioutil.ReadAll(rec.Body) - if err != nil { - t.Error("Test: Expected not empty response body") - } - - var errResp models.Error - err = json.Unmarshal(respBody, &errResp) - if err != nil { - t.Error("Test: Expected response body to be a valid models.Error object") - } - - return errResp -} diff --git a/api/server/runner.go b/api/server/runner.go index b1595b820..900e9e6cb 100644 --- a/api/server/runner.go +++ b/api/server/runner.go @@ -30,6 +30,11 @@ func handleSpecial(c *gin.Context) { } } +func toEnvName(envtype, name string) string { + name = strings.ToUpper(strings.Replace(name, "-", "_", -1)) + return fmt.Sprintf("%s_%s", envtype, name) +} + func handleRequest(c *gin.Context, enqueue models.Enqueue) { if strings.HasPrefix(c.Request.URL.Path, "/v1") { c.Status(http.StatusNotFound) @@ -56,9 +61,7 @@ func handleRequest(c *gin.Context, enqueue models.Enqueue) { }() } else if c.Request.Method == "GET" { reqPayload := c.Request.URL.Query().Get("payload") - if len(reqPayload) > 0 { - payload = strings.NewReader(reqPayload) - } + payload = strings.NewReader(reqPayload) } appName := c.Param("app") @@ -114,22 +117,22 @@ func handleRequest(c *gin.Context, enqueue models.Enqueue) { // app config for k, v := range app.Config { - envVars["CONFIG_"+strings.ToUpper(k)] = v + envVars[toEnvName("CONFIG", k)] = v } // route config for k, v := range el.Config { - envVars["CONFIG_"+strings.ToUpper(k)] = v + envVars[toEnvName("CONFIG", k)] = v } // params for _, param := range params { - envVars["PARAM_"+strings.ToUpper(param.Key)] = param.Value + envVars[toEnvName("PARAM", param.Key)] = param.Value } // headers for header, value := range c.Request.Header { - envVars["HEADER_"+strings.ToUpper(header)] = strings.Join(value, " ") + envVars[toEnvName("HEADER", header)] = strings.Join(value, " ") } cfg := &runner.Config{ @@ -148,6 +151,14 @@ func handleRequest(c *gin.Context, enqueue models.Enqueue) { var result drivers.RunResult switch el.Type { case "async": + // Read payload + pl, err := ioutil.ReadAll(cfg.Stdin) + if err != nil { + log.WithError(err).Error(models.ErrInvalidPayload) + c.JSON(http.StatusBadRequest, simpleError(models.ErrInvalidPayload)) + return + } + // Create Task priority := int32(0) task := &models.Task{} @@ -155,6 +166,8 @@ func handleRequest(c *gin.Context, enqueue models.Enqueue) { task.ID = cfg.ID task.RouteName = cfg.AppName task.Priority = &priority + task.EnvVars = cfg.Env + task.Payload = string(pl) // Push to queue enqueue(task) log.Info("Added new task to queue") @@ -175,10 +188,6 @@ func handleRequest(c *gin.Context, enqueue models.Enqueue) { } } - if err != nil { - log.WithError(err).Error(models.ErrRunnerRunRoute) - c.JSON(http.StatusInternalServerError, simpleError(models.ErrRunnerRunRoute)) - } return } } diff --git a/api/server/runner_async_test.go b/api/server/runner_async_test.go new file mode 100644 index 000000000..a21d15a44 --- /dev/null +++ b/api/server/runner_async_test.go @@ -0,0 +1,105 @@ +package server + +import ( + "bytes" + "context" + "net/http" + "sync" + "testing" + + "github.com/gin-gonic/gin" + "github.com/iron-io/functions/api/datastore" + "github.com/iron-io/functions/api/models" + "github.com/iron-io/functions/api/mqs" + "github.com/iron-io/runner/common" +) + +func testRouterAsync(enqueueFunc models.Enqueue) *gin.Engine { + r := gin.New() + r.Use(gin.Logger()) + ctx := context.Background() + r.Use(func(c *gin.Context) { + ctx, _ := common.LoggerWithFields(ctx, extractFields(c)) + c.Set("ctx", ctx) + c.Next() + }) + bindHandlers(r, + func(ctx *gin.Context) { + handleRequest(ctx, enqueueFunc) + }, + func(ctx *gin.Context) {}) + return r +} + +func TestRouteRunnerAsyncExecution(t *testing.T) { + New(&datastore.Mock{ + FakeApps: []*models.App{ + {Name: "myapp", Config: map[string]string{"app": "true"}}, + }, + FakeRoutes: []*models.Route{ + {Type: "async", Path: "/myroute", AppName: "myapp", Image: "iron/hello", Config: map[string]string{"test": "true"}}, + {Type: "async", Path: "/myerror", AppName: "myapp", Image: "iron/error", Config: map[string]string{"test": "true"}}, + {Type: "async", Path: "/myroute/:param", AppName: "myapp", Image: "iron/hello", Config: map[string]string{"test": "true"}}, + }, + }, &mqs.Mock{}, testRunner(t)) + + for i, test := range []struct { + path string + body string + headers map[string][]string + expectedCode int + expectedEnv map[string]string + }{ + {"/r/myapp/myroute", ``, map[string][]string{}, http.StatusOK, map[string]string{"CONFIG_TEST": "true", "CONFIG_APP": "true"}}, + { + "/r/myapp/myroute/1", + ``, + map[string][]string{"X-Function": []string{"test"}}, + http.StatusOK, + map[string]string{ + "CONFIG_TEST": "true", + "CONFIG_APP": "true", + "PARAM_PARAM": "1", + "HEADER_X_FUNCTION": "test", + }, + }, + {"/r/myapp/myerror", ``, map[string][]string{}, http.StatusOK, map[string]string{"CONFIG_TEST": "true", "CONFIG_APP": "true"}}, + {"/r/myapp/myroute", `{ "name": "test" }`, map[string][]string{}, http.StatusOK, map[string]string{"CONFIG_TEST": "true", "CONFIG_APP": "true"}}, + } { + body := bytes.NewBuffer([]byte(test.body)) + + var wg sync.WaitGroup + + wg.Add(1) + router := testRouterAsync(func(task *models.Task) (*models.Task, error) { + if test.body != task.Payload { + t.Errorf("Test %d: Expected task Payload to be the same as the test body", i) + } + + if test.expectedEnv != nil { + for name, value := range test.expectedEnv { + if value != task.EnvVars[name] { + t.Errorf("Test %d: Expected header `%s` to be `%s` but was `%s`", + i, name, value, task.EnvVars[name]) + } + } + } + + wg.Done() + return task, nil + }) + + req, rec := newRouterRequest(t, "POST", test.path, body) + for name, value := range test.headers { + req.Header.Set(name, value[0]) + } + router.ServeHTTP(rec, req) + + if rec.Code != test.expectedCode { + t.Errorf("Test %d: Expected status code to be %d but was %d", + i, test.expectedCode, rec.Code) + } + + wg.Wait() + } +} diff --git a/api/server/runner_test.go b/api/server/runner_test.go index 19f348e91..81a96782c 100644 --- a/api/server/runner_test.go +++ b/api/server/runner_test.go @@ -10,8 +10,17 @@ import ( "github.com/iron-io/functions/api/datastore" "github.com/iron-io/functions/api/models" "github.com/iron-io/functions/api/mqs" + "github.com/iron-io/functions/api/runner" ) +func testRunner(t *testing.T) *runner.Runner { + r, err := runner.New(runner.NewMetricLogger()) + if err != nil { + t.Fatal("Test: failed to create new runner") + } + return r +} + func TestRouteRunnerGet(t *testing.T) { New(&datastore.Mock{ FakeApps: []*models.App{ @@ -88,7 +97,6 @@ func TestRouteRunnerPost(t *testing.T) { } func TestRouteRunnerExecution(t *testing.T) { - t.Skip() New(&datastore.Mock{ FakeApps: []*models.App{ {Name: "myapp", Config: models.Config{}}, diff --git a/api/server/server.go b/api/server/server.go index 0adb394a1..ef8e639fc 100644 --- a/api/server/server.go +++ b/api/server/server.go @@ -30,7 +30,7 @@ type Server struct { func New(ds models.Datastore, mq models.MessageQueue, r *runner.Runner) *Server { Api = &Server{ - Router: gin.Default(), + Router: gin.New(), Datastore: ds, MQ: mq, Runner: r, @@ -150,6 +150,8 @@ func (s *Server) Run(ctx context.Context) { } func bindHandlers(engine *gin.Engine, reqHandler func(ginC *gin.Context), taskHandler func(ginC *gin.Context)) { + engine.Use(gin.Logger()) + engine.GET("/", handlePing) engine.GET("/version", handleVersion) @@ -185,3 +187,28 @@ func bindHandlers(engine *gin.Engine, reqHandler func(ginC *gin.Context), taskHa func simpleError(err error) *models.Error { return &models.Error{&models.ErrorBody{Message: err.Error()}} } + +type appResponse struct { + Message string `json:"message"` + App *models.App `json:"app"` +} + +type appsResponse struct { + Message string `json:"message"` + Apps models.Apps `json:"apps"` +} + +type routeResponse struct { + Message string `json:"message"` + Route *models.Route `json:"route"` +} + +type routesResponse struct { + Message string `json:"message"` + Routes models.Routes `json:"routes"` +} + +type tasksResponse struct { + Message string `json:"message"` + Task models.Task `json:"tasksResponse"` +} diff --git a/api/server/server_test.go b/api/server/server_test.go index 580ae06f3..e821db2dd 100644 --- a/api/server/server_test.go +++ b/api/server/server_test.go @@ -2,17 +2,79 @@ package server import ( "bytes" + "context" + "encoding/json" + "io" + "io/ioutil" "net/http" + "net/http/httptest" "os" "testing" + "github.com/gin-gonic/gin" "github.com/iron-io/functions/api/datastore" "github.com/iron-io/functions/api/models" "github.com/iron-io/functions/api/mqs" + "github.com/iron-io/runner/common" ) var tmpBolt = "/tmp/func_test_bolt.db" +func testRouter() *gin.Engine { + r := gin.New() + r.Use(gin.Logger()) + ctx := context.Background() + r.Use(func(c *gin.Context) { + ctx, _ := common.LoggerWithFields(ctx, extractFields(c)) + c.Set("ctx", ctx) + c.Next() + }) + bindHandlers(r, + func(ctx *gin.Context) { + handleRequest(ctx, nil) + }, + func(ctx *gin.Context) {}) + return r +} + +func routerRequest(t *testing.T, router *gin.Engine, method, path string, body io.Reader) (*http.Request, *httptest.ResponseRecorder) { + req, err := http.NewRequest(method, "http://localhost:8080"+path, body) + if err != nil { + t.Fatalf("Test: Could not create %s request to %s: %v", method, path, err) + } + + rec := httptest.NewRecorder() + router.ServeHTTP(rec, req) + + return req, rec +} + +func newRouterRequest(t *testing.T, method, path string, body io.Reader) (*http.Request, *httptest.ResponseRecorder) { + req, err := http.NewRequest(method, "http://localhost:8080"+path, body) + if err != nil { + t.Fatalf("Test: Could not create %s request to %s: %v", method, path, err) + } + + rec := httptest.NewRecorder() + + return req, rec +} + +func getErrorResponse(t *testing.T, rec *httptest.ResponseRecorder) models.Error { + respBody, err := ioutil.ReadAll(rec.Body) + if err != nil { + t.Error("Test: Expected not empty response body") + } + + var errResp models.Error + err = json.Unmarshal(respBody, &errResp) + if err != nil { + t.Error("Test: Expected response body to be a valid models.Error object") + } + + return errResp +} + func prepareBolt(t *testing.T) (models.Datastore, func()) { ds, err := datastore.New("bolt://" + tmpBolt) if err != nil { diff --git a/glide.lock b/glide.lock index f3f4d1f55..f6a52362f 100644 --- a/glide.lock +++ b/glide.lock @@ -215,4 +215,4 @@ imports: - internal/scram - name: gopkg.in/yaml.v2 version: e4d366fc3c7938e2958e662b4258c7a89e1f0e3e -testImports: [] +testImports: [] \ No newline at end of file diff --git a/scripts/test.sh b/scripts/test.sh index 4593b2bd9..edb59eeab 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -1,3 +1,3 @@ #!/bin/bash -docker run -ti --privileged --rm -e LOG_LEVEL=debug -v /var/run/docker.sock:/var/run/docker.sock -v "$PWD":/go/src/github.com/iron-io/functions -w /go/src/github.com/iron-io/functions iron/go:dev go test -v $(glide nv | grep -v examples) \ No newline at end of file +docker run -ti --privileged --rm -e GIN_MODE=$GIN_MODE -e LOG_LEVEL=debug -v /var/run/docker.sock:/var/run/docker.sock -v "$PWD":/go/src/github.com/iron-io/functions -w /go/src/github.com/iron-io/functions iron/go:dev go test -v $(glide nv | grep -v examples | grep -v tool) \ No newline at end of file