Hybrid plumby (#585)

* fix configuration of agent and server to be future proof and plumb in the hybrid client agent

* fixes up the tests, turns off /r/ on api nodes

* fix up defaults for runner nodes

* shove the runner async push code down into agent land to use client

* plumb up async-age

* return full call from async dequeue endpoint, since we're storing a whole
call in the MQ we don't need to worry about caching of app/route [for now]
* fast safe shutdown of dequeue looper in runner / tidying of agent
* nice errors for path not found against /r/, /v1/ or other path not found
* removed some stale TODO in agent
* mq backends are only loud mouths in debug mode now

* update tests

* Add caching to hybrid client

* Fix HTTP error handling in hybrid client.

The type switch was on the value rather than a pointer.

* Gofmt.

* Better caching with a nice caching wrapper

* Remove datastore cache which is now unused

* Don't need to manually wrap interface methods

* Go fmt

api/server/hybrid.go

@@ -8,7 +8,6 @@ import (
 	"github.com/fnproject/fn/api/common"
 	"github.com/fnproject/fn/api/models"
 	"github.com/gin-gonic/gin"
-	"github.com/go-openapi/strfmt"
 )
 
 func (s *Server) handleRunnerEnqueue(c *gin.Context) {
@@ -40,11 +39,12 @@ func (s *Server) handleRunnerEnqueue(c *gin.Context) {
 		return
 	}
 
+	// TODO once update call is hooked up, do this
 	// at this point, the message is on the queue and could be picked up by a
 	// runner and enter into 'running' state before we can insert it in the db as
 	// 'queued' state. we can ignore any error inserting into db here and Start
 	// will ensure the call exists in the db in 'running' state there.
-	s.Datastore.InsertCall(ctx, &call)
+	// s.Datastore.InsertCall(ctx, &call)
 
 	c.JSON(200, struct {
 		M string `json:"msg"`
@@ -55,25 +55,23 @@ func (s *Server) handleRunnerDequeue(c *gin.Context) {
 	ctx, cancel := context.WithTimeout(c.Request.Context(), 30*time.Second)
 	defer cancel()
 
-	// TODO finalize (return whole call?) and move
-	type m struct {
-		AppName string `json:"app_name"`
-		Path    string `json:"path"`
-	}
-	type resp struct {
-		M []m `json:"calls"`
+	var resp struct {
+		M []*models.Call `json:"calls"`
 	}
+	var m [1]*models.Call // avoid alloc
+	resp.M = m[:0]
 
 	// long poll until ctx expires / we find a message
 	var b common.Backoff
 	for {
-		msg, err := s.MQ.Reserve(ctx)
+		call, err := s.MQ.Reserve(ctx)
 		if err != nil {
 			handleErrorResponse(c, err)
 			return
 		}
-		if msg != nil {
-			c.JSON(200, resp{M: []m{{AppName: msg.AppName, Path: msg.Path}}})
+		if call != nil {
+			resp.M = append(resp.M, call)
+			c.JSON(200, resp)
 			return
 		}
 
@@ -81,26 +79,25 @@ func (s *Server) handleRunnerDequeue(c *gin.Context) {
 
 		select {
 		case <-ctx.Done():
-			c.JSON(200, resp{M: make([]m, 0)})
+			c.JSON(200, resp) // TODO assert this return `[]` & not 'nil'
 			return
-		default:
+		default: // poll until we find a cookie
 		}
 	}
 }
 
 func (s *Server) handleRunnerStart(c *gin.Context) {
-	var body struct {
-		AppName string `json:"app_name"`
-		CallID  string `json:"id"`
-	}
+	ctx := c.Request.Context()
 
-	// TODO just take a whole call here? maybe the runner wants to mark it as error?
-	err := c.BindJSON(&body)
+	var call models.Call
+	err := c.BindJSON(&call)
 	if err != nil {
 		handleErrorResponse(c, models.ErrInvalidJSON)
 		return
 	}
 
+	// TODO validate call?
+
 	// TODO hook up update. we really just want it to set status to running iff
 	// status=queued, but this must be in a txn in Update with behavior:
 	// queued->running
@@ -112,21 +109,21 @@ func (s *Server) handleRunnerStart(c *gin.Context) {
 	// there is nuance for running->error as in theory it could be the correct machine retrying
 	// and we risk not running a task [ever]. needs further thought, but marking as error will
 	// cover our tracks since if the db is down we can't run anything anyway (treat as such).
-	var call models.Call
-	call.AppName = body.AppName
-	call.ID = body.CallID
-	call.Status = "running"
-	call.StartedAt = strfmt.DateTime(time.Now())
+	// TODO do this client side and validate it here?
+	//call.Status = "running"
+	//call.StartedAt = strfmt.DateTime(time.Now())
 	//err := s.Datastore.UpdateCall(c.Request.Context(), &call)
 	//if err != nil {
 	//if err == InvalidStatusChange {
 	//// TODO we could either let UpdateCall handle setting to error or do it
 	//// here explicitly
 
-	//if err := s.MQ.Delete(&call); err != nil { // TODO change this to take some string(s), not a whole call
-	//logrus.WithFields(logrus.Fields{"id": call.Id}).WithError(err).Error("error deleting mq message")
-	//// just log this one, return error from update call
-	//}
+	// TODO change this to only delete message if the status change fails b/c it already ran
+	// after messaging semantics change
+	if err := s.MQ.Delete(ctx, &call); err != nil { // TODO change this to take some string(s), not a whole call
+		handleErrorResponse(c, err)
+		return
+	}
 	//}
 	//handleErrorResponse(c, err)
 	//return
@@ -166,13 +163,14 @@ func (s *Server) handleRunnerFinish(c *gin.Context) {
 		// note: Not returning err here since the job could have already finished successfully.
 	}
 
+	// TODO open this up after we change messaging semantics.
 	// TODO we don't know whether a call is async or sync. we likely need an additional
 	// arg in params for a message id and can detect based on this. for now, delete messages
 	// for sync and async even though sync doesn't have any (ignore error)
-	if err := s.MQ.Delete(ctx, &call); err != nil { // TODO change this to take some string(s), not a whole call
-		common.Logger(ctx).WithError(err).Error("error deleting mq msg")
-		// note: Not returning err here since the job could have already finished successfully.
-	}
+	//if err := s.MQ.Delete(ctx, &call); err != nil { // TODO change this to take some string(s), not a whole call
+	//common.Logger(ctx).WithError(err).Error("error deleting mq msg")
+	//// note: Not returning err here since the job could have already finished successfully.
+	//}
 
 	c.JSON(200, struct {
 		M string `json:"msg"`

api/server/runner.go

@@ -15,22 +15,11 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
-type runnerResponse struct {
-	RequestID string            `json:"request_id,omitempty"`
-	Error     *models.ErrorBody `json:"error,omitempty"`
-}
-
 // handleFunctionCall executes the function.
 // Requires the following in the context:
 // * "app_name"
 // * "path"
 func (s *Server) handleFunctionCall(c *gin.Context) {
-	// @treeder: Is this necessary? An app could have this prefix too. Leaving here for review.
-	// if strings.HasPrefix(c.Request.URL.Path, "/v1") {
-	// 	c.Status(http.StatusNotFound)
-	// 	return
-	// }
-
 	ctx := c.Request.Context()
 	var p string
 	r := ctx.Value(api.Path)
@@ -95,8 +84,8 @@ func (s *Server) serve(c *gin.Context, appName, path string) {
 		}
 		model.Payload = buf.String()
 
-		// TODO we should probably add this to the datastore too. consider the plumber!
-		_, err = s.MQ.Push(c.Request.Context(), model)
+		// TODO idk where to put this, but agent is all runner really has...
+		err = s.Agent.Enqueue(c.Request.Context(), model)
 		if err != nil {
 			handleErrorResponse(c, err)
 			return
@@ -106,24 +95,19 @@ func (s *Server) serve(c *gin.Context, appName, path string) {
 		return
 	}
 
-	// Don't serve sync requests from API nodes
-	if s.nodeType != ServerTypeAPI {
-		err = s.Agent.Submit(call)
-		if err != nil {
-			// NOTE if they cancel the request then it will stop the call (kind of cool),
-			// we could filter that error out here too as right now it yells a little
-			if err == models.ErrCallTimeoutServerBusy || err == models.ErrCallTimeout {
-				// TODO maneuver
-				// add this, since it means that start may not have been called [and it's relevant]
-				c.Writer.Header().Add("XXX-FXLB-WAIT", time.Now().Sub(time.Time(model.CreatedAt)).String())
-			}
-			// NOTE: if the task wrote the headers already then this will fail to write
-			// a 5xx (and log about it to us) -- that's fine (nice, even!)
-			handleErrorResponse(c, err)
-			return
+	err = s.Agent.Submit(call)
+	if err != nil {
+		// NOTE if they cancel the request then it will stop the call (kind of cool),
+		// we could filter that error out here too as right now it yells a little
+		if err == models.ErrCallTimeoutServerBusy || err == models.ErrCallTimeout {
+			// TODO maneuver
+			// add this, since it means that start may not have been called [and it's relevant]
+			c.Writer.Header().Add("XXX-FXLB-WAIT", time.Now().Sub(time.Time(model.CreatedAt)).String())
 		}
-	} else {
-		handleErrorResponse(c, models.ErrSyncCallNotSupported)
+		// NOTE: if the task wrote the headers already then this will fail to write
+		// a 5xx (and log about it to us) -- that's fine (nice, even!)
+		handleErrorResponse(c, err)
+		return
 	}
 
 	// TODO plumb FXLB-WAIT somehow (api?)

api/server/runner_test.go

@@ -27,7 +27,7 @@ func testRunner(t *testing.T, args ...interface{}) (agent.Agent, context.CancelF
 			mq = arg
 		}
 	}
-	r := agent.New(ds, ds, mq, agent.AgentTypeFull)
+	r := agent.New(agent.NewDirectDataAccess(ds, ds, mq))
 	return r, func() { r.Close() }
 }
 

api/server/server.go

@@ -4,18 +4,18 @@ import (
 	"bytes"
 	"context"
 	"encoding/base64"
-	"errors"
 	"fmt"
 	"net"
 	"net/http"
 	"os"
 	"path"
 	"strconv"
+	"strings"
 	"syscall"
 
 	"github.com/fnproject/fn/api/agent"
+	"github.com/fnproject/fn/api/agent/hybrid"
 	"github.com/fnproject/fn/api/datastore"
-	"github.com/fnproject/fn/api/datastore/cache"
 	"github.com/fnproject/fn/api/id"
 	"github.com/fnproject/fn/api/logs"
 	"github.com/fnproject/fn/api/models"
@@ -37,6 +37,7 @@ const (
 	EnvMQURL     = "FN_MQ_URL"
 	EnvDBURL     = "FN_DB_URL"
 	EnvLOGDBURL  = "FN_LOGSTORE_URL"
+	EnvRunnerURL = "FN_RUNNER_URL"
 	EnvNodeType  = "FN_NODE_TYPE"
 	EnvPort      = "FN_PORT" // be careful, Gin expects this variable to be "port"
 	EnvAPICORS   = "FN_API_CORS"
@@ -80,122 +81,182 @@ func nodeTypeFromString(value string) ServerNodeType {
 
 // NewFromEnv creates a new Functions server based on env vars.
 func NewFromEnv(ctx context.Context, opts ...ServerOption) *Server {
-
-	return NewFromURLs(ctx,
-		getEnv(EnvDBURL, fmt.Sprintf("sqlite3://%s/data/fn.db", currDir)),
-		getEnv(EnvMQURL, fmt.Sprintf("bolt://%s/data/fn.mq", currDir)),
-		getEnv(EnvLOGDBURL, ""),
-		nodeTypeFromString(getEnv(EnvNodeType, "")),
-		opts...,
-	)
+	var defaultDB, defaultMQ string
+	nodeType := nodeTypeFromString(getEnv(EnvNodeType, "")) // default to full
+	if nodeType != ServerTypeRunner {
+		// only want to activate these for full and api nodes
+		defaultDB = fmt.Sprintf("sqlite3://%s/data/fn.db", currDir)
+		defaultMQ = fmt.Sprintf("bolt://%s/data/fn.mq", currDir)
+	}
+	opts = append(opts, WithZipkin(getEnv(EnvZipkinURL, "")))
+	opts = append(opts, WithDBURL(getEnv(EnvDBURL, defaultDB)))
+	opts = append(opts, WithMQURL(getEnv(EnvMQURL, defaultMQ)))
+	opts = append(opts, WithLogURL(getEnv(EnvLOGDBURL, "")))
+	opts = append(opts, WithRunnerURL(getEnv(EnvRunnerURL, "")))
+	opts = append(opts, WithType(nodeType))
+	return New(ctx, opts...)
 }
 
-// Create a new server based on the string URLs for each service.
-// Sits in the middle of NewFromEnv and New
-func NewFromURLs(ctx context.Context, dbURL, mqURL, logstoreURL string, nodeType ServerNodeType, opts ...ServerOption) *Server {
-	ds, err := datastore.New(dbURL)
-	if err != nil {
-		logrus.WithError(err).Fatalln("Error initializing datastore.")
+func WithDBURL(dbURL string) ServerOption {
+	if dbURL != "" {
+		ds, err := datastore.New(dbURL)
+		if err != nil {
+			logrus.WithError(err).Fatalln("Error initializing datastore.")
+		}
+		return WithDatastore(ds)
 	}
+	return noop
+}
 
-	mq, err := mqs.New(mqURL)
-	if err != nil {
-		logrus.WithError(err).Fatal("Error initializing message queue.")
+func WithMQURL(mqURL string) ServerOption {
+	if mqURL != "" {
+		mq, err := mqs.New(mqURL)
+		if err != nil {
+			logrus.WithError(err).Fatal("Error initializing message queue.")
+		}
+		return WithMQ(mq)
 	}
+	return noop
+}
 
-	var logDB models.LogStore = ds
-	if ldb := logstoreURL; ldb != "" && ldb != dbURL {
-		logDB, err = logs.New(logstoreURL)
+func WithLogURL(logstoreURL string) ServerOption {
+	if ldb := logstoreURL; ldb != "" {
+		logDB, err := logs.New(logstoreURL)
 		if err != nil {
 			logrus.WithError(err).Fatal("Error initializing logs store.")
 		}
+		return WithLogstore(logDB)
 	}
-
-	return New(ctx, ds, mq, logDB, nodeType, opts...)
+	return noop
 }
 
-// New creates a new Functions server with the passed in datastore, message queue and API URL
-func New(ctx context.Context, ds models.Datastore, mq models.MessageQueue, ls models.LogStore, nodeType ServerNodeType, opts ...ServerOption) *Server {
-	setTracer()
-
-	var tp agent.AgentNodeType
-	switch nodeType {
-	case ServerTypeAPI:
-		tp = agent.AgentTypeAPI
-	case ServerTypeRunner:
-		tp = agent.AgentTypeRunner
-	default:
-		tp = agent.AgentTypeFull
+func WithRunnerURL(runnerURL string) ServerOption {
+	if runnerURL != "" {
+		cl, err := hybrid.NewClient(runnerURL)
+		if err != nil {
+			logrus.WithError(err).Fatal("Error initializing runner API client.")
+		}
+		return WithAgent(agent.New(agent.NewCachedDataAccess(cl)))
 	}
+	return noop
+}
 
+func noop(s *Server) {}
+
+func WithType(t ServerNodeType) ServerOption {
+	return func(s *Server) { s.nodeType = t }
+}
+
+func WithDatastore(ds models.Datastore) ServerOption {
+	return func(s *Server) { s.Datastore = ds }
+}
+
+func WithMQ(mq models.MessageQueue) ServerOption {
+	return func(s *Server) { s.MQ = mq }
+}
+
+func WithLogstore(ls models.LogStore) ServerOption {
+	return func(s *Server) { s.LogDB = ls }
+}
+
+func WithAgent(agent agent.Agent) ServerOption {
+	return func(s *Server) { s.Agent = agent }
+}
+
+// New creates a new Functions server with the opts given. For convenience, users may
+// prefer to use NewFromEnv but New is more flexible if needed.
+func New(ctx context.Context, opts ...ServerOption) *Server {
 	s := &Server{
-		Agent:     agent.New(cache.Wrap(ds), ls, mq, tp), // only add datastore caching to agent
-		Router:    gin.New(),
-		Datastore: ds,
-		MQ:        mq,
-		LogDB:     ls,
-		nodeType:  nodeType,
+		Router: gin.New(),
+		// Almost everything else is configured through opts (see NewFromEnv for ex.) or below
 	}
 
-	// NOTE: testServer() in tests doesn't use these
-	setMachineID()
-	s.Router.Use(loggerWrap, traceWrap, panicWrap)
-	optionalCorsWrap(s.Router)
-	s.bindHandlers(ctx)
-
 	for _, opt := range opts {
 		if opt == nil {
 			continue
 		}
 		opt(s)
 	}
+
+	if s.LogDB == nil { // TODO seems weird?
+		s.LogDB = s.Datastore
+	}
+
+	// TODO we maybe should use the agent.DirectDataAccess in the /runner endpoints server side?
+
+	switch s.nodeType {
+	case ServerTypeAPI:
+		s.Agent = nil
+	case ServerTypeRunner:
+		if s.Agent == nil {
+			logrus.Fatal("No agent started for a runner node, add FN_RUNNER_URL to configuration.")
+		}
+	default:
+		s.nodeType = ServerTypeFull
+		if s.Datastore == nil || s.LogDB == nil || s.MQ == nil {
+			logrus.Fatal("Full nodes must configure FN_DB_URL, FN_LOG_URL, FN_MQ_URL.")
+		}
+
+		// TODO force caller to use WithAgent option ?
+		// TODO for tests we don't want cache, really, if we force WithAgent this can be fixed. cache needs to be moved anyway so that runner nodes can use it...
+		s.Agent = agent.New(agent.NewCachedDataAccess(agent.NewDirectDataAccess(s.Datastore, s.LogDB, s.MQ)))
+	}
+
+	setMachineID()
+	s.Router.Use(loggerWrap, traceWrap, panicWrap) // TODO should be opts
+	optionalCorsWrap(s.Router)                     // TODO should be an opt
+	s.bindHandlers(ctx)
 	return s
 }
 
-func setTracer() {
-	var (
-		debugMode          = false
-		serviceName        = "fnserver"
-		serviceHostPort    = "localhost:8080" // meh
-		zipkinHTTPEndpoint = getEnv(EnvZipkinURL, "")
-		// ex: "http://zipkin:9411/api/v1/spans"
-	)
+// TODO this doesn't need to be an option necessarily since it's just setting
+// globals but it makes things uniform. change if you've a better idear
+func WithZipkin(zipkinURL string) ServerOption {
+	return func(s *Server) {
+		var (
+			debugMode          = false
+			serviceName        = "fnserver"
+			serviceHostPort    = "localhost:8080" // meh
+			zipkinHTTPEndpoint = zipkinURL
+			// ex: "http://zipkin:9411/api/v1/spans"
+		)
 
-	var collector zipkintracer.Collector
+		var collector zipkintracer.Collector
 
-	// custom Zipkin collector to send tracing spans to Prometheus
-	promCollector, promErr := NewPrometheusCollector()
-	if promErr != nil {
-		logrus.WithError(promErr).Fatalln("couldn't start Prometheus trace collector")
-	}
-
-	logger := zipkintracer.LoggerFunc(func(i ...interface{}) error { logrus.Error(i...); return nil })
-
-	if zipkinHTTPEndpoint != "" {
-		// Custom PrometheusCollector and Zipkin HTTPCollector
-		httpCollector, zipErr := zipkintracer.NewHTTPCollector(zipkinHTTPEndpoint, zipkintracer.HTTPLogger(logger))
-		if zipErr != nil {
-			logrus.WithError(zipErr).Fatalln("couldn't start Zipkin trace collector")
+		// custom Zipkin collector to send tracing spans to Prometheus
+		promCollector, promErr := NewPrometheusCollector()
+		if promErr != nil {
+			logrus.WithError(promErr).Fatalln("couldn't start Prometheus trace collector")
 		}
-		collector = zipkintracer.MultiCollector{httpCollector, promCollector}
-	} else {
-		// Custom PrometheusCollector only
-		collector = promCollector
+
+		logger := zipkintracer.LoggerFunc(func(i ...interface{}) error { logrus.Error(i...); return nil })
+
+		if zipkinHTTPEndpoint != "" {
+			// Custom PrometheusCollector and Zipkin HTTPCollector
+			httpCollector, zipErr := zipkintracer.NewHTTPCollector(zipkinHTTPEndpoint, zipkintracer.HTTPLogger(logger))
+			if zipErr != nil {
+				logrus.WithError(zipErr).Fatalln("couldn't start Zipkin trace collector")
+			}
+			collector = zipkintracer.MultiCollector{httpCollector, promCollector}
+		} else {
+			// Custom PrometheusCollector only
+			collector = promCollector
+		}
+
+		ziptracer, err := zipkintracer.NewTracer(zipkintracer.NewRecorder(collector, debugMode, serviceHostPort, serviceName),
+			zipkintracer.ClientServerSameSpan(true),
+			zipkintracer.TraceID128Bit(true),
+		)
+		if err != nil {
+			logrus.WithError(err).Fatalln("couldn't start tracer")
+		}
+
+		// wrap the Zipkin tracer in a FnTracer which will also send spans to Prometheus
+		fntracer := NewFnTracer(ziptracer)
+
+		opentracing.SetGlobalTracer(fntracer)
+		logrus.WithFields(logrus.Fields{"url": zipkinHTTPEndpoint}).Info("started tracer")
 	}
-
-	ziptracer, err := zipkintracer.NewTracer(zipkintracer.NewRecorder(collector, debugMode, serviceHostPort, serviceName),
-		zipkintracer.ClientServerSameSpan(true),
-		zipkintracer.TraceID128Bit(true),
-	)
-	if err != nil {
-		logrus.WithError(err).Fatalln("couldn't start tracer")
-	}
-
-	// wrap the Zipkin tracer in a FnTracer which will also send spans to Prometheus
-	fntracer := NewFnTracer(ziptracer)
-
-	opentracing.SetGlobalTracer(fntracer)
-	logrus.WithFields(logrus.Fields{"url": zipkinHTTPEndpoint}).Info("started tracer")
 }
 
 func setMachineID() {
@@ -284,7 +345,9 @@ func (s *Server) startGears(ctx context.Context, cancel context.CancelFunc) {
 		logrus.WithError(err).Error("server shutdown error")
 	}
 
-	s.Agent.Close() // after we stop taking requests, wait for all tasks to finish
+	if s.Agent != nil {
+		s.Agent.Close() // after we stop taking requests, wait for all tasks to finish
+	}
 }
 
 func (s *Server) bindHandlers(ctx context.Context) {
@@ -335,7 +398,7 @@ func (s *Server) bindHandlers(ctx context.Context) {
 		}
 	}
 
-	{
+	if s.nodeType != ServerTypeAPI {
 		runner := engine.Group("/r")
 		runner.Use(appWrap)
 		runner.Any("/:app", s.handleFunctionCall)
@@ -343,8 +406,17 @@ func (s *Server) bindHandlers(ctx context.Context) {
 	}
 
 	engine.NoRoute(func(c *gin.Context) {
-		logrus.Debugln("not found", c.Request.URL.Path)
-		c.JSON(http.StatusNotFound, simpleError(errors.New("Path not found")))
+		var err error
+		switch {
+		case s.nodeType == ServerTypeAPI && strings.HasPrefix(c.Request.URL.Path, "/r/"):
+			err = models.ErrInvokeNotSupported
+		case s.nodeType == ServerTypeRunner && strings.HasPrefix(c.Request.URL.Path, "/v1/"):
+			err = models.ErrAPINotSupported
+		default:
+			var e models.APIError = models.ErrPathNotFound
+			err = models.NewAPIError(e.Code(), fmt.Errorf("%v: %s", e.Error(), c.Request.URL.Path))
+		}
+		handleErrorResponse(c, err)
 	})
 }
 

api/server/server_test.go

@@ -24,23 +24,13 @@ import (
 var tmpDatastoreTests = "/tmp/func_test_datastore.db"
 
 func testServer(ds models.Datastore, mq models.MessageQueue, logDB models.LogStore, rnr agent.Agent, nodeType ServerNodeType) *Server {
-	ctx := context.Background()
-
-	s := &Server{
-		Agent:     rnr,
-		Router:    gin.New(),
-		Datastore: ds,
-		LogDB:     logDB,
-		MQ:        mq,
-		nodeType:  nodeType,
-	}
-
-	r := s.Router
-	r.Use(gin.Logger())
-
-	s.Router.Use(loggerWrap)
-	s.bindHandlers(ctx)
-	return s
+	return New(context.Background(),
+		WithDatastore(ds),
+		WithMQ(mq),
+		WithLogstore(logDB),
+		WithAgent(rnr),
+		WithType(nodeType),
+	)
 }
 
 func routerRequest(t *testing.T, router *gin.Engine, method, path string, body io.Reader) (*http.Request, *httptest.ResponseRecorder) {
@@ -181,14 +171,14 @@ func TestRunnerNode(t *testing.T) {
 		{"execute async route succeeds", "POST", "/r/myapp/myasyncroute", `{ "name": "Teste" }`, http.StatusAccepted, 1},
 
 		// All other API functions should not be available on runner nodes
-		{"create app not found", "POST", "/v1/apps", `{ "app": { "name": "myapp" } }`, http.StatusNotFound, 0},
-		{"list apps not found", "GET", "/v1/apps", ``, http.StatusNotFound, 0},
-		{"get app not found", "GET", "/v1/apps/myapp", ``, http.StatusNotFound, 0},
+		{"create app not found", "POST", "/v1/apps", `{ "app": { "name": "myapp" } }`, http.StatusBadRequest, 0},
+		{"list apps not found", "GET", "/v1/apps", ``, http.StatusBadRequest, 0},
+		{"get app not found", "GET", "/v1/apps/myapp", ``, http.StatusBadRequest, 0},
 
-		{"add route not found", "POST", "/v1/apps/myapp/routes", `{ "route": { "name": "myroute", "path": "/myroute", "image": "fnproject/hello", "type": "sync" } }`, http.StatusNotFound, 0},
-		{"get route not found", "GET", "/v1/apps/myapp/routes/myroute", ``, http.StatusNotFound, 0},
-		{"get all routes not found", "GET", "/v1/apps/myapp/routes", ``, http.StatusNotFound, 0},
-		{"delete app not found", "DELETE", "/v1/apps/myapp", ``, http.StatusNotFound, 0},
+		{"add route not found", "POST", "/v1/apps/myapp/routes", `{ "route": { "name": "myroute", "path": "/myroute", "image": "fnproject/hello", "type": "sync" } }`, http.StatusBadRequest, 0},
+		{"get route not found", "GET", "/v1/apps/myapp/routes/myroute", ``, http.StatusBadRequest, 0},
+		{"get all routes not found", "GET", "/v1/apps/myapp/routes", ``, http.StatusBadRequest, 0},
+		{"delete app not found", "DELETE", "/v1/apps/myapp", ``, http.StatusBadRequest, 0},
 	} {
 		_, rec := routerRequest(t, srv.Router, test.method, test.path, bytes.NewBuffer([]byte(test.body)))
 
@@ -231,12 +221,10 @@ func TestApiNode(t *testing.T) {
 		{"get myroute2", "GET", "/v1/apps/myapp/routes/myroute2", ``, http.StatusOK, 0},
 		{"get all routes", "GET", "/v1/apps/myapp/routes", ``, http.StatusOK, 0},
 
-		// Don't support calling sync
+		// Don't support calling sync or async
 		{"execute myroute", "POST", "/r/myapp/myroute", `{ "name": "Teste" }`, http.StatusBadRequest, 1},
 		{"execute myroute2", "POST", "/r/myapp/myroute2", `{ "name": "Teste" }`, http.StatusBadRequest, 2},
-
-		// Do support calling async
-		{"execute myasyncroute", "POST", "/r/myapp/myasyncroute", `{ "name": "Teste" }`, http.StatusAccepted, 1},
+		{"execute myasyncroute", "POST", "/r/myapp/myasyncroute", `{ "name": "Teste" }`, http.StatusBadRequest, 1},
 
 		{"get myroute2", "GET", "/v1/apps/myapp/routes/myroute2", ``, http.StatusOK, 2},
 		{"delete myroute", "DELETE", "/v1/apps/myapp/routes/myroute", ``, http.StatusOK, 1},