hybrid mergy (#581)

* so it begins

* add clarification to /dequeue, change response to list to future proof

* Specify that runner endpoints are also under /v1

* Add a flag to choose operation mode (node type).

This is specified using the `FN_NODE_TYPE` environment variable. The
default is the existing behaviour, where the server supports all
operations (full API plus asynchronous and synchronous runners).

The additional modes are:
* API - the full API is available, but no functions are executed by the
  node. Async calls are placed into a message queue, and synchronous
  calls are not supported (invoking them results in an API error).
* Runner - only the invocation/route API is present. Asynchronous and
  synchronous invocation requests are supported, but asynchronous
  requests are placed onto the message queue, so might be handled by
  another runner.

* Add agent type and checks on Submit

* Sketch of a factored out data access abstraction for api/runner agents

* Fix tests, adding node/agent types to constructors

* Add tests for full, API, and runner server modes.

* Added atomic UpdateCall to datastore

* adds in server side endpoints

* Made ServerNodeType public because tests use it

* Made ServerNodeType public because tests use it

* fix test build

* add hybrid runner client

pretty simple go api client that covers surface area needed for hybrid,
returning structs from models that the agent can use directly. not exactly
sure where to put this, so put it in `/clients/hybrid` but maybe we should
make `/api/runner/client` or something and shove it in there. want to get
integration tests set up and use the real endpoints next and then wrap this up
in the DataAccessLayer stuff.

* gracefully handles errors from fn
* handles backoff & retry on 500s
* will add to existing spans for debuggo action

* minor fixes

* meh

api/server/apps_test.go

@@ -56,7 +56,7 @@ func TestAppCreate(t *testing.T) {
 		{datastore.NewMock(), logs.NewMock(), "/v1/apps", `{ "app": { "name": "teste" } }`, http.StatusOK, nil},
 	} {
 		rnr, cancel := testRunner(t)
-		srv := testServer(test.mock, &mqs.Mock{}, test.logDB, rnr)
+		srv := testServer(test.mock, &mqs.Mock{}, test.logDB, rnr, ServerTypeFull)
 		router := srv.Router
 
 		body := bytes.NewBuffer([]byte(test.body))
@@ -103,7 +103,7 @@ func TestAppDelete(t *testing.T) {
 		), logs.NewMock(), "/v1/apps/myapp", "", http.StatusOK, nil},
 	} {
 		rnr, cancel := testRunner(t)
-		srv := testServer(test.ds, &mqs.Mock{}, test.logDB, rnr)
+		srv := testServer(test.ds, &mqs.Mock{}, test.logDB, rnr, ServerTypeFull)
 
 		_, rec := routerRequest(t, srv.Router, "DELETE", test.path, nil)
 
@@ -144,7 +144,7 @@ func TestAppList(t *testing.T) {
 		nil, // no calls
 	)
 	fnl := logs.NewMock()
-	srv := testServer(ds, &mqs.Mock{}, fnl, rnr)
+	srv := testServer(ds, &mqs.Mock{}, fnl, rnr, ServerTypeFull)
 
 	a1b := base64.RawURLEncoding.EncodeToString([]byte("myapp"))
 	a2b := base64.RawURLEncoding.EncodeToString([]byte("myapp2"))
@@ -209,7 +209,7 @@ func TestAppGet(t *testing.T) {
 	defer cancel()
 	ds := datastore.NewMock()
 	fnl := logs.NewMock()
-	srv := testServer(ds, &mqs.Mock{}, fnl, rnr)
+	srv := testServer(ds, &mqs.Mock{}, fnl, rnr, ServerTypeFull)
 
 	for i, test := range []struct {
 		path          string
@@ -271,7 +271,7 @@ func TestAppUpdate(t *testing.T) {
 		), logs.NewMock(), "/v1/apps/myapp", `{ "app": { "name": "othername" } }`, http.StatusConflict, nil},
 	} {
 		rnr, cancel := testRunner(t)
-		srv := testServer(test.mock, &mqs.Mock{}, test.logDB, rnr)
+		srv := testServer(test.mock, &mqs.Mock{}, test.logDB, rnr, ServerTypeFull)
 
 		body := bytes.NewBuffer([]byte(test.body))
 		_, rec := routerRequest(t, srv.Router, "PATCH", test.path, body)

api/server/calls_test.go

@@ -49,7 +49,7 @@ func TestCallGet(t *testing.T) {
 		[]*models.Call{call},
 	)
 	fnl := logs.NewMock()
-	srv := testServer(ds, &mqs.Mock{}, fnl, rnr)
+	srv := testServer(ds, &mqs.Mock{}, fnl, rnr, ServerTypeFull)
 
 	for i, test := range []struct {
 		path          string
@@ -124,7 +124,7 @@ func TestCallList(t *testing.T) {
 		[]*models.Call{call, &c2, &c3},
 	)
 	fnl := logs.NewMock()
-	srv := testServer(ds, &mqs.Mock{}, fnl, rnr)
+	srv := testServer(ds, &mqs.Mock{}, fnl, rnr, ServerTypeFull)
 
 	// add / sub 1 second b/c unix time will lop off millis and mess up our comparisons
 	rangeTest := fmt.Sprintf("from_time=%d&to_time=%d",

api/server/hybrid.go

@@ -0,0 +1,180 @@
+package server
+
+import (
+	"context"
+	"strings"
+	"time"
+
+	"github.com/fnproject/fn/api/common"
+	"github.com/fnproject/fn/api/models"
+	"github.com/gin-gonic/gin"
+	"github.com/go-openapi/strfmt"
+)
+
+func (s *Server) handleRunnerEnqueue(c *gin.Context) {
+	ctx := c.Request.Context()
+
+	// TODO make this a list & let Push take a list!
+	var call models.Call
+	err := c.BindJSON(&call)
+	if err != nil {
+		handleErrorResponse(c, models.ErrInvalidJSON)
+		return
+	}
+
+	// XXX (reed): validate the call struct
+
+	// TODO/NOTE: if this endpoint is called multiple times for the same call we
+	// need to figure out the behavior we want. as it stands, there will be N
+	// messages for 1 call which only clogs up the MQ with spurious messages
+	// (possibly useful if things get wedged, not the point), the task will still
+	// just run once by the first runner to set it to status=running. we may well
+	// want to push msg only if inserting the call fails, but then we have a call
+	// in queued state with no message (much harder to handle). having this
+	// endpoint be retry safe seems ideal and runners likely won't spam it, so current
+	// behavior is okay [but beware of implications].
+	call.Status = "queued"
+	_, err = s.MQ.Push(ctx, &call)
+	if err != nil {
+		handleErrorResponse(c, err)
+		return
+	}
+
+	// at this point, the message is on the queue and could be picked up by a
+	// runner and enter into 'running' state before we can insert it in the db as
+	// 'queued' state. we can ignore any error inserting into db here and Start
+	// will ensure the call exists in the db in 'running' state there.
+	s.Datastore.InsertCall(ctx, &call)
+
+	c.JSON(200, struct {
+		M string `json:"msg"`
+	}{M: "enqueued call"})
+}
+
+func (s *Server) handleRunnerDequeue(c *gin.Context) {
+	ctx, cancel := context.WithTimeout(c.Request.Context(), 30*time.Second)
+	defer cancel()
+
+	// TODO finalize (return whole call?) and move
+	type m struct {
+		AppName string `json:"app_name"`
+		Path    string `json:"path"`
+	}
+	type resp struct {
+		M []m `json:"calls"`
+	}
+
+	// long poll until ctx expires / we find a message
+	var b common.Backoff
+	for {
+		msg, err := s.MQ.Reserve(ctx)
+		if err != nil {
+			handleErrorResponse(c, err)
+			return
+		}
+		if msg != nil {
+			c.JSON(200, resp{M: []m{{AppName: msg.AppName, Path: msg.Path}}})
+			return
+		}
+
+		b.Sleep(ctx)
+
+		select {
+		case <-ctx.Done():
+			c.JSON(200, resp{M: make([]m, 0)})
+			return
+		default:
+		}
+	}
+}
+
+func (s *Server) handleRunnerStart(c *gin.Context) {
+	var body struct {
+		AppName string `json:"app_name"`
+		CallID  string `json:"id"`
+	}
+
+	// TODO just take a whole call here? maybe the runner wants to mark it as error?
+	err := c.BindJSON(&body)
+	if err != nil {
+		handleErrorResponse(c, models.ErrInvalidJSON)
+		return
+	}
+
+	// TODO hook up update. we really just want it to set status to running iff
+	// status=queued, but this must be in a txn in Update with behavior:
+	// queued->running
+	// running->error (returning error)
+	// error->error (returning error)
+	// success->success (returning error)
+	// timeout->timeout (returning error)
+	//
+	// there is nuance for running->error as in theory it could be the correct machine retrying
+	// and we risk not running a task [ever]. needs further thought, but marking as error will
+	// cover our tracks since if the db is down we can't run anything anyway (treat as such).
+	var call models.Call
+	call.AppName = body.AppName
+	call.ID = body.CallID
+	call.Status = "running"
+	call.StartedAt = strfmt.DateTime(time.Now())
+	//err := s.Datastore.UpdateCall(c.Request.Context(), &call)
+	//if err != nil {
+	//if err == InvalidStatusChange {
+	//// TODO we could either let UpdateCall handle setting to error or do it
+	//// here explicitly
+
+	//if err := s.MQ.Delete(&call); err != nil { // TODO change this to take some string(s), not a whole call
+	//logrus.WithFields(logrus.Fields{"id": call.Id}).WithError(err).Error("error deleting mq message")
+	//// just log this one, return error from update call
+	//}
+	//}
+	//handleErrorResponse(c, err)
+	//return
+	//}
+
+	c.JSON(200, struct {
+		M string `json:"msg"`
+	}{M: "slingshot: engage"})
+}
+
+func (s *Server) handleRunnerFinish(c *gin.Context) {
+	ctx := c.Request.Context()
+
+	var body struct {
+		Call models.Call `json:"call"`
+		Log  string      `json:"log"` // TODO use multipart so that we don't have to serialize/deserialize this? measure..
+	}
+	err := c.BindJSON(&body)
+	if err != nil {
+		handleErrorResponse(c, models.ErrInvalidJSON)
+		return
+	}
+
+	// TODO validate?
+	call := body.Call
+
+	// TODO this needs UpdateCall functionality to work for async and should only work if:
+	// running->error|timeout|success
+	// TODO all async will fail here :( all sync will work fine :) -- *feeling conflicted*
+	if err := s.Datastore.InsertCall(ctx, &call); err != nil {
+		common.Logger(ctx).WithError(err).Error("error inserting call into datastore")
+		// note: Not returning err here since the job could have already finished successfully.
+	}
+
+	if err := s.LogDB.InsertLog(ctx, call.AppName, call.ID, strings.NewReader(body.Log)); err != nil {
+		common.Logger(ctx).WithError(err).Error("error uploading log")
+		// note: Not returning err here since the job could have already finished successfully.
+	}
+
+	// TODO we don't know whether a call is async or sync. we likely need an additional
+	// arg in params for a message id and can detect based on this. for now, delete messages
+	// for sync and async even though sync doesn't have any (ignore error)
+	if err := s.MQ.Delete(ctx, &call); err != nil { // TODO change this to take some string(s), not a whole call
+		common.Logger(ctx).WithError(err).Error("error deleting mq msg")
+		// note: Not returning err here since the job could have already finished successfully.
+	}
+
+	c.JSON(200, struct {
+		M string `json:"msg"`
+	}{M: "good night, sweet prince"})
+}

api/server/middleware_test.go

@@ -85,7 +85,7 @@ func TestRootMiddleware(t *testing.T) {
 	defer cancelrnr()
 
 	fnl := logs.NewMock()
-	srv := testServer(ds, &mqs.Mock{}, fnl, rnr)
+	srv := testServer(ds, &mqs.Mock{}, fnl, rnr, ServerTypeFull)
 	srv.AddRootMiddlewareFunc(func(next http.Handler) http.Handler {
 		// this one will override a call to the API based on a header
 		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {

api/server/routes_test.go

@@ -26,7 +26,7 @@ type routeTestCase struct {
 
 func (test *routeTestCase) run(t *testing.T, i int, buf *bytes.Buffer) {
 	rnr, cancel := testRunner(t)
-	srv := testServer(test.ds, &mqs.Mock{}, test.logDB, rnr)
+	srv := testServer(test.ds, &mqs.Mock{}, test.logDB, rnr, ServerTypeFull)
 
 	body := bytes.NewBuffer([]byte(test.body))
 	_, rec := routerRequest(t, srv.Router, test.method, test.path, body)
@@ -123,7 +123,7 @@ func TestRouteDelete(t *testing.T) {
 		{datastore.NewMockInit(apps, routes, nil), logs.NewMock(), "/v1/apps/a/routes/myroute", "", http.StatusOK, nil},
 	} {
 		rnr, cancel := testRunner(t)
-		srv := testServer(test.ds, &mqs.Mock{}, test.logDB, rnr)
+		srv := testServer(test.ds, &mqs.Mock{}, test.logDB, rnr, ServerTypeFull)
 		_, rec := routerRequest(t, srv.Router, "DELETE", test.path, nil)
 
 		if rec.Code != test.expectedCode {
@@ -178,7 +178,7 @@ func TestRouteList(t *testing.T) {
 	r2b := base64.RawURLEncoding.EncodeToString([]byte("/myroute1"))
 	r3b := base64.RawURLEncoding.EncodeToString([]byte("/myroute2"))
 
-	srv := testServer(ds, &mqs.Mock{}, fnl, rnr)
+	srv := testServer(ds, &mqs.Mock{}, fnl, rnr, ServerTypeFull)
 
 	for i, test := range []struct {
 		path string
@@ -242,7 +242,7 @@ func TestRouteGet(t *testing.T) {
 	ds := datastore.NewMock()
 	fnl := logs.NewMock()
 
-	srv := testServer(ds, &mqs.Mock{}, fnl, rnr)
+	srv := testServer(ds, &mqs.Mock{}, fnl, rnr, ServerTypeFull)
 
 	for i, test := range []struct {
 		path          string

api/server/runner.go

@@ -106,19 +106,24 @@ func (s *Server) serve(c *gin.Context, appName, path string) {
 		return
 	}
 
-	err = s.Agent.Submit(call)
-	if err != nil {
-		// NOTE if they cancel the request then it will stop the call (kind of cool),
-		// we could filter that error out here too as right now it yells a little
-		if err == models.ErrCallTimeoutServerBusy || err == models.ErrCallTimeout {
-			// TODO maneuver
-			// add this, since it means that start may not have been called [and it's relevant]
-			c.Writer.Header().Add("XXX-FXLB-WAIT", time.Now().Sub(time.Time(model.CreatedAt)).String())
+	// Don't serve sync requests from API nodes
+	if s.nodeType != ServerTypeAPI {
+		err = s.Agent.Submit(call)
+		if err != nil {
+			// NOTE if they cancel the request then it will stop the call (kind of cool),
+			// we could filter that error out here too as right now it yells a little
+			if err == models.ErrCallTimeoutServerBusy || err == models.ErrCallTimeout {
+				// TODO maneuver
+				// add this, since it means that start may not have been called [and it's relevant]
+				c.Writer.Header().Add("XXX-FXLB-WAIT", time.Now().Sub(time.Time(model.CreatedAt)).String())
+			}
+			// NOTE: if the task wrote the headers already then this will fail to write
+			// a 5xx (and log about it to us) -- that's fine (nice, even!)
+			handleErrorResponse(c, err)
+			return
 		}
-		// NOTE: if the task wrote the headers already then this will fail to write
-		// a 5xx (and log about it to us) -- that's fine (nice, even!)
-		handleErrorResponse(c, err)
-		return
+	} else {
+		handleErrorResponse(c, models.ErrSyncCallNotSupported)
 	}
 
 	// TODO plumb FXLB-WAIT somehow (api?)

api/server/runner_async_test.go

@@ -21,6 +21,7 @@ func testRouterAsync(ds models.Datastore, mq models.MessageQueue, rnr agent.Agen
 		Router:    gin.New(),
 		Datastore: ds,
 		MQ:        mq,
+		nodeType:  ServerTypeFull,
 	}
 
 	r := s.Router

api/server/runner_test.go

@@ -27,7 +27,7 @@ func testRunner(t *testing.T, args ...interface{}) (agent.Agent, context.CancelF
 			mq = arg
 		}
 	}
-	r := agent.New(ds, ds, mq)
+	r := agent.New(ds, ds, mq, agent.AgentTypeFull)
 	return r, func() { r.Close() }
 }
 
@@ -42,7 +42,7 @@ func TestRouteRunnerGet(t *testing.T) {
 	rnr, cancel := testRunner(t, ds)
 	defer cancel()
 	logDB := logs.NewMock()
-	srv := testServer(ds, &mqs.Mock{}, logDB, rnr)
+	srv := testServer(ds, &mqs.Mock{}, logDB, rnr, ServerTypeFull)
 
 	for i, test := range []struct {
 		path          string
@@ -87,7 +87,7 @@ func TestRouteRunnerPost(t *testing.T) {
 	defer cancel()
 
 	fnl := logs.NewMock()
-	srv := testServer(ds, &mqs.Mock{}, fnl, rnr)
+	srv := testServer(ds, &mqs.Mock{}, fnl, rnr, ServerTypeFull)
 
 	for i, test := range []struct {
 		path          string
@@ -141,7 +141,8 @@ func TestRouteRunnerExecution(t *testing.T) {
 	defer cancelrnr()
 
 	fnl := logs.NewMock()
-	srv := testServer(ds, &mqs.Mock{}, fnl, rnr)
+
+	srv := testServer(ds, &mqs.Mock{}, fnl, rnr, ServerTypeFull)
 
 	for i, test := range []struct {
 		path            string
@@ -211,7 +212,7 @@ func TestFailedEnqueue(t *testing.T) {
 	rnr, cancelrnr := testRunner(t, ds, mq)
 	defer cancelrnr()
 
-	srv := testServer(ds, mq, fnl, rnr)
+	srv := testServer(ds, mq, fnl, rnr, ServerTypeFull)
 	for i, test := range []struct {
 		path            string
 		body            string
@@ -252,7 +253,7 @@ func TestRouteRunnerTimeout(t *testing.T) {
 	defer cancelrnr()
 
 	fnl := logs.NewMock()
-	srv := testServer(ds, &mqs.Mock{}, fnl, rnr)
+	srv := testServer(ds, &mqs.Mock{}, fnl, rnr, ServerTypeFull)
 
 	for i, test := range []struct {
 		path            string

api/server/server.go

@@ -37,6 +37,7 @@ const (
 	EnvMQURL     = "FN_MQ_URL"
 	EnvDBURL     = "FN_DB_URL"
 	EnvLOGDBURL  = "FN_LOGSTORE_URL"
+	EnvNodeType  = "FN_NODE_TYPE"
 	EnvPort      = "FN_PORT" // be careful, Gin expects this variable to be "port"
 	EnvAPICORS   = "FN_API_CORS"
 	EnvZipkinURL = "FN_ZIPKIN_URL"
@@ -46,31 +47,52 @@ const (
 	DefaultPort     = 8080
 )
 
-type Server struct {
-	Router    *gin.Engine
-	Agent     agent.Agent
-	Datastore models.Datastore
-	MQ        models.MessageQueue
-	LogDB     models.LogStore
+type ServerNodeType int32
 
+const (
+	ServerTypeFull ServerNodeType = iota
+	ServerTypeAPI
+	ServerTypeRunner
+)
+
+type Server struct {
+	Router          *gin.Engine
+	Agent           agent.Agent
+	Datastore       models.Datastore
+	MQ              models.MessageQueue
+	LogDB           models.LogStore
+	nodeType        ServerNodeType
 	appListeners    []fnext.AppListener
 	rootMiddlewares []fnext.Middleware
 	apiMiddlewares  []fnext.Middleware
 }
 
+func nodeTypeFromString(value string) ServerNodeType {
+	switch value {
+	case "api":
+		return ServerTypeAPI
+	case "runner":
+		return ServerTypeRunner
+	default:
+		return ServerTypeFull
+	}
+}
+
 // NewFromEnv creates a new Functions server based on env vars.
 func NewFromEnv(ctx context.Context, opts ...ServerOption) *Server {
+
 	return NewFromURLs(ctx,
 		getEnv(EnvDBURL, fmt.Sprintf("sqlite3://%s/data/fn.db", currDir)),
 		getEnv(EnvMQURL, fmt.Sprintf("bolt://%s/data/fn.mq", currDir)),
 		getEnv(EnvLOGDBURL, ""),
+		nodeTypeFromString(getEnv(EnvNodeType, "")),
 		opts...,
 	)
 }
 
 // Create a new server based on the string URLs for each service.
 // Sits in the middle of NewFromEnv and New
-func NewFromURLs(ctx context.Context, dbURL, mqURL, logstoreURL string, opts ...ServerOption) *Server {
+func NewFromURLs(ctx context.Context, dbURL, mqURL, logstoreURL string, nodeType ServerNodeType, opts ...ServerOption) *Server {
 	ds, err := datastore.New(dbURL)
 	if err != nil {
 		logrus.WithError(err).Fatalln("Error initializing datastore.")
@@ -89,19 +111,30 @@ func NewFromURLs(ctx context.Context, dbURL, mqURL, logstoreURL string, opts ...
 		}
 	}
 
-	return New(ctx, ds, mq, logDB, opts...)
+	return New(ctx, ds, mq, logDB, nodeType, opts...)
 }
 
 // New creates a new Functions server with the passed in datastore, message queue and API URL
-func New(ctx context.Context, ds models.Datastore, mq models.MessageQueue, ls models.LogStore, opts ...ServerOption) *Server {
+func New(ctx context.Context, ds models.Datastore, mq models.MessageQueue, ls models.LogStore, nodeType ServerNodeType, opts ...ServerOption) *Server {
 	setTracer()
 
+	var tp agent.AgentNodeType
+	switch nodeType {
+	case ServerTypeAPI:
+		tp = agent.AgentTypeAPI
+	case ServerTypeRunner:
+		tp = agent.AgentTypeRunner
+	default:
+		tp = agent.AgentTypeFull
+	}
+
 	s := &Server{
-		Agent:     agent.New(cache.Wrap(ds), ls, mq), // only add datastore caching to agent
+		Agent:     agent.New(cache.Wrap(ds), ls, mq, tp), // only add datastore caching to agent
 		Router:    gin.New(),
 		Datastore: ds,
 		MQ:        mq,
 		LogDB:     ls,
+		nodeType:  nodeType,
 	}
 
 	// NOTE: testServer() in tests doesn't use these
@@ -265,7 +298,7 @@ func (s *Server) bindHandlers(ctx context.Context) {
 	engine.GET("/stats", s.handleStats)
 	engine.GET("/metrics", s.handlePrometheusMetrics)
 
-	{
+	if s.nodeType != ServerTypeRunner {
 		v1 := engine.Group("/v1")
 		v1.Use(s.apiMiddlewareWrapper())
 		v1.GET("/apps", s.handleAppList)
@@ -291,6 +324,15 @@ func (s *Server) bindHandlers(ctx context.Context) {
 			apps.GET("/calls/:call", s.handleCallGet)
 			apps.GET("/calls/:call/log", s.handleCallLogGet)
 		}
+
+		{
+			runner := v1.Group("/runner")
+			runner.PUT("/async", s.handleRunnerEnqueue)
+			runner.GET("/async", s.handleRunnerDequeue)
+
+			runner.POST("/start", s.handleRunnerStart)
+			runner.POST("/finish", s.handleRunnerFinish)
+		}
 	}
 
 	{

api/server/server_test.go

@@ -9,10 +9,13 @@ import (
 	"net/http"
 	"net/http/httptest"
 	"os"
+	"strings"
 	"testing"
 
 	"github.com/fnproject/fn/api/agent"
 	"github.com/fnproject/fn/api/datastore"
+	"github.com/fnproject/fn/api/id"
+	"github.com/fnproject/fn/api/logs"
 	"github.com/fnproject/fn/api/models"
 	"github.com/fnproject/fn/api/mqs"
 	"github.com/gin-gonic/gin"
@@ -20,7 +23,7 @@ import (
 
 var tmpDatastoreTests = "/tmp/func_test_datastore.db"
 
-func testServer(ds models.Datastore, mq models.MessageQueue, logDB models.LogStore, rnr agent.Agent) *Server {
+func testServer(ds models.Datastore, mq models.MessageQueue, logDB models.LogStore, rnr agent.Agent, nodeType ServerNodeType) *Server {
 	ctx := context.Background()
 
 	s := &Server{
@@ -29,6 +32,7 @@ func testServer(ds models.Datastore, mq models.MessageQueue, logDB models.LogSto
 		Datastore: ds,
 		LogDB:     logDB,
 		MQ:        mq,
+		nodeType:  nodeType,
 	}
 
 	r := s.Router
@@ -102,7 +106,7 @@ func TestFullStack(t *testing.T) {
 	rnr, rnrcancel := testRunner(t, ds)
 	defer rnrcancel()
 
-	srv := testServer(ds, &mqs.Mock{}, logDB, rnr)
+	srv := testServer(ds, &mqs.Mock{}, logDB, rnr, ServerTypeFull)
 
 	for _, test := range []struct {
 		name              string
@@ -139,3 +143,169 @@ func TestFullStack(t *testing.T) {
 		}
 	}
 }
+
+func TestRunnerNode(t *testing.T) {
+	ctx := context.Background()
+	buf := setLogBuffer()
+	ds, logDB, close := prepareDB(ctx, t)
+	defer close()
+
+	rnr, rnrcancel := testRunner(t, ds)
+	defer rnrcancel()
+
+	// Add route with an API server using the same DB
+	{
+		apiServer := testServer(ds, &mqs.Mock{}, logDB, rnr, ServerTypeAPI)
+		_, rec := routerRequest(t, apiServer.Router, "POST", "/v1/apps/myapp/routes", bytes.NewBuffer([]byte(`{ "route": { "name": "myroute", "path": "/myroute", "image": "fnproject/hello", "type": "sync" } }`)))
+		if rec.Code != http.StatusOK {
+			t.Errorf("Expected status code 200 when creating sync route, but got %d", rec.Code)
+		}
+		_, rec = routerRequest(t, apiServer.Router, "POST", "/v1/apps/myapp/routes", bytes.NewBuffer([]byte(`{ "route": { "name": "myasyncroute", "path": "/myasyncroute", "image": "fnproject/hello", "type": "async" } }`)))
+		if rec.Code != http.StatusOK {
+			t.Errorf("Expected status code 200 when creating async route, but got %d", rec.Code)
+		}
+	}
+
+	srv := testServer(ds, &mqs.Mock{}, logDB, rnr, ServerTypeRunner)
+
+	for _, test := range []struct {
+		name              string
+		method            string
+		path              string
+		body              string
+		expectedCode      int
+		expectedCacheSize int // TODO kill me
+	}{
+		// Support sync and async API calls
+		{"execute sync route succeeds", "POST", "/r/myapp/myroute", `{ "name": "Teste" }`, http.StatusOK, 1},
+		{"execute async route succeeds", "POST", "/r/myapp/myasyncroute", `{ "name": "Teste" }`, http.StatusAccepted, 1},
+
+		// All other API functions should not be available on runner nodes
+		{"create app not found", "POST", "/v1/apps", `{ "app": { "name": "myapp" } }`, http.StatusNotFound, 0},
+		{"list apps not found", "GET", "/v1/apps", ``, http.StatusNotFound, 0},
+		{"get app not found", "GET", "/v1/apps/myapp", ``, http.StatusNotFound, 0},
+
+		{"add route not found", "POST", "/v1/apps/myapp/routes", `{ "route": { "name": "myroute", "path": "/myroute", "image": "fnproject/hello", "type": "sync" } }`, http.StatusNotFound, 0},
+		{"get route not found", "GET", "/v1/apps/myapp/routes/myroute", ``, http.StatusNotFound, 0},
+		{"get all routes not found", "GET", "/v1/apps/myapp/routes", ``, http.StatusNotFound, 0},
+		{"delete app not found", "DELETE", "/v1/apps/myapp", ``, http.StatusNotFound, 0},
+	} {
+		_, rec := routerRequest(t, srv.Router, test.method, test.path, bytes.NewBuffer([]byte(test.body)))
+
+		if rec.Code != test.expectedCode {
+			t.Log(buf.String())
+			t.Errorf("Test \"%s\": Expected status code to be %d but was %d",
+				test.name, test.expectedCode, rec.Code)
+		}
+	}
+}
+
+func TestApiNode(t *testing.T) {
+	ctx := context.Background()
+	buf := setLogBuffer()
+	ds, logDB, close := prepareDB(ctx, t)
+	defer close()
+
+	rnr, rnrcancel := testRunner(t, ds)
+	defer rnrcancel()
+
+	srv := testServer(ds, &mqs.Mock{}, logDB, rnr, ServerTypeAPI)
+
+	for _, test := range []struct {
+		name              string
+		method            string
+		path              string
+		body              string
+		expectedCode      int
+		expectedCacheSize int // TODO kill me
+	}{
+		// All routes should be supported
+		{"create my app", "POST", "/v1/apps", `{ "app": { "name": "myapp" } }`, http.StatusOK, 0},
+		{"list apps", "GET", "/v1/apps", ``, http.StatusOK, 0},
+		{"get app", "GET", "/v1/apps/myapp", ``, http.StatusOK, 0},
+
+		{"add myroute", "POST", "/v1/apps/myapp/routes", `{ "route": { "name": "myroute", "path": "/myroute", "image": "fnproject/hello", "type": "sync" } }`, http.StatusOK, 0},
+		{"add myroute2", "POST", "/v1/apps/myapp/routes", `{ "route": { "name": "myroute2", "path": "/myroute2", "image": "fnproject/error", "type": "sync"  } }`, http.StatusOK, 0},
+		{"add myasyncroute", "POST", "/v1/apps/myapp/routes", `{ "route": { "name": "myasyncroute", "path": "/myasyncroute", "image": "fnproject/hello", "type": "async" } }`, http.StatusOK, 0},
+		{"get myroute", "GET", "/v1/apps/myapp/routes/myroute", ``, http.StatusOK, 0},
+		{"get myroute2", "GET", "/v1/apps/myapp/routes/myroute2", ``, http.StatusOK, 0},
+		{"get all routes", "GET", "/v1/apps/myapp/routes", ``, http.StatusOK, 0},
+
+		// Don't support calling sync
+		{"execute myroute", "POST", "/r/myapp/myroute", `{ "name": "Teste" }`, http.StatusBadRequest, 1},
+		{"execute myroute2", "POST", "/r/myapp/myroute2", `{ "name": "Teste" }`, http.StatusBadRequest, 2},
+
+		// Do support calling async
+		{"execute myasyncroute", "POST", "/r/myapp/myasyncroute", `{ "name": "Teste" }`, http.StatusAccepted, 1},
+
+		{"get myroute2", "GET", "/v1/apps/myapp/routes/myroute2", ``, http.StatusOK, 2},
+		{"delete myroute", "DELETE", "/v1/apps/myapp/routes/myroute", ``, http.StatusOK, 1},
+		{"delete myroute2", "DELETE", "/v1/apps/myapp/routes/myroute2", ``, http.StatusOK, 0},
+		{"delete app (success)", "DELETE", "/v1/apps/myapp", ``, http.StatusOK, 0},
+		{"get deleted app", "GET", "/v1/apps/myapp", ``, http.StatusNotFound, 0},
+		{"get deleted route on deleted app", "GET", "/v1/apps/myapp/routes/myroute", ``, http.StatusNotFound, 0},
+	} {
+		_, rec := routerRequest(t, srv.Router, test.method, test.path, bytes.NewBuffer([]byte(test.body)))
+		if rec.Code != test.expectedCode {
+			t.Log(buf.String())
+			t.Errorf("Test \"%s\": Expected status code to be %d but was %d",
+				test.name, test.expectedCode, rec.Code)
+		}
+	}
+}
+
+func TestHybridEndpoints(t *testing.T) {
+	buf := setLogBuffer()
+	ds := datastore.NewMockInit(
+		[]*models.App{{
+			Name: "myapp",
+		}},
+		[]*models.Route{{
+			AppName: "myapp",
+			Path:    "yodawg",
+		}}, nil,
+	)
+
+	logDB := logs.NewMock()
+
+	srv := testServer(ds, &mqs.Mock{}, logDB, nil /* TODO */, ServerTypeAPI)
+
+	newCallBody := func() string {
+		call := &models.Call{
+			ID:      id.New().String(),
+			AppName: "myapp",
+			Path:    "yodawg",
+			// TODO ?
+		}
+		var b bytes.Buffer
+		json.NewEncoder(&b).Encode(&call)
+		return b.String()
+	}
+
+	for _, test := range []struct {
+		name         string
+		method       string
+		path         string
+		body         string
+		expectedCode int
+	}{
+		// TODO change all these tests to just do an async task in normal order once plumbing is done
+
+		{"post async call", "PUT", "/v1/runner/async", newCallBody(), http.StatusOK},
+
+		// TODO this one only works if it's not the same as the first since update isn't hooked up
+		{"finish call", "POST", "/v1/runner/finish", newCallBody(), http.StatusOK},
+
+		// TODO these won't work until update works and the agent gets shut off
+		//{"get async call", "GET", "/v1/runner/async", "", http.StatusOK},
+		//{"start call", "POST", "/v1/runner/start", "TODO", http.StatusOK},
+	} {
+		_, rec := routerRequest(t, srv.Router, test.method, test.path, strings.NewReader(test.body))
+
+		if rec.Code != test.expectedCode {
+			t.Log(buf.String())
+			t.Errorf("Test \"%s\": Expected status code to be %d but was %d",
+				test.name, test.expectedCode, rec.Code)
+		}
+	}
+}