Hybrid plumby (#585)

* fix configuration of agent and server to be future proof and plumb in the hybrid client agent

* fixes up the tests, turns off /r/ on api nodes

* fix up defaults for runner nodes

* shove the runner async push code down into agent land to use client

* plumb up async-age

* return full call from async dequeue endpoint, since we're storing a whole
call in the MQ we don't need to worry about caching of app/route [for now]
* fast safe shutdown of dequeue looper in runner / tidying of agent
* nice errors for path not found against /r/, /v1/ or other path not found
* removed some stale TODO in agent
* mq backends are only loud mouths in debug mode now

* update tests

* Add caching to hybrid client

* Fix HTTP error handling in hybrid client.

The type switch was on the value rather than a pointer.

* Gofmt.

* Better caching with a nice caching wrapper

* Remove datastore cache which is now unused

* Don't need to manually wrap interface methods

* Go fmt

api/datastore/cache/cache.go

@@ -1,71 +0,0 @@
-package cache
-
-import (
-	"context"
-	"time"
-
-	"github.com/fnproject/fn/api/common/singleflight"
-	"github.com/fnproject/fn/api/models"
-	"github.com/patrickmn/go-cache"
-)
-
-type cacheDB struct {
-	models.Datastore
-
-	cache        *cache.Cache
-	singleflight singleflight.SingleFlight // singleflight assists Datastore
-}
-
-// Wrap implements models.Datastore by wrapping an existing datastore and
-// adding caching around certain methods. At present, GetApp and GetRoute add
-// caching.
-func Wrap(ds models.Datastore) models.Datastore {
-	return &cacheDB{
-		Datastore: ds,
-		cache:     cache.New(5*time.Second, 1*time.Minute), // TODO configurable from env
-	}
-}
-
-func (c *cacheDB) GetApp(ctx context.Context, appName string) (*models.App, error) {
-	key := appCacheKey(appName)
-	app, ok := c.cache.Get(key)
-	if ok {
-		return app.(*models.App), nil
-	}
-
-	resp, err := c.singleflight.Do(key,
-		func() (interface{}, error) { return c.Datastore.GetApp(ctx, appName) },
-	)
-	if err != nil {
-		return nil, err
-	}
-	app = resp.(*models.App)
-	c.cache.Set(key, app, cache.DefaultExpiration)
-	return app.(*models.App), nil
-}
-
-func (c *cacheDB) GetRoute(ctx context.Context, appName, path string) (*models.Route, error) {
-	key := routeCacheKey(appName, path)
-	route, ok := c.cache.Get(key)
-	if ok {
-		return route.(*models.Route), nil
-	}
-
-	resp, err := c.singleflight.Do(key,
-		func() (interface{}, error) { return c.Datastore.GetRoute(ctx, appName, path) },
-	)
-	if err != nil {
-		return nil, err
-	}
-	route = resp.(*models.Route)
-	c.cache.Set(key, route, cache.DefaultExpiration)
-	return route.(*models.Route), nil
-}
-
-func routeCacheKey(appname, path string) string {
-	return "r:" + appname + "\x00" + path
-}
-
-func appCacheKey(appname string) string {
-	return "a:" + appname
-}