fn: Call extensions/overriding and more customization friendly docker driver (#1065)

In pure-runner and LB agent, service providers might want to set specific driver options.

For example, to add cpu-shares to functions, LB can add the information as extensions
to the Call and pass this via gRPC to runners. Runners then pick these extensions from
gRPC call and pass it to driver. Using a custom driver implementation, pure-runners can
process these extensions to modify docker.CreateContainerOptions.

To achieve this, LB agents can now be configured using a call overrider.

Pure-runners can be configured using a custom docker driver.

RunnerCall and Call interfaces both expose call extensions.

An example to demonstrate this is implemented in test/fn-system-tests/system_test.go
which registers a call overrider for LB agent as well as a simple custom docker driver.
In this example, LB agent adds a key-value to extensions and runners add this key-value
as an environment variable to the container.

test/fn-system-tests/exec_test.go

@@ -17,6 +17,7 @@ import (
 	sdkmodels "github.com/fnproject/fn_go/models"
 )
 
+// See fn-test-utils for json response
 func getEchoContent(respBytes []byte) (string, error) {
 
 	var respJs map[string]interface{}
@@ -39,6 +40,29 @@ func getEchoContent(respBytes []byte) (string, error) {
 	return echo, nil
 }
 
+// See fn-test-utils for json response
+func getConfigContent(key string, respBytes []byte) (string, error) {
+
+	var respJs map[string]interface{}
+
+	err := json.Unmarshal(respBytes, &respJs)
+	if err != nil {
+		return "", err
+	}
+
+	cfg, ok := respJs["config"].(map[string]interface{})
+	if !ok {
+		return "", errors.New("unexpected json: config map")
+	}
+
+	val, ok := cfg[key].(string)
+	if !ok {
+		return "", fmt.Errorf("unexpected json: %s string", key)
+	}
+
+	return val, nil
+}
+
 func TestCanExecuteFunction(t *testing.T) {
 	s := apiutils.SetupHarness()
 	s.GivenAppExists(t, &sdkmodels.App{Name: s.AppName})
@@ -79,6 +103,13 @@ func TestCanExecuteFunction(t *testing.T) {
 	if resp.StatusCode != http.StatusOK {
 		t.Fatalf("StatusCode check failed on %v", resp.StatusCode)
 	}
+
+	// Now let's check FN_CHEESE, since LB and runners have override/extension mechanism
+	// to insert FN_CHEESE into config
+	cheese, err := getConfigContent("FN_CHEESE", output.Bytes())
+	if err != nil || cheese != "Tete de Moine" {
+		t.Fatalf("getConfigContent/FN_CHEESE check failed (%v) on %v", err, output)
+	}
 }
 
 func TestCanExecuteBigOutput(t *testing.T) {

test/fn-system-tests/system_test.go

@@ -6,12 +6,17 @@ import (
 	"fmt"
 
 	"github.com/fnproject/fn/api/agent"
+	"github.com/fnproject/fn/api/agent/drivers"
 	"github.com/fnproject/fn/api/agent/hybrid"
 	"github.com/fnproject/fn/api/common"
+	"github.com/fnproject/fn/api/models"
 	pool "github.com/fnproject/fn/api/runnerpool"
 	"github.com/fnproject/fn/api/server"
 	_ "github.com/fnproject/fn/api/server/defaultexts"
 
+	// We need docker client here, since we have a custom driver that wraps generic
+	// docker driver.
+	"github.com/fsouza/go-dockerclient"
 	"github.com/sirupsen/logrus"
 
 	"io/ioutil"
@@ -219,12 +224,14 @@ func SetUpLBNode(ctx context.Context) (*server.Server, error) {
 	keys := []string{"fn_appname", "fn_path"}
 	pool.RegisterPlacerViews(keys)
 
-	agent, err := agent.NewLBAgent(agent.NewCachedDataAccess(cl), nodePool, placer)
+	// Create an LB Agent with a Call Overrider to intercept calls in GetCall(). Overrider in this example
+	// scrubs CPU/TmpFsSize and adds FN_CHEESE key/value into extensions.
+	lbAgent, err := agent.NewLBAgent(agent.NewCachedDataAccess(cl), nodePool, placer, agent.WithCallOverrider(LBCallOverrider))
 	if err != nil {
 		return nil, err
 	}
-	opts = append(opts, server.WithAgent(agent))
 
+	opts = append(opts, server.WithAgent(lbAgent))
 	return server.New(ctx, opts...), nil
 }
 
@@ -246,14 +253,30 @@ func SetUpPureRunnerNode(ctx context.Context, nodeNum int) (*server.Server, erro
 		return nil, err
 	}
 	grpcAddr := fmt.Sprintf(":%d", 9190+nodeNum)
-	cancelCtx, cancel := context.WithCancel(ctx)
 
-	prAgent, err := agent.NewPureRunner(cancel, grpcAddr, ds, "", "", "", nil)
+	// This is our Agent config, which we will use for both inner agent and docker.
+	cfg, err := agent.NewAgentConfig()
 	if err != nil {
 		return nil, err
 	}
-	opts = append(opts, server.WithAgent(prAgent), server.WithExtraCtx(cancelCtx))
 
+	// customer driver that overrides generic docker driver
+	drv := &customDriver{
+		drv: agent.NewDockerDriver(cfg),
+	}
+
+	// inner agent for pure-runners
+	innerAgent := agent.New(ds, agent.WithConfig(cfg), agent.WithDockerDriver(drv), agent.WithoutAsyncDequeue())
+
+	cancelCtx, cancel := context.WithCancel(ctx)
+
+	// now create pure-runner that wraps agent.
+	pureRunner, err := agent.NewPureRunner(cancel, grpcAddr, agent.PureRunnerWithAgent(innerAgent))
+	if err != nil {
+		return nil, err
+	}
+
+	opts = append(opts, server.WithAgent(pureRunner), server.WithExtraCtx(cancelCtx))
 	return server.New(ctx, opts...), nil
 }
 
@@ -326,3 +349,65 @@ func TestMain(m *testing.M) {
 	}
 	os.Exit(result)
 }
+
+// Memory Only LB Agent Call Option
+func LBCallOverrider(c *models.Call, exts map[string]string) (map[string]string, error) {
+
+	// Set TmpFsSize and CPU to unlimited. This means LB operates on Memory
+	// only. Operators/Service providers are expected to override this
+	// and apply their own filter to set/override CPU/TmpFsSize/Memory
+	// and Extension variables.
+	c.TmpFsSize = 0
+	c.CPUs = models.MilliCPUs(0)
+	delete(c.Config, "FN_CPUS")
+
+	if exts == nil {
+		exts = make(map[string]string)
+	}
+
+	// Add an FN_CHEESE extension to be intercepted and specially handled by Pure Runner customDriver below
+	exts["FN_CHEESE"] = "Tete de Moine"
+	return exts, nil
+}
+
+// An example Pure Runner docker driver. Using CreateCookie, it intercepts a generated cookie to
+// add an environment variable FN_CHEESE if it finds a FN_CHEESE extension.
+type customDriver struct {
+	drv drivers.Driver
+}
+
+// implements Driver
+func (d *customDriver) CreateCookie(ctx context.Context, task drivers.ContainerTask) (drivers.Cookie, error) {
+	cookie, err := d.drv.CreateCookie(ctx, task)
+	if err != nil {
+		return cookie, err
+	}
+
+	// if call extensions include 'foo', then let's add FN_CHEESE env vars, which should
+	// end up in Env/Config.
+	ext := task.Extensions()
+	cheese, ok := ext["FN_CHEESE"]
+	if ok {
+		// docker driver specific data
+		obj := cookie.ContainerOptions()
+		opts, ok := obj.(docker.CreateContainerOptions)
+		if !ok {
+			logrus.Fatal("Unexpected driver, should be docker")
+		}
+		opts.Config.Env = append(opts.Config.Env, "FN_CHEESE="+cheese)
+	}
+
+	return cookie, nil
+}
+
+// implements Driver
+func (d *customDriver) PrepareCookie(ctx context.Context, cookie drivers.Cookie) error {
+	return d.drv.PrepareCookie(ctx, cookie)
+}
+
+// implements Driver
+func (d *customDriver) Close() error {
+	return d.drv.Close()
+}
+
+var _ drivers.Driver = &customDriver{}