fn: Call extensions/overriding and more customization friendly docker driver (#1065)

In pure-runner and LB agent, service providers might want to set specific driver options.

For example, to add cpu-shares to functions, LB can add the information as extensions
to the Call and pass this via gRPC to runners. Runners then pick these extensions from
gRPC call and pass it to driver. Using a custom driver implementation, pure-runners can
process these extensions to modify docker.CreateContainerOptions.

To achieve this, LB agents can now be configured using a call overrider.

Pure-runners can be configured using a custom docker driver.

RunnerCall and Call interfaces both expose call extensions.

An example to demonstrate this is implemented in test/fn-system-tests/system_test.go
which registers a call overrider for LB agent as well as a simple custom docker driver.
In this example, LB agent adds a key-value to extensions and runners add this key-value
as an environment variable to the container.

api/agent/lb_agent.go

@@ -17,34 +17,49 @@ import (
 	"github.com/fnproject/fn/fnext"
 )
 
+type CallOverrider func(*models.Call, map[string]string) (map[string]string, error)
+
 type lbAgent struct {
 	cfg           AgentConfig
 	da            DataAccess
 	callListeners []fnext.CallListener
 	rp            pool.RunnerPool
 	placer        pool.Placer
-
-	shutWg       *common.WaitGroup
-	callEndCount int64
+	callOverrider CallOverrider
+	shutWg        *common.WaitGroup
+	callEndCount  int64
 }
 
-func NewLBAgentConfig() (*AgentConfig, error) {
+type LBAgentOption func(*lbAgent) error
+
+func WithLBAgentConfig(cfg *AgentConfig) LBAgentOption {
+	return func(a *lbAgent) error {
+		a.cfg = *cfg
+		return nil
+	}
+}
+
+// LB agents can use this to register a CallOverrider to modify a Call and extensions
+func WithCallOverrider(fn CallOverrider) LBAgentOption {
+	return func(a *lbAgent) error {
+		if a.callOverrider != nil {
+			return errors.New("lb-agent call overriders already exists")
+		}
+		a.callOverrider = fn
+		return nil
+	}
+}
+
+// NewLBAgent creates an Agent that knows how to load-balance function calls
+// across a group of runner nodes.
+func NewLBAgent(da DataAccess, rp pool.RunnerPool, p pool.Placer, options ...LBAgentOption) (Agent, error) {
+
+	// Yes, LBAgent and Agent both use an AgentConfig.
 	cfg, err := NewAgentConfig()
 	if err != nil {
-		return cfg, err
+		logrus.WithError(err).Fatalf("error in lb-agent config cfg=%+v", cfg)
 	}
-	if cfg.MaxRequestSize == 0 {
-		return cfg, errors.New("lb-agent requires MaxRequestSize limit")
-	}
-	if cfg.MaxResponseSize == 0 {
-		return cfg, errors.New("lb-agent requires MaxResponseSize limit")
-	}
-	return cfg, nil
-}
 
-// NewLBAgentWithConfig creates an Agent configured with a supplied AgentConfig
-func NewLBAgentWithConfig(da DataAccess, rp pool.RunnerPool, p pool.Placer, cfg *AgentConfig) (Agent, error) {
-	logrus.Infof("lb-agent starting cfg=%+v", cfg)
 	a := &lbAgent{
 		cfg:    *cfg,
 		da:     da,
@@ -52,47 +67,52 @@ func NewLBAgentWithConfig(da DataAccess, rp pool.RunnerPool, p pool.Placer, cfg
 		placer: p,
 		shutWg: common.NewWaitGroup(),
 	}
+
+	// Allow overriding config
+	for _, option := range options {
+		err = option(a)
+		if err != nil {
+			logrus.WithError(err).Fatalf("error in lb-agent options")
+		}
+	}
+
+	logrus.Infof("lb-agent starting cfg=%+v", a.cfg)
 	return a, nil
 }
 
-// NewLBAgent creates an Agent that knows how to load-balance function calls
-// across a group of runner nodes.
-func NewLBAgent(da DataAccess, rp pool.RunnerPool, p pool.Placer) (Agent, error) {
-
-	// TODO: Move the constants above to Agent Config or an LB specific LBAgentConfig
-	cfg, err := NewLBAgentConfig()
-	if err != nil {
-		logrus.WithError(err).Fatalf("error in lb-agent config cfg=%+v", cfg)
-	}
-	return NewLBAgentWithConfig(da, rp, p, cfg)
-}
-
+// implements Agent
 func (a *lbAgent) AddCallListener(listener fnext.CallListener) {
 	a.callListeners = append(a.callListeners, listener)
 }
 
+// implements callTrigger
 func (a *lbAgent) fireBeforeCall(ctx context.Context, call *models.Call) error {
 	return fireBeforeCallFun(a.callListeners, ctx, call)
 }
 
+// implements callTrigger
 func (a *lbAgent) fireAfterCall(ctx context.Context, call *models.Call) error {
 	return fireAfterCallFun(a.callListeners, ctx, call)
 }
 
+// implements Agent
 // GetAppID is to get the match of an app name to its ID
 func (a *lbAgent) GetAppID(ctx context.Context, appName string) (string, error) {
 	return a.da.GetAppID(ctx, appName)
 }
 
+// implements Agent
 // GetAppByID is to get the app by ID
 func (a *lbAgent) GetAppByID(ctx context.Context, appID string) (*models.App, error) {
 	return a.da.GetAppByID(ctx, appID)
 }
 
+// implements Agent
 func (a *lbAgent) GetRoute(ctx context.Context, appID string, path string) (*models.Route, error) {
 	return a.da.GetRoute(ctx, appID, path)
 }
 
+// implements Agent
 func (a *lbAgent) GetCall(opts ...CallOpt) (Call, error) {
 	var c call
 
@@ -108,6 +128,16 @@ func (a *lbAgent) GetCall(opts ...CallOpt) (Call, error) {
 		return nil, errors.New("no model or request provided for call")
 	}
 
+	// If overrider is present, let's allow it to modify models.Call
+	// and call extensions
+	if a.callOverrider != nil {
+		ext, err := a.callOverrider(c.Call, c.extensions)
+		if err != nil {
+			return nil, err
+		}
+		c.extensions = ext
+	}
+
 	err := setMaxBodyLimit(&a.cfg, &c)
 	if err != nil {
 		return nil, err
@@ -120,10 +150,10 @@ func (a *lbAgent) GetCall(opts ...CallOpt) (Call, error) {
 	c.ct = a
 	c.stderr = &nullReadWriter{}
 	c.slotHashId = getSlotQueueKey(&c)
-
 	return &c, nil
 }
 
+// implements Agent
 func (a *lbAgent) Close() error {
 
 	// start closing the front gate first
@@ -140,6 +170,7 @@ func (a *lbAgent) Close() error {
 	return err
 }
 
+// implements Agent
 func (a *lbAgent) Submit(callI Call) error {
 	if !a.shutWg.AddSession(1) {
 		return models.ErrCallTimeoutServerBusy
@@ -229,6 +260,7 @@ func (a *lbAgent) setRequestBody(ctx context.Context, call *call) (*bytes.Buffer
 	}
 }
 
+// implements Agent
 func (a *lbAgent) Enqueue(context.Context, *models.Call) error {
 	logrus.Error("Enqueue not implemented")
 	return errors.New("Enqueue not implemented")
@@ -260,3 +292,6 @@ func (a *lbAgent) handleCallEnd(ctx context.Context, call *call, err error, isSt
 	handleStatsDequeue(ctx, err)
 	return transformTimeout(err, true)
 }
+
+var _ Agent = &lbAgent{}
+var _ callTrigger = &lbAgent{}