fn: non-blocking resource tracker and notification (#841)

* fn: non-blocking resource tracker and notification

For some types of errors, we might want to notify
the actual caller if the error is directly 1-1 tied
to that request. If hotLauncher is triggered with
signaller, then here we send a back communication
error notification channel. This is passed to
checkLaunch to send back synchronous responses
to the caller that initiated this hot container
launch.

This is useful if we want to run the agent in
quick fail mode, where instead of waiting for
CPU/Mem to become available, we prefer to fail
quick in order not to hold up the caller.
To support this, non-blocking resource tracker
option/functions are now available.

* fn: test env var rename tweak

* fn: fixup merge

* fn: rebase test fix

* fn: merge fixup

* fn: test tweak down to 70MB for 128MB total

* fn: refactor token creation and use broadcast regardless

* fn: nb description

* fn: bugfix

api/server/runner_test.go

@@ -146,6 +146,101 @@ func TestRouteRunnerPost(t *testing.T) {
 	}
 }
 
+func TestRouteRunnerFastFail(t *testing.T) {
+	buf := setLogBuffer()
+	isFailure := false
+
+	tweaker1 := envTweaker("FN_MAX_TOTAL_MEMORY_BYTES", "134217728") // 128MB
+	tweaker2 := envTweaker("FN_ENABLE_NB_RESOURCE_TRACKER", "yes")   // enable fast-fail (no wait on CPU/Mem)
+	defer tweaker1()
+	defer tweaker2()
+
+	// Log once after we are done, flow of events are important (hot/cold containers, idle timeout, etc.)
+	// for figuring out why things failed.
+	defer func() {
+		if isFailure {
+			t.Log(buf.String())
+		}
+	}()
+
+	rCfg := map[string]string{"ENABLE_HEADER": "yes", "ENABLE_FOOTER": "yes"} // enable container start/end header/footer
+	rImg := "fnproject/fn-test-utils"
+
+	app := &models.App{Name: "foo"}
+	app.SetDefaults()
+
+	ds := datastore.NewMockInit(
+		[]*models.App{app},
+		[]*models.Route{
+			{Path: "/json", AppID: app.ID, Image: rImg, Type: "sync", Format: "json", Memory: 70, Timeout: 30, IdleTimeout: 30, Config: rCfg},
+		},
+	)
+
+	rnr, cancelrnr := testRunner(t, ds)
+	defer cancelrnr()
+
+	srv := testServer(ds, &mqs.Mock{}, ds, rnr, ServerTypeFull)
+	ok := `{"sleepTime": 1000, "isDebug": true}`
+
+	results := make(chan error)
+
+	type tester struct {
+		path              string
+		body              string
+		method            string
+		expectedCode      int
+		expectedErrSubStr string
+	}
+
+	for idx, test := range []tester{
+		{"/r/foo/json/", ok, "GET", http.StatusOK, ""},
+		{"/r/foo/json/", ok, "GET", http.StatusOK, ""},
+		{"/r/foo/json/", ok, "GET", http.StatusOK, ""},
+		{"/r/foo/json/", ok, "GET", http.StatusOK, ""},
+	} {
+		go func(i int, test tester) {
+			body := strings.NewReader(test.body)
+			_, rec := routerRequest(t, srv.Router, test.method, test.path, body)
+			respBytes, _ := ioutil.ReadAll(rec.Body)
+			respBody := string(respBytes)
+			maxBody := len(respBody)
+			if maxBody > 1024 {
+				maxBody = 1024
+			}
+
+			if rec.Code != test.expectedCode {
+				results <- fmt.Errorf("Test %d: Expected status code to be %d but was %d. body: %s",
+					i, test.expectedCode, rec.Code, respBody[:maxBody])
+			} else if test.expectedErrSubStr != "" && !strings.Contains(respBody, test.expectedErrSubStr) {
+				results <- fmt.Errorf("Test %d: Expected response to include %s but got body: %s",
+					i, test.expectedErrSubStr, respBody[:maxBody])
+			} else {
+				results <- nil
+			}
+
+		}(idx, test)
+	}
+
+	totalSuccess := 0
+	totalFail := 0
+
+	// Scan for 4 test results
+	for i := 0; i < 4; i++ {
+		err := <-results
+		if err != nil {
+			t.Logf("Test %d: received: %s (this is probably OK)", i, err.Error())
+			totalFail++
+		} else {
+			totalSuccess++
+		}
+	}
+
+	if totalSuccess != 1 {
+		t.Errorf("Expected 1 success but got %d (fail: %d)", totalSuccess, totalFail)
+		isFailure = true
+	}
+}
+
 func TestRouteRunnerIOPipes(t *testing.T) {
 	buf := setLogBuffer()
 	isFailure := false