fn: lb and pure-runner with non-blocking agent (#989)

* fn: lb and pure-runner with non-blocking agent

*) Removed pure-runner capacity tracking code. This did
not play well with internal agent resource tracker.
*) In LB and runner gRPC comm, removed ACK. Now,
upon TryCall, pure-runner quickly proceeds to call
Submit. This is good since at this stage pure-runner
already has all relevant data to initiate the call.
*) Unless pure-runner emits a NACK, LB immediately
streams http body to runners.
*) For retriable requests added a CachedReader for
http.Request Body.
*) Idempotenty/retry is similar to previous code.
After initial success in Engament, after attempting
a TryCall, unless we receive NACK, we cannot retry
that call.
*) ch and naive places now wraps each TryExec with
a cancellable context to clean up gRPC contexts
quicker.

* fn: err for simpler one-time read GetBody approach

This allows for a more flexible approach since we let
users to define GetBody() to allow repetitive http body
read. In default LB case, LB executes a one-time io.ReadAll
and sets of GetBody, which is detected by RunnerCall.RequestBody().

* fn: additional check for non-nil req.body

* fn: attempt to override IO errors with ctx for TryExec

* fn: system-tests log dest

* fn: LB: EOF send handling

* fn: logging for partial IO

* fn: use buffer pool for IO storage in lb agent

* fn: pure runner should use chunks for data msgs

* fn: required config validations and pass APIErrors

* fn: additional tests and gRPC proto simplification

*) remove ACK/NACK messages as Finish message type works
OK for this purpose.
*) return resp in api tests for check for status code
*) empty body json test in api tests for lb & pure-runner

* fn: buffer adjustments

*) setRequestBody result handling correction
*) switch to bytes.Reader for read-only safety
*) io.EOF can be returned for non-nil Body in request.

* fn: clarify detection of 503 / Server Too Busy

test/fn-api-tests/exec_test.go

@@ -43,14 +43,14 @@ func CallAsync(t *testing.T, u url.URL, content io.Reader) string {
 
 func CallSync(t *testing.T, u url.URL, content io.Reader) string {
 	output := &bytes.Buffer{}
-	hdrs, err := CallFN(u.String(), content, output, "POST", []string{})
+	resp, err := CallFN(u.String(), content, output, "POST", []string{})
 	if err != nil {
 		t.Errorf("Got unexpected error: %v", err)
 	}
 
-	callId := hdrs.Get("FN_CALL_ID")
+	callId := resp.Header.Get("FN_CALL_ID")
 	if callId == "" {
-		t.Errorf("Assertion error.\n\tExpected call id header in response, got: %v", hdrs)
+		t.Errorf("Assertion error.\n\tExpected call id header in response, got: %v", resp.Header)
 	}
 
 	t.Logf("Sync execution call ID: %v", callId)
@@ -221,13 +221,13 @@ func TestCanCauseTimeout(t *testing.T) {
 	}{Seconds: 11})
 	output := &bytes.Buffer{}
 
-	headers, _ := CallFN(u.String(), content, output, "POST", []string{})
+	resp, _ := CallFN(u.String(), content, output, "POST", []string{})
 
 	if !strings.Contains(output.String(), "Timed out") {
 		t.Errorf("Must fail because of timeout, but got error message: %v", output.String())
 	}
 	cfg := &call.GetAppsAppCallsCallParams{
-		Call:    headers.Get("FN_CALL_ID"),
+		Call:    resp.Header.Get("FN_CALL_ID"),
 		App:     s.AppName,
 		Context: s.Context,
 	}

test/fn-api-tests/formats_test.go

@@ -41,7 +41,7 @@ func TestFnJSONFormats(t *testing.T) {
 	})
 	content := bytes.NewBuffer(b)
 	output := &bytes.Buffer{}
-	headers, err := CallFN(u.String(), content, output, "POST", []string{})
+	resp, err := CallFN(u.String(), content, output, "POST", []string{})
 	if err != nil {
 		t.Errorf("Got unexpected error: %v", err)
 	}
@@ -56,7 +56,7 @@ func TestFnJSONFormats(t *testing.T) {
 	expectedHeaderNames := []string{"Content-Type", "Content-Length"}
 	expectedHeaderValues := []string{"application/json; charset=utf-8", strconv.Itoa(output.Len())}
 	for i, name := range expectedHeaderNames {
-		actual := headers.Get(name)
+		actual := resp.Header.Get(name)
 		expected := expectedHeaderValues[i]
 		if !strings.Contains(expected, actual) {
 			t.Errorf("HTTP header assertion error for %v."+

test/fn-api-tests/utils.go

@@ -212,7 +212,7 @@ func EnvAsHeader(req *http.Request, selectedEnv []string) {
 	}
 }
 
-func CallFN(u string, content io.Reader, output io.Writer, method string, env []string) (http.Header, error) {
+func CallFN(u string, content io.Reader, output io.Writer, method string, env []string) (*http.Response, error) {
 	if method == "" {
 		if content == nil {
 			method = "GET"
@@ -239,7 +239,7 @@ func CallFN(u string, content io.Reader, output io.Writer, method string, env []
 
 	io.Copy(output, resp.Body)
 
-	return resp.Header, nil
+	return resp, nil
 }
 
 func init() {