Files
fn-serverless/test/fn-system-tests/exec_test.go
Tolga Ceylan f24172aa9d fn: introducing lb placer basic metrics (#1058)
* fn: introducing lb placer basic metrics

This change adds basic metrics to naive and consistent
hash LB placers. The stats show how many times we scanned
the full runner list, if runner pool failed to return a
runner list or if runner pool returned an empty list.

Placed and not placed status are also tracked along with
if TryExec returned an error or not. Most common error
code, Too-Busy is specifically tracked.

If client cancels/times out, this is also tracked as
a client cancel metric.

For placer latency, we would like to know how much time
the placer spent on searching for a runner until it
successfully places a call. This includes round-trip
times for NACK responses from the runners until a successful
TryExec() call. By excluding last successful TryExec() latency,
we try to exclude function execution & runner container
startup time from this metric in an attempt to isolate
Placer only latency.

* fn: latency and attempt tracker

Removing full scan metric. Tracking number of
runners attempted is a better metric for this
purpose.

Also, if rp.Runners() fail, this is an unrecoverable
error and we should bail out instead of retrying.

* fn: typo fix, ch placer finalize err return

* fn: enable LB placer metrics in WithAgentFromEnv if prometheus is enabled
2018-06-12 13:36:05 -07:00

319 lines
7.4 KiB
Go

package tests
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"net/http"
"net/url"
"path"
"strings"
"testing"
"time"
apiutils "github.com/fnproject/fn/test/fn-api-tests"
sdkmodels "github.com/fnproject/fn_go/models"
)
func getEchoContent(respBytes []byte) (string, error) {
var respJs map[string]interface{}
err := json.Unmarshal(respBytes, &respJs)
if err != nil {
return "", err
}
req, ok := respJs["request"].(map[string]interface{})
if !ok {
return "", errors.New("unexpected json: request map")
}
echo, ok := req["echoContent"].(string)
if !ok {
return "", errors.New("unexpected json: echoContent string")
}
return echo, nil
}
func TestCanExecuteFunction(t *testing.T) {
s := apiutils.SetupHarness()
s.GivenAppExists(t, &sdkmodels.App{Name: s.AppName})
defer s.Cleanup()
rt := s.BasicRoute()
rt.Image = "fnproject/fn-test-utils"
rt.Format = "json"
rt.Memory = 64
rt.Type = "sync"
s.GivenRouteExists(t, s.AppName, rt)
lb, err := LB()
if err != nil {
t.Fatalf("Got unexpected error: %v", err)
}
u := url.URL{
Scheme: "http",
Host: lb,
}
u.Path = path.Join(u.Path, "r", s.AppName, s.RoutePath)
body := `{"echoContent": "HelloWorld", "sleepTime": 0, "isDebug": true}`
content := bytes.NewBuffer([]byte(body))
output := &bytes.Buffer{}
resp, err := apiutils.CallFN(s.Context, u.String(), content, output, "POST", []string{})
if err != nil {
t.Fatalf("Got unexpected error: %v", err)
}
echo, err := getEchoContent(output.Bytes())
if err != nil || echo != "HelloWorld" {
t.Fatalf("getEchoContent/HelloWorld check failed on %v", output)
}
if resp.StatusCode != http.StatusOK {
t.Fatalf("StatusCode check failed on %v", resp.StatusCode)
}
}
func TestCanExecuteBigOutput(t *testing.T) {
s := apiutils.SetupHarness()
s.GivenAppExists(t, &sdkmodels.App{Name: s.AppName})
defer s.Cleanup()
rt := s.BasicRoute()
rt.Image = "fnproject/fn-test-utils"
rt.Format = "json"
rt.Memory = 64
rt.Type = "sync"
s.GivenRouteExists(t, s.AppName, rt)
lb, err := LB()
if err != nil {
t.Fatalf("Got unexpected error: %v", err)
}
u := url.URL{
Scheme: "http",
Host: lb,
}
u.Path = path.Join(u.Path, "r", s.AppName, s.RoutePath)
// Approx 5.3MB output
body := `{"echoContent": "HelloWorld", "sleepTime": 0, "isDebug": true, "trailerRepeat": 410000}`
content := bytes.NewBuffer([]byte(body))
output := &bytes.Buffer{}
resp, err := apiutils.CallFN(s.Context, u.String(), content, output, "POST", []string{})
if err != nil {
t.Fatalf("Got unexpected error: %v", err)
}
t.Logf("getEchoContent/HelloWorld size %d", len(output.Bytes()))
echo, err := getEchoContent(output.Bytes())
if err != nil || echo != "HelloWorld" {
t.Fatalf("getEchoContent/HelloWorld check failed on %v", output)
}
if resp.StatusCode != http.StatusOK {
t.Fatalf("StatusCode check failed on %v", resp.StatusCode)
}
}
func TestCanExecuteTooBigOutput(t *testing.T) {
s := apiutils.SetupHarness()
s.GivenAppExists(t, &sdkmodels.App{Name: s.AppName})
defer s.Cleanup()
rt := s.BasicRoute()
rt.Image = "fnproject/fn-test-utils"
rt.Format = "json"
rt.Memory = 64
rt.Type = "sync"
s.GivenRouteExists(t, s.AppName, rt)
lb, err := LB()
if err != nil {
t.Fatalf("Got unexpected error: %v", err)
}
u := url.URL{
Scheme: "http",
Host: lb,
}
u.Path = path.Join(u.Path, "r", s.AppName, s.RoutePath)
// > 6MB output
body := `{"echoContent": "HelloWorld", "sleepTime": 0, "isDebug": true, "trailerRepeat": 600000}`
content := bytes.NewBuffer([]byte(body))
output := &bytes.Buffer{}
resp, err := apiutils.CallFN(s.Context, u.String(), content, output, "POST", []string{})
if err != nil {
t.Fatalf("Got unexpected error: %v", err)
}
exp := "{\"error\":{\"message\":\"function response too large\"}}\n"
actual := output.String()
if !strings.Contains(exp, actual) || len(exp) != len(actual) {
t.Fatalf("Assertion error.\n\tExpected: %v\n\tActual: %v", exp, output.String())
}
if resp.StatusCode != http.StatusBadGateway {
t.Fatalf("StatusCode check failed on %v", resp.StatusCode)
}
}
func TestCanExecuteEmptyOutput(t *testing.T) {
s := apiutils.SetupHarness()
s.GivenAppExists(t, &sdkmodels.App{Name: s.AppName})
defer s.Cleanup()
rt := s.BasicRoute()
rt.Image = "fnproject/fn-test-utils"
rt.Format = "json"
rt.Memory = 64
rt.Type = "sync"
s.GivenRouteExists(t, s.AppName, rt)
lb, err := LB()
if err != nil {
t.Fatalf("Got unexpected error: %v", err)
}
u := url.URL{
Scheme: "http",
Host: lb,
}
u.Path = path.Join(u.Path, "r", s.AppName, s.RoutePath)
// empty body output
body := `{"sleepTime": 0, "isDebug": true, "isEmptyBody": true}`
content := bytes.NewBuffer([]byte(body))
output := &bytes.Buffer{}
resp, err := apiutils.CallFN(s.Context, u.String(), content, output, "POST", []string{})
if err != nil {
t.Fatalf("Got unexpected error: %v", err)
}
actual := output.String()
if 0 != len(actual) {
t.Fatalf("Assertion error.\n\tExpected empty\n\tActual: %v", output.String())
}
if resp.StatusCode != http.StatusOK {
t.Fatalf("StatusCode check failed on %v", resp.StatusCode)
}
}
func TestBasicConcurrentExecution(t *testing.T) {
s := apiutils.SetupHarness()
s.GivenAppExists(t, &sdkmodels.App{Name: s.AppName})
defer s.Cleanup()
rt := s.BasicRoute()
rt.Image = "fnproject/fn-test-utils"
rt.Format = "json"
rt.Memory = 32
rt.Type = "sync"
s.GivenRouteExists(t, s.AppName, rt)
lb, err := LB()
if err != nil {
t.Fatalf("Got unexpected error: %v", err)
}
u := url.URL{
Scheme: "http",
Host: lb,
}
u.Path = path.Join(u.Path, "r", s.AppName, s.RoutePath)
results := make(chan error)
concurrentFuncs := 10
for i := 0; i < concurrentFuncs; i++ {
go func() {
body := `{"echoContent": "HelloWorld", "sleepTime": 0, "isDebug": true}`
content := bytes.NewBuffer([]byte(body))
output := &bytes.Buffer{}
resp, err := apiutils.CallFN(s.Context, u.String(), content, output, "POST", []string{})
if err != nil {
results <- fmt.Errorf("Got unexpected error: %v", err)
return
}
echo, err := getEchoContent(output.Bytes())
if err != nil || echo != "HelloWorld" {
results <- fmt.Errorf("Assertion error.\n\tActual: %v", output.String())
return
}
if resp.StatusCode != http.StatusOK {
results <- fmt.Errorf("StatusCode check failed on %v", resp.StatusCode)
return
}
results <- nil
}()
}
for i := 0; i < concurrentFuncs; i++ {
err := <-results
if err != nil {
t.Fatalf("Error in basic concurrency execution test: %v", err)
}
}
}
func TestSaturatedSystem(t *testing.T) {
s := apiutils.SetupHarness()
// override default 60 secs with shorter.
s.Cancel()
s.Context, s.Cancel = context.WithTimeout(context.Background(), 4*time.Second)
s.GivenAppExists(t, &sdkmodels.App{Name: s.AppName})
defer s.Cleanup()
timeout := int32(1)
rt := s.BasicRoute()
rt.Image = "fnproject/fn-test-utils"
rt.Format = "json"
rt.Timeout = &timeout
rt.Memory = 300
rt.Type = "sync"
s.GivenRouteExists(t, s.AppName, rt)
lb, err := LB()
if err != nil {
t.Fatalf("Got unexpected error: %v", err)
}
u := url.URL{
Scheme: "http",
Host: lb,
}
u.Path = path.Join(u.Path, "r", s.AppName, s.RoutePath)
body := `{"echoContent": "HelloWorld", "sleepTime": 0, "isDebug": true}`
content := bytes.NewBuffer([]byte(body))
output := &bytes.Buffer{}
resp, err := apiutils.CallFN(s.Context, u.String(), content, output, "POST", []string{})
if resp != nil || err == nil || s.Context.Err() == nil {
t.Fatalf("Expected response: %v err:%v", resp, err)
}
}