alihan/fn-serverless
1
0
Fork 0
mirror of https://github.com/fnproject/fn.git synced 2022-10-28 21:29:17 +03:00
Code Issues Packages Projects Releases Wiki Activity

fn: I/O related improvements (#809)

Browse Source 
*) I/O protocol parse issues should shutdown the container as the container
goes to inconsistent state between calls. (eg. next call may receive previous
calls left overs.)
*) Move ghost read/write code into io_utils in common.
*) Clean unused error from docker Wait()
*) We can catch one case in JSON, if there's remaining unparsed data in
decoder buffer, we can shut the container
*) stdout/stderr when container is not handling a request are now blocked if freezer is also enabled.
*) if a fatal err is set for slot, we do not requeue it and proceed to shutdown
*) added a test function for a few cases with freezer strict behavior
This commit is contained in:
Tolga Ceylan
2018-03-07 15:09:24 -08:00
committed by GitHub
parent f044e509fc
commit 7677aad450
11 changed files with 568 additions and 202 deletions
Download Patch File Download Diff File Expand all files Collapse all files

283
api/server/runner_test.go
View File

@@ -3,6 +3,7 @@ package server
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io/ioutil"
@@ -10,6 +11,7 @@ import (
"os"
"strings"
"testing"
"time"
"github.com/fnproject/fn/api/agent"
"github.com/fnproject/fn/api/datastore"
@@ -18,6 +20,27 @@ import (
"github.com/fnproject/fn/api/mqs"
)
func envTweaker(name, value string) func() {
bck, ok := os.LookupEnv(name)
err := os.Setenv(name, value)
if err != nil {
panic(err.Error())
}
return func() {
var err error
if !ok {
err = os.Unsetenv(name)
} else {
err = os.Setenv(name, bck)
}
if err != nil {
panic(err.Error())
}
}
}
func testRunner(t *testing.T, args ...interface{}) (agent.Agent, context.CancelFunc) {
ds := datastore.NewMock()
var mq models.MessageQueue = &mqs.Mock{}
@@ -123,9 +146,169 @@ func TestRouteRunnerPost(t *testing.T) {
}
}
func TestRouteRunnerIOPipes(t *testing.T) {
buf := setLogBuffer()
isFailure := false
// let's make freezer immediate, so that we don't deal with
// more timing related issues below. Slightly gains us a bit more
// determinism.
tweaker := envTweaker("FN_FREEZE_IDLE_MSECS", "0")
defer tweaker()
// Log once after we are done, flow of events are important (hot/cold containers, idle timeout, etc.)
// for figuring out why things failed.
defer func() {
if isFailure {
t.Log(buf.String())
}
}()
rCfg := map[string]string{"ENABLE_HEADER": "yes", "ENABLE_FOOTER": "yes"} // enable container start/end header/footer
rImg := "fnproject/fn-test-utils"
ds := datastore.NewMockInit(
[]*models.App{
{Name: "zoo", Config: models.Config{}},
},
[]*models.Route{
{Path: "/json", AppName: "zoo", Image: rImg, Type: "sync", Format: "json", Memory: 64, Timeout: 30, IdleTimeout: 30, Config: rCfg},
{Path: "/http", AppName: "zoo", Image: rImg, Type: "sync", Format: "http", Memory: 64, Timeout: 30, IdleTimeout: 30, Config: rCfg},
}, nil,
)
rnr, cancelrnr := testRunner(t, ds)
defer cancelrnr()
srv := testServer(ds, &mqs.Mock{}, ds, rnr, ServerTypeFull)
// sleep between logs and with debug enabled, fn-test-utils will log header/footer below:
immediateGarbage := `{"isDebug": true, "postOutGarbage": "YOGURT_YOGURT_YOGURT", "postSleepTime": 0}`
delayedGarbage := `{"isDebug": true, "postOutGarbage": "YOGURT_YOGURT_YOGURT", "postSleepTime": 1000}`
ok := `{"isDebug": true}`
//multiLogExpect := []string{"BeginOfLogs", "EndOfLogs"}
containerIds := make([]string, 0)
for i, test := range []struct {
path string
body string
method string
expectedCode int
expectedErrSubStr string
expectedLogsSubStr []string
sleepAmount time.Duration
}{
//
// JSON WORLD
//
// CASE I: immediate garbage: likely to be in the json decoder buffer after json resp parsing
{"/r/zoo/json/", immediateGarbage, "GET", http.StatusOK, "", nil, 0},
// CASE II: delayed garbage: make sure delayed output lands in between request processing, should be blocked until next req
{"/r/zoo/json/", delayedGarbage, "GET", http.StatusOK, "", nil, time.Second * 2},
// CASE III: normal, but should get faulty I/O from previous
{"/r/zoo/json/", ok, "GET", http.StatusBadGateway, "invalid json", nil, 0},
// CASE IV: should land on CASE III container
{"/r/zoo/json/", ok, "GET", http.StatusOK, "", nil, 0},
//
// HTTP WORLD
//
// CASE I: immediate garbage: should be ignored (TODO: this should test immediateGarbage case, FIX THIS)
{"/r/zoo/http", ok, "GET", http.StatusOK, "", nil, 0},
// CASE II: delayed garbage: make sure delayed output lands in between request processing, freezer should block,
// bad IO lands on next request.
{"/r/zoo/http", delayedGarbage, "GET", http.StatusOK, "", nil, time.Second * 2},
// CASE III: normal, but should not land on any container from case I/II.
{"/r/zoo/http/", ok, "GET", http.StatusBadGateway, "invalid http", nil, 0},
// CASE IV: should land on CASE III container
{"/r/zoo/http/", ok, "GET", http.StatusOK, "", nil, 0},
} {
body := strings.NewReader(test.body)
_, rec := routerRequest(t, srv.Router, test.method, test.path, body)
respBytes, _ := ioutil.ReadAll(rec.Body)
respBody := string(respBytes)
maxBody := len(respBody)
if maxBody > 1024 {
maxBody = 1024
}
containerIds = append(containerIds, "N/A")
if rec.Code != test.expectedCode {
isFailure = true
t.Errorf("Test %d: Expected status code to be %d but was %d. body: %s",
i, test.expectedCode, rec.Code, respBody[:maxBody])
}
if test.expectedErrSubStr != "" && !strings.Contains(respBody, test.expectedErrSubStr) {
isFailure = true
t.Errorf("Test %d: Expected response to include %s but got body: %s",
i, test.expectedErrSubStr, respBody[:maxBody])
}
if test.expectedLogsSubStr != nil {
callID := rec.Header().Get("Fn_call_id")
if !checkLogs(t, i, ds, callID, test.expectedLogsSubStr) {
isFailure = true
}
}
if rec.Code == http.StatusOK {
dockerId, err := getDockerId(respBytes)
if err != nil {
isFailure = true
t.Errorf("Test %d: cannot fetch docker id body: %s",
i, respBody[:maxBody])
}
containerIds[i] = dockerId
}
t.Logf("Test %d: dockerId: %v", i, containerIds[i])
time.Sleep(test.sleepAmount)
}
jsonIds := containerIds[0:4]
// now cross check JSON container ids:
if jsonIds[0] != jsonIds[1] &&
jsonIds[2] == "N/A" &&
jsonIds[1] != jsonIds[2] &&
jsonIds[2] != jsonIds[3] {
t.Logf("json container ids are OK, ids=%v", jsonIds)
} else {
isFailure = true
t.Errorf("json container ids are not OK, ids=%v", jsonIds)
}
httpids := containerIds[4:]
// now cross check HTTP container ids:
if httpids[0] == httpids[1] &&
httpids[2] == "N/A" &&
httpids[1] != httpids[2] &&
httpids[2] != httpids[3] {
t.Logf("http container ids are OK, ids=%v", httpids)
} else {
isFailure = true
t.Errorf("http container ids are not OK, ids=%v", httpids)
}
}
func TestRouteRunnerExecution(t *testing.T) {
buf := setLogBuffer()
isFailure := false
tweaker := envTweaker("FN_MAX_RESPONSE_SIZE", "2048")
defer tweaker()
// Log once after we are done, flow of events are important (hot/cold containers, idle timeout, etc.)
// for figuring out why things failed.
defer func() {
@@ -140,12 +323,6 @@ func TestRouteRunnerExecution(t *testing.T) {
rImgBs1 := "fnproject/imagethatdoesnotexist"
rImgBs2 := "localhost:5000/fnproject/imagethatdoesnotexist"
err := os.Setenv("FN_MAX_RESPONSE_SIZE", "2048")
if err != nil {
t.Errorf("Cannot set response size %v", err)
}
defer os.Setenv("FN_MAX_RESPONSE_SIZE", "")
ds := datastore.NewMockInit(
[]*models.App{
{Name: "myapp", Config: models.Config{}},
@@ -182,10 +359,10 @@ func TestRouteRunnerExecution(t *testing.T) {
respTypeJason := `{"jasonContentType": "foo/bar", "isDebug": true}` // Content-Type: foo/bar
// sleep between logs and with debug enabled, fn-test-utils will log header/footer below:
multiLog := `{"sleepTime": 1, "isDebug": true}`
multiLog := `{"sleepTime": 1000, "isDebug": true}`
multiLogExpect := []string{"BeginOfLogs", "EndOfLogs"}
bigoutput := `{"sleepTime": 0, "isDebug": true, "echoContent": "repeatme", "trailerRepeat": 1000}` // 1000 trailers to exceed 2K
smalloutput := `{"sleepTime": 0, "isDebug": true, "echoContent": "repeatme", "trailerRepeat": 1}` // 1 trailer < 2K
bigoutput := `{"isDebug": true, "echoContent": "repeatme", "trailerRepeat": 1000}` // 1000 trailers to exceed 2K
smalloutput := `{"isDebug": true, "echoContent": "repeatme", "trailerRepeat": 1}` // 1 trailer < 2K
for i, test := range []struct {
path string
@@ -199,17 +376,17 @@ func TestRouteRunnerExecution(t *testing.T) {
{"/r/myapp/", ok, "GET", http.StatusOK, expHeaders, "", nil},
{"/r/myapp/myhot", badHot, "GET", http.StatusBadGateway, expHeaders, "invalid http response", nil},
// hot container now back to normal, we should get OK
// hot container now back to normal:
{"/r/myapp/myhot", ok, "GET", http.StatusOK, expHeaders, "", nil},
{"/r/myapp/myhotjason", badHot, "GET", http.StatusBadGateway, expHeaders, "invalid json response", nil},
// hot container now back to normal:
{"/r/myapp/myhotjason", ok, "GET", http.StatusOK, expHeaders, "", nil},
{"/r/myapp/myhot", respTypeLie, "GET", http.StatusOK, expCTHeaders, "", nil},
{"/r/myapp/myhotjason", respTypeLie, "GET", http.StatusOK, expCTHeaders, "", nil},
{"/r/myapp/myhotjason", respTypeJason, "GET", http.StatusOK, expCTHeaders, "", nil},
{"/r/myapp/myhotjason", badHot, "GET", http.StatusBadGateway, expHeaders, "invalid json response", nil},
{"/r/myapp/myroute", ok, "GET", http.StatusOK, expHeaders, "", nil},
{"/r/myapp/myerror", crasher, "GET", http.StatusBadGateway, expHeaders, "container exit code 2", nil},
{"/r/myapp/mydne", ``, "GET", http.StatusNotFound, nil, "pull access denied", nil},
@@ -261,37 +438,69 @@ func TestRouteRunnerExecution(t *testing.T) {
if test.expectedLogsSubStr != nil {
callID := rec.Header().Get("Fn_call_id")
logReader, err := ds.GetLog(context.Background(), "myapp", callID)
if err != nil {
if !checkLogs(t, i, ds, callID, test.expectedLogsSubStr) {
isFailure = true
t.Errorf("Test %d: GetLog for call_id:%s returned err %s",
i, callID, err.Error())
} else {
logBytes, err := ioutil.ReadAll(logReader)
if err != nil {
isFailure = true
t.Errorf("Test %d: GetLog read IO call_id:%s returned err %s",
i, callID, err.Error())
} else {
logBody := string(logBytes)
maxLog := len(logBody)
if maxLog > 1024 {
maxLog = 1024
}
for _, match := range test.expectedLogsSubStr {
if !strings.Contains(logBody, match) {
isFailure = true
t.Errorf("Test %d: GetLog read IO call_id:%s cannot find: %s in logs: %s",
i, callID, match, logBody[:maxLog])
}
}
}
}
}
}
}
func getDockerId(respBytes []byte) (string, error) {
var respJs map[string]interface{}
var data map[string]interface{}
err := json.Unmarshal(respBytes, &respJs)
if err != nil {
return "", err
}
data, ok := respJs["data"].(map[string]interface{})
if !ok {
return "", errors.New("unexpected json: data map")
}
id, ok := data["DockerId"].(string)
if !ok {
return "", errors.New("unexpected json: docker id string")
}
return id, nil
}
func checkLogs(t *testing.T, tnum int, ds models.Datastore, callID string, expected []string) bool {
logReader, err := ds.GetLog(context.Background(), "myapp", callID)
if err != nil {
t.Errorf("Test %d: GetLog for call_id:%s returned err %s",
tnum, callID, err.Error())
return false
}
logBytes, err := ioutil.ReadAll(logReader)
if err != nil {
t.Errorf("Test %d: GetLog read IO call_id:%s returned err %s",
tnum, callID, err.Error())
return false
}
logBody := string(logBytes)
maxLog := len(logBody)
if maxLog > 1024 {
maxLog = 1024
}
for _, match := range expected {
if !strings.Contains(logBody, match) {
t.Errorf("Test %d: GetLog read IO call_id:%s cannot find: %s in logs: %s",
tnum, callID, match, logBody[:maxLog])
return false
}
}
return true
}
// implement models.MQ and models.APIError
type errorMQ struct {
error