alihan/fn-serverless
1
0
Fork 0
mirror of https://github.com/fnproject/fn.git synced 2022-10-28 21:29:17 +03:00
Code Issues Packages Projects Releases Wiki Activity

Stream test commence (#1224)

Browse Source 
* initial invoke testing

this assures that Content-Type and Fn-Http-Status are set for an http-stream
function. it took some fixing up of the test utils code for the plumbing to
work, looking forward to deleting most stuff in fn-test-utils.go file around
each format -- had to update fdk-go to latest for http-stream support. this
only adds 1 test, since there's some machinery here, and would like to unblock
working on the http gateway simultaneously while adding a full suite of invoke
tests (this work can be parallelized)...

i added debug logs back to the debugging output. turns out this is useful, but
it can get noisy (only when things fail, hopefully).

* fix oom tests?
This commit is contained in:
Reed Allman
2018-09-19 08:48:48 -07:00
committed by GitHub
parent 111f2d4a1c
commit 485fa465a0
6 changed files with 33 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
api/agent/call.go
View File

@@ -64,9 +64,9 @@ func FromHTTPTriggerRequest(app *models.App, fn *models.Fn, trigger *models.Trig
// Expected Content-Type for a CloudEvent: application/cloudevents+json; charset=UTF-8
contentType := req.Header.Get("Content-Type")
t, _, err := mime.ParseMediaType(contentType)
if err != nil {
if err != nil && contentType != "" {
// won't fail here, but log
log.Debugf("Could not parse Content-Type header: %v", err)
log.Debugf("Could not parse Content-Type header: %v %v", contentType, err)
} else {
if t == ceMimeType {
c.IsCloudEvent = true
@@ -135,9 +135,9 @@ func FromHTTPFnRequest(app *models.App, fn *models.Fn, req *http.Request) CallOp
// Expected Content-Type for a CloudEvent: application/cloudevents+json; charset=UTF-8
contentType := req.Header.Get("Content-Type")
t, _, err := mime.ParseMediaType(contentType)
if err != nil {
if err != nil && contentType != "" {
// won't fail here, but log
log.Debugf("Could not parse Content-Type header: %v", err)
log.Debugf("Could not parse Content-Type header: %v %v", contentType, err)
} else {
if t == ceMimeType {
c.IsCloudEvent = true

22
api/server/runner_fninvoke_test.go
View File

@@ -154,11 +154,12 @@ func TestFnInvokeRunnerExecution(t *testing.T) {
httpDneFn := &models.Fn{ID: "http_dne_fn_id", Name: "http_dne_fn", AppID: app.ID, Image: rImgBs1, Format: "http", ResourceConfig: models.ResourceConfig{Memory: 64, Timeout: 30, IdleTimeout: 30}, Config: rCfg}
httpDneRegistryFn := &models.Fn{ID: "http_dnereg_fn_id", Name: "http_dnereg_fn", AppID: app.ID, Image: rImgBs2, Format: "http", ResourceConfig: models.ResourceConfig{Memory: 64, Timeout: 30, IdleTimeout: 30}, Config: rCfg}
jsonFn := &models.Fn{ID: "json_fn_id", Name: "json_fn", AppID: app.ID, Image: rImg, Format: "json", ResourceConfig: models.ResourceConfig{Memory: 64, Timeout: 30, IdleTimeout: 30}, Config: rCfg}
oomFn := &models.Fn{ID: "http_fn_id", Name: "http_fn", AppID: app.ID, Image: rImg, Format: "http", ResourceConfig: models.ResourceConfig{Memory: 8, Timeout: 30, IdleTimeout: 30}, Config: rCfg}
oomFn := &models.Fn{ID: "http_oom_fn_id", Name: "http_fn", AppID: app.ID, Image: rImg, Format: "http", ResourceConfig: models.ResourceConfig{Memory: 8, Timeout: 30, IdleTimeout: 30}, Config: rCfg}
httpStreamFn := &models.Fn{ID: "http_stream_fn_id", Name: "http_stream_fn", AppID: app.ID, Image: rImg, Format: "http-stream", ResourceConfig: models.ResourceConfig{Memory: 64, Timeout: 30, IdleTimeout: 30}, Config: rCfg}
ds := datastore.NewMockInit(
[]*models.App{app},
[]*models.Fn{defaultFn, defaultDneFn, httpDneRegistryFn, oomFn, httpFn, jsonFn, httpDneFn},
[]*models.Fn{defaultFn, defaultDneFn, httpDneRegistryFn, httpFn, jsonFn, httpDneFn, oomFn, httpStreamFn},
)
ls := logs.NewMock()
@@ -169,18 +170,19 @@ func TestFnInvokeRunnerExecution(t *testing.T) {
expHeaders := map[string][]string{"Content-Type": {"application/json; charset=utf-8"}}
expCTHeaders := map[string][]string{"Content-Type": {"foo/bar"}}
expStreamHeaders := map[string][]string{"Content-Type": {"application/json; charset=utf-8"}, "Fn-Http-Status": {"200"}}
// Checking for EndOfLogs currently depends on scheduling of go-routines (in docker/containerd) that process stderr & stdout.
// Therefore, not testing for EndOfLogs for hot containers (which has complex I/O processing) anymore.
multiLogExpectCold := []string{"BeginOfLogs", "EndOfLogs"}
multiLogExpectHot := []string{"BeginOfLogs" /*, "EndOfLogs" */}
crasher := `{"echoContent": "_TRX_ID_", "isDebug": true, "isCrash": true}` // crash container
oomer := `{"echoContent": "_TRX_ID_", "isDebug": true, "allocateMemory": 12000000}` // ask for 12MB
badHot := `{"echoContent": "_TRX_ID_", "invalidResponse": true, "isDebug": true}` // write a not json/http as output
ok := `{"echoContent": "_TRX_ID_", "isDebug": true}` // good response / ok
respTypeLie := `{"echoContent": "_TRX_ID_", "responseContentType": "foo/bar", "isDebug": true}` // Content-Type: foo/bar
respTypeJason := `{"echoContent": "_TRX_ID_", "jasonContentType": "foo/bar", "isDebug": true}` // Content-Type: foo/bar
crasher := `{"echoContent": "_TRX_ID_", "isDebug": true, "isCrash": true}` // crash container
oomer := `{"echoContent": "_TRX_ID_", "isDebug": true, "allocateMemory": 12000000}` // ask for 12MB
badHot := `{"echoContent": "_TRX_ID_", "invalidResponse": true, "isDebug": true}` // write a not json/http as output
ok := `{"echoContent": "_TRX_ID_", "responseContentType": "application/json; charset=utf-8", "isDebug": true}` // good response / ok
respTypeLie := `{"echoContent": "_TRX_ID_", "responseContentType": "foo/bar", "isDebug": true}` // Content-Type: foo/bar
respTypeJason := `{"echoContent": "_TRX_ID_", "jasonContentType": "foo/bar", "isDebug": true}` // Content-Type: foo/bar
// sleep between logs and with debug enabled, fn-test-utils will log header/footer below:
multiLog := `{"echoContent": "_TRX_ID_", "sleepTime": 1000, "isDebug": true}`
@@ -210,6 +212,8 @@ func TestFnInvokeRunnerExecution(t *testing.T) {
// hot container now back to normal:
{"/invoke/json_fn_id", ok, "POST", http.StatusOK, expHeaders, "", nil},
{"/invoke/http_stream_fn_id", ok, "POST", http.StatusOK, expStreamHeaders, "", nil},
{"/invoke/http_fn_id", respTypeLie, "POST", http.StatusOK, expCTHeaders, "", nil},
{"/invoke/json_fn_id", respTypeLie, "POST", http.StatusOK, expCTHeaders, "", nil},
{"/invoke/json_fn_id", respTypeJason, "POST", http.StatusOK, expCTHeaders, "", nil},
@@ -219,7 +223,7 @@ func TestFnInvokeRunnerExecution(t *testing.T) {
{"/invoke/default_dne_fn_id", ``, "POST", http.StatusNotFound, nil, "pull access denied", nil},
{"/invoke/http_dne_fn_id", ``, "POST", http.StatusNotFound, nil, "pull access denied", nil},
{"/invoke/http_dnereg_fn_id", ``, "POST", http.StatusInternalServerError, nil, "connection refused", nil},
{"/invoke/http_fn_id", oomer, "POST", http.StatusBadGateway, nil, "container out of memory", nil},
{"/invoke/http_oom_fn_id", oomer, "POST", http.StatusBadGateway, nil, "container out of memory", nil},
{"/invoke/http_fn_id", multiLog, "POST", http.StatusOK, nil, "", multiLogExpectHot},
{"/invoke/default_fn_id", multiLog, "POST", http.StatusOK, nil, "", multiLogExpectCold},
{"/invoke/json_fn_id", bigoutput, "POST", http.StatusBadGateway, nil, "function response too large", nil},

2
api/server/runner_httptrigger_test.go
View File

@@ -272,7 +272,7 @@ func TestTriggerRunnerExecution(t *testing.T) {
httpDneFn := &models.Fn{ID: "http_dne_fn_id", Name: "http_dne_fn", AppID: app.ID, Image: rImgBs1, Format: "http", ResourceConfig: models.ResourceConfig{Memory: 64, Timeout: 30, IdleTimeout: 30}, Config: rCfg}
httpDneRegistryFn := &models.Fn{ID: "http_dnereg_fn_id", Name: "http_dnereg_fn", AppID: app.ID, Image: rImgBs2, Format: "http", ResourceConfig: models.ResourceConfig{Memory: 64, Timeout: 30, IdleTimeout: 30}, Config: rCfg}
jsonFn := &models.Fn{ID: "json_fn_id", Name: "json_fn", AppID: app.ID, Image: rImg, Format: "json", ResourceConfig: models.ResourceConfig{Memory: 64, Timeout: 30, IdleTimeout: 30}, Config: rCfg}
oomFn := &models.Fn{ID: "http_fn_id", Name: "http_fn", AppID: app.ID, Image: rImg, Format: "http", ResourceConfig: models.ResourceConfig{Memory: 8, Timeout: 30, IdleTimeout: 30}, Config: rCfg}
oomFn := &models.Fn{ID: "http_oom_fn_id", Name: "http_fn", AppID: app.ID, Image: rImg, Format: "http", ResourceConfig: models.ResourceConfig{Memory: 8, Timeout: 30, IdleTimeout: 30}, Config: rCfg}
ds := datastore.NewMockInit(
[]*models.App{app},

2
api/server/server_test.go
View File

@@ -19,7 +19,7 @@ import (
func testServer(ds models.Datastore, mq models.MessageQueue, logDB models.LogStore, rnr agent.Agent, nodeType NodeType, opts ...Option) *Server {
return New(context.Background(), append(opts,
WithLogLevel(getEnv(EnvLogLevel, DefaultLogLevel)),
WithLogLevel("debug"),
WithDatastore(ds),
WithMQ(mq),
WithLogstore(logDB),

2
images/fn-test-utils/Gopkg.lock generated
View File

@@ -8,7 +8,7 @@
".",
"utils"
]
revision = "5d768b2006f11737b6a69a758ddd6d2fac04923e"
revision = "d1fa834929bb1579d770eded860a3a54e1f3502b"
[solve-meta]
analyzer-name = "dep"

35
images/fn-test-utils/fn-test-utils.go
View File

@@ -111,33 +111,30 @@ func AppHandler(ctx context.Context, in io.Reader, out io.Writer) {
}
}
var outto fdkresponse
outto.Writer = out
finalizeRequest(&outto, req, resp)
finalizeRequest(out, req, resp)
err := postProcessRequest(req, out)
if err != nil {
panic(err.Error())
}
}
func finalizeRequest(out *fdkresponse, req *AppRequest, resp *AppResponse) {
func finalizeRequest(out io.Writer, req *AppRequest, resp *AppResponse) {
// custom response code
if req.ResponseCode != 0 {
out.Status = req.ResponseCode
} else {
out.Status = 200
fdk.WriteStatus(out, req.ResponseCode)
}
// custom content type
if req.ResponseContentType != "" {
out.Header.Set("Content-Type", req.ResponseContentType)
fdk.SetHeader(out, "Content-Type", req.ResponseContentType)
}
// NOTE: don't add 'application/json' explicitly here as an else,
// we will test that go's auto-detection logic does not fade since
// some people are relying on it now
if req.JasonContentType != "" {
out.JasonContentType = req.JasonContentType
// this will get picked up by our json out handler...
fdk.SetHeader(out, "Content-Type", req.JasonContentType)
}
if !req.IsEmptyBody {
@@ -294,6 +291,8 @@ func testDo(format string, in io.Reader, out io.Writer) {
testDoJSON(ctx, in, out)
case "default":
fdkutils.DoDefault(fdk.HandlerFunc(AppHandler), ctx, in, out)
case "http-stream":
fdk.Handle(fdk.HandlerFunc(AppHandler)) // XXX(reed): can extract & instrument
default:
panic("unknown format (fdk-go): " + format)
}
@@ -331,7 +330,7 @@ func testDoJSON(ctx context.Context, in io.Reader, out io.Writer) {
func testDoJSONOnce(ctx context.Context, in io.Reader, out io.Writer, buf *bytes.Buffer, hdr http.Header) error {
buf.Reset()
fdkutils.ResetHeaders(hdr)
var resp fdkresponse
var resp fdkutils.Response
resp.Writer = buf
resp.Status = 200
resp.Header = hdr
@@ -383,19 +382,11 @@ func testDoJSONOnce(ctx context.Context, in io.Reader, out io.Writer, buf *bytes
return postProcessRequest(appRequest, out)
}
// since we need to test little jason's content type since he's special. but we
// don't want to add redundant and confusing fields to the fdk...
type fdkresponse struct {
fdkutils.Response
JasonContentType string // dumb
}
// copy of fdk.GetJSONResp but with sugar for stupid jason's little fields
func getJSONResp(buf *bytes.Buffer, fnResp *fdkresponse, req *fdkutils.JsonIn) *fdkutils.JsonOut {
func getJSONResp(buf *bytes.Buffer, fnResp *fdkutils.Response, req *fdkutils.JsonIn) *fdkutils.JsonOut {
return &fdkutils.JsonOut{
Body: buf.String(),
ContentType: fnResp.JasonContentType,
ContentType: fnResp.Header.Get("Content-Type"),
Protocol: fdkutils.CallResponseHTTP{
StatusCode: fnResp.Status,
Headers: fnResp.Header,
@@ -406,7 +397,7 @@ func getJSONResp(buf *bytes.Buffer, fnResp *fdkresponse, req *fdkutils.JsonIn) *
func testDoHTTPOnce(ctx context.Context, in io.Reader, out io.Writer, buf *bytes.Buffer, hdr http.Header) error {
buf.Reset()
fdkutils.ResetHeaders(hdr)
var resp fdkresponse
var resp fdkutils.Response
resp.Writer = buf
resp.Status = 200
resp.Header = hdr
@@ -442,7 +433,7 @@ func testDoHTTPOnce(ctx context.Context, in io.Reader, out io.Writer, buf *bytes
appRequest = appReq
}
hResp := fdkutils.GetHTTPResp(buf, &resp.Response, req)
hResp := fdkutils.GetHTTPResp(buf, &resp, req)
err = hResp.Write(out)
if err != nil {