HTTP trigger http-stream tests (#1241)

2022-10-28 21:29:17 +03:00 · 2018-09-26 05:25:48 -07:00
parent 5d907821b1
commit 01b8e8679d
20 changed files with 548 additions and 403 deletions
--- a/api/server/runner_fninvoke.go
+++ b/api/server/runner_fninvoke.go
@@ -6,7 +6,6 @@ import (
 	"net/http"
 	"strconv"
 	"sync"
-	"time"

 	"github.com/fnproject/fn/api"
 	"github.com/fnproject/fn/api/agent"
@@ -20,16 +19,6 @@ var (
 	bufPool = &sync.Pool{New: func() interface{} { return new(bytes.Buffer) }}
 )

-type ResponseBufferingWriter interface {
-	http.ResponseWriter
-	io.Reader
-	Status() int
-	GetBuffer() *bytes.Buffer
-	SetBuffer(*bytes.Buffer)
-}
-
-var _ ResponseBufferingWriter = new(syncResponseWriter)
-
 // implements http.ResponseWriter
 // this little guy buffers responses from user containers and lets them still
 // set headers and such without us risking writing partial output [as much, the
@@ -41,13 +30,10 @@ type syncResponseWriter struct {
 	*bytes.Buffer
 }

-func (s *syncResponseWriter) Header() http.Header { return s.headers }
+var _ http.ResponseWriter = new(syncResponseWriter) // nice compiler errors

-// By storing the status here, we effectively buffer the response
-func (s *syncResponseWriter) WriteHeader(code int)        { s.status = code }
-func (s *syncResponseWriter) Status() int                 { return s.status }
-func (s *syncResponseWriter) GetBuffer() *bytes.Buffer    { return s.Buffer }
-func (s *syncResponseWriter) SetBuffer(buf *bytes.Buffer) { s.Buffer = buf }
+func (s *syncResponseWriter) Header() http.Header  { return s.headers }
+func (s *syncResponseWriter) WriteHeader(code int) { s.status = code }

 // handleFnInvokeCall executes the function, for router handlers
 func (s *Server) handleFnInvokeCall(c *gin.Context) {
@@ -79,71 +65,84 @@ func (s *Server) handleFnInvokeCall2(c *gin.Context) error {
 }

 func (s *Server) ServeFnInvoke(c *gin.Context, app *models.App, fn *models.Fn) error {
-	writer := &syncResponseWriter{
-		headers: c.Writer.Header(),
+	return s.fnInvoke(c.Writer, c.Request, app, fn, nil)
+}
+
+func (s *Server) fnInvoke(resp http.ResponseWriter, req *http.Request, app *models.App, fn *models.Fn, trig *models.Trigger) error {
+	// TODO: we should get rid of the buffers, and stream back (saves memory (+splice), faster (splice), allows streaming, don't have to cap resp size)
+	// buffer the response before writing it out to client to prevent partials from trying to stream
+	buf := bufPool.Get().(*bytes.Buffer)
+	buf.Reset()
+	bufWriter := syncResponseWriter{
+		headers: resp.Header(),
+		status:  200,
+		Buffer:  buf,
 	}

-	call, err := s.agent.GetCall(agent.WithWriter(writer), // XXX (reed): order matters [for now]
-		agent.FromHTTPFnRequest(app, fn, c.Request))
+	var writer http.ResponseWriter = &bufWriter
+	writer = &jsonContentTypeTrapper{ResponseWriter: writer}

+	opts := []agent.CallOpt{
+		agent.WithWriter(writer), // XXX (reed): order matters [for now]
+		agent.FromHTTPFnRequest(app, fn, req),
+	}
+	if trig != nil {
+		opts = append(opts, agent.WithTrigger(trig))
+	}
+
+	call, err := s.agent.GetCall(opts...)
 	if err != nil {
 		return err
 	}

-	return s.fnInvoke(c, app, fn, writer, call)
+	err = s.agent.Submit(call)
+	if err != nil {
+		return err
+	}
+
+	// because we can...
+	writer.Header().Set("Content-Length", strconv.Itoa(int(buf.Len())))
+
+	// buffered response writer traps status (so we can add headers), we need to write it still
+	if bufWriter.status > 0 {
+		resp.WriteHeader(bufWriter.status)
+	}
+
+	io.Copy(resp, buf)
+	bufPool.Put(buf) // at this point, submit returned without timing out, so we can re-use this one
+	return nil
 }

-func (s *Server) fnInvoke(c *gin.Context, app *models.App, fn *models.Fn, writer ResponseBufferingWriter, call agent.Call) error {
-	// TODO: we should get rid of the buffers, and stream back (saves memory (+splice), faster (splice), allows streaming, don't have to cap resp size)
-	buf := bufPool.Get().(*bytes.Buffer)
-	buf.Reset()
-	var submitErr error
-	defer func() {
-		if buf.Len() == 0 && submitErr == nil {
-			bufPool.Put(buf) // TODO need to ensure this is safe with Dispatch?
-		}
-	}()
-	writer.SetBuffer(buf)
+// TODO kill this thing after removing tests for http/json/default formats
+type jsonContentTypeTrapper struct {
+	http.ResponseWriter
+	committed bool
+}

-	model := call.Model()
-	{ // scope this, to disallow ctx use outside of this scope. add id for handleV1ErrorResponse logger
-		ctx, _ := common.LoggerWithFields(c.Request.Context(), logrus.Fields{"id": model.ID})
-		c.Request = c.Request.WithContext(ctx)
-	}
+var _ http.ResponseWriter = new(jsonContentTypeTrapper) // nice compiler errors

-	submitErr = s.agent.Submit(call)
-	if submitErr != nil {
-		// NOTE if they cancel the request then it will stop the call (kind of cool),
-		// we could filter that error out here too as right now it yells a little
-		if submitErr == models.ErrCallTimeoutServerBusy || submitErr == models.ErrCallTimeout {
-			// TODO maneuver
-			// add this, since it means that start may not have been called [and it's relevant]
-			c.Writer.Header().Add("XXX-FXLB-WAIT", time.Now().Sub(time.Time(model.CreatedAt)).String())
-		}
-		return submitErr
+func (j *jsonContentTypeTrapper) Write(b []byte) (int, error) {
+	if !j.committed {
+		// override default content type detection behavior to add json
+		j.detectContentType(b)
 	}
-	// if they don't set a content-type - detect it
-	// TODO: remove this after removing all the formats (too many tests to scrub til then)
-	if writer.Header().Get("Content-Type") == "" {
-		// see http.DetectContentType, the go server is supposed to do this for us but doesn't appear to?
+	j.committed = true
+
+	// write inner
+	return j.ResponseWriter.Write(b)
+}
+
+func (j *jsonContentTypeTrapper) detectContentType(b []byte) {
+	if j.Header().Get("Content-Type") == "" {
+		// see http.DetectContentType
 		var contentType string
 		jsonPrefix := [1]byte{'{'} // stack allocated
-		if bytes.HasPrefix(writer.GetBuffer().Bytes(), jsonPrefix[:]) {
+		if bytes.HasPrefix(b, jsonPrefix[:]) {
 			// try to detect json, since DetectContentType isn't a hipster.
 			contentType = "application/json; charset=utf-8"
 		} else {
-			contentType = http.DetectContentType(writer.GetBuffer().Bytes())
+			contentType = http.DetectContentType(b)
 		}
-		writer.Header().Set("Content-Type", contentType)
+		j.Header().Set("Content-Type", contentType)
 	}
-
-	writer.Header().Set("Content-Length", strconv.Itoa(int(writer.GetBuffer().Len())))
-
-	if writer.Status() > 0 {
-		c.Writer.WriteHeader(writer.Status())
-	}
-
-	io.Copy(c.Writer, writer)
-
-	return nil
 }
--- a/api/server/runner_fninvoke_test.go
+++ b/api/server/runner_fninvoke_test.go
@@ -261,7 +261,7 @@ func TestFnInvokeRunnerExecution(t *testing.T) {
 				maxBody = 1024
 			}

-			callIds[i] = rec.Header().Get("Fn_call_id")
+			callIds[i] = rec.Header().Get("Fn-Call-Id")
 			cid := callIds[i]

 			if rec.Code != test.expectedCode {
--- a/api/server/runner_httptrigger.go
+++ b/api/server/runner_httptrigger.go
@@ -1,13 +1,14 @@
 package server

 import (
+	"fmt"
 	"net/http"
+	"net/textproto"
 	"strconv"

 	"strings"

 	"github.com/fnproject/fn/api"
-	"github.com/fnproject/fn/api/agent"
 	"github.com/fnproject/fn/api/models"
 	"github.com/gin-gonic/gin"
 )
@@ -60,21 +61,19 @@ func (s *Server) handleTriggerHTTPFunctionCall2(c *gin.Context) error {
 }

 type triggerResponseWriter struct {
-	syncResponseWriter
+	inner     http.ResponseWriter
 	committed bool
 }

-var _ ResponseBufferingWriter = new(triggerResponseWriter)
-
 func (trw *triggerResponseWriter) Header() http.Header {
-	return trw.headers
+	return trw.inner.Header()
 }

 func (trw *triggerResponseWriter) Write(b []byte) (int, error) {
 	if !trw.committed {
 		trw.WriteHeader(http.StatusOK)
 	}
-	return trw.GetBuffer().Write(b)
+	return trw.inner.Write(b)
 }

 func (trw *triggerResponseWriter) WriteHeader(statusCode int) {
@@ -83,56 +82,98 @@ func (trw *triggerResponseWriter) WriteHeader(statusCode int) {
 	}
 	trw.committed = true

-	for k, vs := range trw.Header() {
-		if strings.HasPrefix(k, "Fn-Http-H-") {
-			// TODO strip out content-length and stuff here.
-			realHeader := strings.TrimPrefix(k, "Fn-Http-H-")
-			if realHeader != "" { // case where header is exactly the prefix
-				for _, v := range vs {
-					trw.Header().Del(k)
-					trw.Header().Add(realHeader, v)
+	var fnStatus int
+	realHeaders := trw.Header()
+	gwHeaders := make(http.Header, len(realHeaders))
+	for k, vs := range realHeaders {
+		switch {
+		case strings.HasPrefix(k, "Fn-Http-H-"):
+			gwHeader := strings.TrimPrefix(k, "Fn-Http-H-")
+			if gwHeader != "" { // case where header is exactly the prefix
+				gwHeaders[gwHeader] = vs
+			}
+		case k == "Fn-Http-Status":
+			if len(vs) > 0 {
+				statusInt, err := strconv.Atoi(vs[0])
+				if err == nil {
+					fnStatus = statusInt
 				}
 			}
+		case k == "Content-Type", k == "Fn-Call-Id":
+			gwHeaders[k] = vs
 		}
 	}

-	gatewayStatus := 200
+	// XXX(reed): this is O(3n)... yes sorry for making it work without making it perfect first
+	for k := range realHeaders {
+		realHeaders.Del(k)
+	}
+	for k, vs := range gwHeaders {
+		realHeaders[k] = vs
+	}

+	// XXX(reed): simplify / add tests for these behaviors...
+	gatewayStatus := 200
 	if statusCode >= 400 {
 		gatewayStatus = 502
+	} else if fnStatus > 0 {
+		gatewayStatus = fnStatus
 	}

-	status := trw.Header().Get("Fn-Http-Status")
-	if status != "" {
-		statusInt, err := strconv.Atoi(status)
-		if err == nil {
-			gatewayStatus = statusInt
+	trw.inner.WriteHeader(gatewayStatus)
+}
+
+var skipTriggerHeaders = map[string]bool{
+	"Connection":        true,
+	"Keep-Alive":        true,
+	"Trailer":           true,
+	"Transfer-Encoding": true,
+	"TE":                true,
+	"Upgrade":           true,
+}
+
+func reqURL(req *http.Request) string {
+	if req.URL.Scheme == "" {
+		if req.TLS == nil {
+			req.URL.Scheme = "http"
+		} else {
+			req.URL.Scheme = "https"
 		}
 	}
-
-	trw.WriteHeader(gatewayStatus)
+	if req.URL.Host == "" {
+		req.URL.Host = req.Host
+	}
+	return req.URL.String()
 }

-//ServeHTTPTr	igger serves an HTTP trigger for a given app/fn/trigger  based on the current request
+// ServeHTTPTrigger serves an HTTP trigger for a given app/fn/trigger based on the current request
 // This is exported to allow extensions to handle their own trigger naming and publishing
 func (s *Server) ServeHTTPTrigger(c *gin.Context, app *models.App, fn *models.Fn, trigger *models.Trigger) error {
-	triggerWriter := &triggerResponseWriter{
-		syncResponseWriter{
-			headers: c.Writer.Header()},
-		false,
+	// transpose trigger headers into the request
+	req := c.Request
+	headers := make(http.Header, len(req.Header))
+	for k, vs := range req.Header {
+		// should be generally unnecessary but to be doubly sure.
+		k = textproto.CanonicalMIMEHeaderKey(k)
+		if skipTriggerHeaders[k] {
+			continue
+		}
+		switch k {
+		case "Content-Type":
+		default:
+			k = fmt.Sprintf("Fn-Http-H-%s", k)
+		}
+		headers[k] = vs
 	}
-	// GetCall can mod headers, assign an id, look up the route/app (cached),
-	// strip params, etc.
-	// this should happen ASAP to turn app name to app ID
+	requestURL := reqURL(req)

-	// GetCall can mod headers, assign an id, look up the route/app (cached),
-	// strip params, etc.
+	headers.Set("Fn-Http-Method", req.Method)
+	headers.Set("Fn-Http-Request-Url", requestURL)
+	headers.Set("Fn-Intent", "httprequest")
+	req.Header = headers

-	call, err := s.agent.GetCall(agent.WithWriter(triggerWriter), // XXX (reed): order matters [for now]
-		agent.FromHTTPTriggerRequest(app, fn, trigger, c.Request))
+	// trap the headers and rewrite them for http trigger
+	rw := &triggerResponseWriter{inner: c.Writer}

-	if err != nil {
-		return err
-	}
-	return s.fnInvoke(c, app, fn, triggerWriter, call)
+	return s.fnInvoke(rw, req, app, fn, trigger)
 }
--- a/api/server/runner_httptrigger_test.go
+++ b/api/server/runner_httptrigger_test.go
@@ -16,6 +16,7 @@ import (
 	"github.com/fnproject/fn/api/logs"
 	"github.com/fnproject/fn/api/models"
 	"github.com/fnproject/fn/api/mqs"
+	"reflect"
 )

 func envTweaker(name, value string) func() {
@@ -273,10 +274,11 @@ func TestTriggerRunnerExecution(t *testing.T) {
 	httpDneRegistryFn := &models.Fn{ID: "http_dnereg_fn_id", Name: "http_dnereg_fn", AppID: app.ID, Image: rImgBs2, Format: "http", ResourceConfig: models.ResourceConfig{Memory: 64, Timeout: 30, IdleTimeout: 30}, Config: rCfg}
 	jsonFn := &models.Fn{ID: "json_fn_id", Name: "json_fn", AppID: app.ID, Image: rImg, Format: "json", ResourceConfig: models.ResourceConfig{Memory: 64, Timeout: 30, IdleTimeout: 30}, Config: rCfg}
 	oomFn := &models.Fn{ID: "http_oom_fn_id", Name: "http_fn", AppID: app.ID, Image: rImg, Format: "http", ResourceConfig: models.ResourceConfig{Memory: 8, Timeout: 30, IdleTimeout: 30}, Config: rCfg}
+	httpStreamFn := &models.Fn{ID: "http_stream_fn_id", Name: "http_stream_fn", AppID: app.ID, Image: rImg, Format: "http-stream", ResourceConfig: models.ResourceConfig{Memory: 64, Timeout: 30, IdleTimeout: 30}, Config: rCfg}

 	ds := datastore.NewMockInit(
 		[]*models.App{app},
-		[]*models.Fn{defaultFn, defaultDneFn, httpDneRegistryFn, oomFn, httpFn, jsonFn, httpDneFn},
+		[]*models.Fn{defaultFn, defaultDneFn, httpDneRegistryFn, oomFn, httpFn, jsonFn, httpDneFn, httpStreamFn},
 		[]*models.Trigger{
 			{ID: "1", Name: "1", Source: "/", Type: "http", AppID: app.ID, FnID: defaultFn.ID},
 			{ID: "2", Name: "2", Source: "/myhot", Type: "http", AppID: app.ID, FnID: httpFn.ID},
@@ -290,6 +292,7 @@ func TestTriggerRunnerExecution(t *testing.T) {
 			{ID: "10", Name: "10", Source: "/mybigoutputcold", Type: "http", AppID: app.ID, FnID: defaultFn.ID},
 			{ID: "11", Name: "11", Source: "/mybigoutputhttp", Type: "http", AppID: app.ID, FnID: httpFn.ID},
 			{ID: "12", Name: "12", Source: "/mybigoutputjson", Type: "http", AppID: app.ID, FnID: jsonFn.ID},
+			{ID: "13", Name: "13", Source: "/httpstream", Type: "http", AppID: app.ID, FnID: httpStreamFn.ID},
 		},
 	)
 	ls := logs.NewMock()
@@ -307,20 +310,38 @@ func TestTriggerRunnerExecution(t *testing.T) {
 	multiLogExpectCold := []string{"BeginOfLogs", "EndOfLogs"}
 	multiLogExpectHot := []string{"BeginOfLogs" /*, "EndOfLogs" */}

-	crasher := `{"echoContent": "_TRX_ID_", "isDebug": true, "isCrash": true}`                      // crash container
-	oomer := `{"echoContent": "_TRX_ID_", "isDebug": true, "allocateMemory": 12000000}`             // ask for 12MB
-	badHot := `{"echoContent": "_TRX_ID_", "invalidResponse": true, "isDebug": true}`               // write a not json/http as output
-	ok := `{"echoContent": "_TRX_ID_", "isDebug": true}`                                            // good response / ok
-	respTypeLie := `{"echoContent": "_TRX_ID_", "responseContentType": "foo/bar", "isDebug": true}` // Content-Type: foo/bar
-	respTypeJason := `{"echoContent": "_TRX_ID_", "jasonContentType": "foo/bar", "isDebug": true}`  // Content-Type: foo/bar
+	crasher := `{"echoContent": "_TRX_ID_", "isDebug": true, "isCrash": true}`                                     // crash container
+	oomer := `{"echoContent": "_TRX_ID_", "isDebug": true, "allocateMemory": 12000000}`                            // ask for 12MB
+	badHot := `{"echoContent": "_TRX_ID_", "invalidResponse": true, "isDebug": true}`                              // write a not json/http as output
+	ok := `{"echoContent": "_TRX_ID_", "responseContentType": "application/json; charset=utf-8", "isDebug": true}` // good response / ok
+	respTypeLie := `{"echoContent": "_TRX_ID_", "responseContentType": "foo/bar", "isDebug": true}`                // Content-Type: foo/bar
+	respTypeJason := `{"echoContent": "_TRX_ID_", "jasonContentType": "foo/bar", "isDebug": true}`                 // Content-Type: foo/bar

 	// sleep between logs and with debug enabled, fn-test-utils will log header/footer below:
 	multiLog := `{"echoContent": "_TRX_ID_", "sleepTime": 1000, "isDebug": true}`
 	bigoutput := `{"echoContent": "_TRX_ID_", "isDebug": true, "trailerRepeat": 1000}` // 1000 trailers to exceed 2K
 	smalloutput := `{"echoContent": "_TRX_ID_", "isDebug": true, "trailerRepeat": 1}`  // 1 trailer < 2K

+	statusChecker := `{"echoContent": "_TRX_ID_", "isDebug": true, "responseCode":202, "responseContentType": "application/json; charset=utf-8"}`
+
+	fooHeader := map[string][]string{"Content-Type": {"application/hateson"}, "Test-Header": {"foo"}}
+	expFooHeaders := map[string][]string{"Content-Type": {"application/hateson"}, "Return-Header": {"foo", "bar"}}
+	expFooHeadersBody := `{"echoContent": "_TRX_ID_",
+		"expectHeaders": {
+			"Content-Type":["application/hateson"],
+			"Fn-Http-H-Test-Header":["foo"],
+			"Fn-Http-Method":["POST"],
+			"Fn-Http-Request-Url":["http://127.0.0.1:8080/t/myapp/httpstream"]
+		},
+		"returnHeaders": {
+			"Return-Header":["foo","bar"]
+		},
+		"responseContentType":"application/hateson",
+		"isDebug": true}`
+
 	testCases := []struct {
 		path               string
+		headers            map[string][]string
 		body               string
 		method             string
 		expectedCode       int
@@ -328,34 +349,47 @@ func TestTriggerRunnerExecution(t *testing.T) {
 		expectedErrSubStr  string
 		expectedLogsSubStr []string
 	}{
-		{"/t/myapp/", ok, "GET", http.StatusOK, expHeaders, "", nil},
+		{"/t/myapp/", nil, ok, "GET", http.StatusOK, expHeaders, "", nil},

-		{"/t/myapp/myhot", badHot, "GET", http.StatusBadGateway, expHeaders, "invalid http response", nil},
+		{"/t/myapp/myhot", nil, badHot, "GET", http.StatusBadGateway, expHeaders, "invalid http response", nil},
 		// hot container now back to normal:
-		{"/t/myapp/myhot", ok, "GET", http.StatusOK, expHeaders, "", nil},
+		{"/t/myapp/myhot", nil, ok, "GET", http.StatusOK, expHeaders, "", nil},

-		{"/t/myapp/myhotjason", badHot, "GET", http.StatusBadGateway, expHeaders, "invalid json response", nil},
+		{"/t/myapp/myhotjason", nil, badHot, "GET", http.StatusBadGateway, expHeaders, "invalid json response", nil},
 		// hot container now back to normal:
-		{"/t/myapp/myhotjason", ok, "GET", http.StatusOK, expHeaders, "", nil},
+		{"/t/myapp/myhotjason", nil, ok, "GET", http.StatusOK, expHeaders, "", nil},

-		{"/t/myapp/myhot", respTypeLie, "GET", http.StatusOK, expCTHeaders, "", nil},
-		{"/t/myapp/myhotjason", respTypeLie, "GET", http.StatusOK, expCTHeaders, "", nil},
-		{"/t/myapp/myhotjason", respTypeJason, "GET", http.StatusOK, expCTHeaders, "", nil},
+		{"/t/myapp/myhot", nil, respTypeLie, "GET", http.StatusOK, expCTHeaders, "", nil},
+		{"/t/myapp/myhotjason", nil, respTypeLie, "GET", http.StatusOK, expCTHeaders, "", nil},
+		{"/t/myapp/myhotjason", nil, respTypeJason, "GET", http.StatusOK, expCTHeaders, "", nil},

-		{"/t/myapp/myroute", ok, "GET", http.StatusOK, expHeaders, "", nil},
-		{"/t/myapp/myerror", crasher, "GET", http.StatusBadGateway, expHeaders, "container exit code 1", nil},
-		{"/t/myapp/mydne", ``, "GET", http.StatusNotFound, nil, "pull access denied", nil},
-		{"/t/myapp/mydnehot", ``, "GET", http.StatusNotFound, nil, "pull access denied", nil},
-		{"/t/myapp/mydneregistry", ``, "GET", http.StatusInternalServerError, nil, "connection refused", nil},
-		{"/t/myapp/myoom", oomer, "GET", http.StatusBadGateway, nil, "container out of memory", nil},
-		{"/t/myapp/myhot", multiLog, "GET", http.StatusOK, nil, "", multiLogExpectHot},
-		{"/t/myapp/", multiLog, "GET", http.StatusOK, nil, "", multiLogExpectCold},
-		{"/t/myapp/mybigoutputjson", bigoutput, "GET", http.StatusBadGateway, nil, "function response too large", nil},
-		{"/t/myapp/mybigoutputjson", smalloutput, "GET", http.StatusOK, nil, "", nil},
-		{"/t/myapp/mybigoutputhttp", bigoutput, "GET", http.StatusBadGateway, nil, "", nil},
-		{"/t/myapp/mybigoutputhttp", smalloutput, "GET", http.StatusOK, nil, "", nil},
-		{"/t/myapp/mybigoutputcold", bigoutput, "GET", http.StatusBadGateway, nil, "", nil},
-		{"/t/myapp/mybigoutputcold", smalloutput, "GET", http.StatusOK, nil, "", nil},
+		// XXX(reed): we test a lot of stuff in invoke, we really only need to test headers / status code here dude...
+		{"/t/myapp/httpstream", nil, ok, "POST", http.StatusOK, expHeaders, "", nil},
+		{"/t/myapp/httpstream", nil, statusChecker, "POST", 202, expHeaders, "", nil},
+		{"/t/myapp/httpstream", fooHeader, expFooHeadersBody, "POST", http.StatusOK, expFooHeaders, "", nil},
+		// NOTE: we can't test bad response framing anymore easily (eg invalid http response), should we even worry about it?
+		{"/t/myapp/httpstream", nil, respTypeLie, "POST", http.StatusOK, expCTHeaders, "", nil},
+		//{"/t/myapp/httpstream", nil, crasher, "POST", http.StatusBadGateway, expHeaders, "error receiving function response", nil},
+		//// XXX(reed): we could stop buffering function responses so that we can stream things?
+		//{"/t/myapp/httpstream", nil, bigoutput, "POST", http.StatusBadGateway, nil, "function response too large", nil},
+		//{"/t/myapp/httpstream", nil, smalloutput, "POST", http.StatusOK, expHeaders, "", nil},
+		//// XXX(reed): meh we really should try to get oom out, but maybe it's better left to the logs?
+		//{"/t/myapp/httpstream", nil, oomer, "POST", http.StatusBadGateway, nil, "error receiving function response", nil},
+
+		{"/t/myapp/myroute", nil, ok, "GET", http.StatusOK, expHeaders, "", nil},
+		{"/t/myapp/myerror", nil, crasher, "GET", http.StatusBadGateway, expHeaders, "container exit code 1", nil},
+		{"/t/myapp/mydne", nil, ``, "GET", http.StatusNotFound, nil, "pull access denied", nil},
+		{"/t/myapp/mydnehot", nil, ``, "GET", http.StatusNotFound, nil, "pull access denied", nil},
+		{"/t/myapp/mydneregistry", nil, ``, "GET", http.StatusInternalServerError, nil, "connection refused", nil},
+		{"/t/myapp/myoom", nil, oomer, "GET", http.StatusBadGateway, nil, "container out of memory", nil},
+		{"/t/myapp/myhot", nil, multiLog, "GET", http.StatusOK, nil, "", multiLogExpectHot},
+		{"/t/myapp/", nil, multiLog, "GET", http.StatusOK, nil, "", multiLogExpectCold},
+		{"/t/myapp/mybigoutputjson", nil, bigoutput, "GET", http.StatusBadGateway, nil, "function response too large", nil},
+		{"/t/myapp/mybigoutputjson", nil, smalloutput, "GET", http.StatusOK, nil, "", nil},
+		{"/t/myapp/mybigoutputhttp", nil, bigoutput, "GET", http.StatusBadGateway, nil, "", nil},
+		{"/t/myapp/mybigoutputhttp", nil, smalloutput, "GET", http.StatusOK, nil, "", nil},
+		{"/t/myapp/mybigoutputcold", nil, bigoutput, "GET", http.StatusBadGateway, nil, "", nil},
+		{"/t/myapp/mybigoutputcold", nil, smalloutput, "GET", http.StatusOK, nil, "", nil},
 	}

 	callIds := make([]string, len(testCases))
@@ -364,7 +398,11 @@ func TestTriggerRunnerExecution(t *testing.T) {
 		t.Run(fmt.Sprintf("Test_%d_%s", i, strings.Replace(test.path, "/", "_", -1)), func(t *testing.T) {
 			trx := fmt.Sprintf("_trx_%d_", i)
 			body := strings.NewReader(strings.Replace(test.body, "_TRX_ID_", trx, 1))
-			_, rec := routerRequest(t, srv.Router, test.method, test.path, body)
+			req := createRequest(t, test.method, test.path, body)
+			if test.headers != nil {
+				req.Header = test.headers
+			}
+			_, rec := routerRequest2(t, srv.Router, req)
 			respBytes, _ := ioutil.ReadAll(rec.Body)
 			respBody := string(respBytes)
 			maxBody := len(respBody)
@@ -372,7 +410,7 @@ func TestTriggerRunnerExecution(t *testing.T) {
 				maxBody = 1024
 			}

-			callIds[i] = rec.Header().Get("Fn_call_id")
+			callIds[i] = rec.Header().Get("Fn-Call-Id")

 			if rec.Code != test.expectedCode {
 				isFailure = true
@@ -396,10 +434,10 @@ func TestTriggerRunnerExecution(t *testing.T) {

 			if test.expectedHeaders != nil {
 				for name, header := range test.expectedHeaders {
-					if header[0] != rec.Header().Get(name) {
+					if !reflect.DeepEqual(header, rec.Header()[name]) {
 						isFailure = true
-						t.Errorf("Test %d: Expected header `%s` to be `%s` but was `%s`. body: `%s`",
-							i, name, header[0], rec.Header().Get(name), respBody)
+						t.Errorf("Test %d: Expected header `%s` to be `%v` but was `%v`. body: `%s`",
+							i, name, header, rec.Header()[name], respBody)
 					}
 				}
 			}