Merge pull request #364 from fnproject/json-format-new

[FEATURE] JSON hot format
This commit is contained in:
Reed Allman
2017-10-09 12:25:03 -07:00
committed by GitHub
26 changed files with 565 additions and 38 deletions

View File

@@ -36,6 +36,7 @@ type Protocol string
const (
Default Protocol = models.FormatDefault
HTTP Protocol = models.FormatHTTP
JSON Protocol = models.FormatJSON
Empty Protocol = ""
)
@@ -45,6 +46,8 @@ func (p *Protocol) UnmarshalJSON(b []byte) error {
*p = Default
case HTTP:
*p = HTTP
case JSON:
*p = JSON
default:
return errInvalidProtocol
}
@@ -57,6 +60,8 @@ func (p Protocol) MarshalJSON() ([]byte, error) {
return []byte(Default), nil
case HTTP:
return []byte(HTTP), nil
case JSON:
return []byte(JSON), nil
}
return nil, errInvalidProtocol
}
@@ -67,6 +72,8 @@ func New(p Protocol, in io.Writer, out io.Reader) ContainerIO {
switch p {
case HTTP:
return &HTTPProtocol{in, out}
case JSON:
return &JSONProtocol{in, out}
case Default, Empty:
return &DefaultProtocol{}
}

View File

@@ -0,0 +1,96 @@
package protocol
import (
"bytes"
"encoding/json"
"io"
"net/http"
)
// This is sent into the function
// All HTTP request headers should be set in env
type jsonio struct {
Headers http.Header `json:"headers,omitempty"`
Body string `json:"body"`
StatusCode int `json:"status_code,omitempty"`
}
// JSONProtocol converts stdin/stdout streams from HTTP into JSON format.
type JSONProtocol struct {
in io.Writer
out io.Reader
}
func (p *JSONProtocol) IsStreamable() bool {
return true
}
func writeString(err error, dst io.Writer, str string) error {
if err != nil {
return err
}
_, err = io.WriteString(dst, str)
return err
}
func (h *JSONProtocol) DumpJSON(req *http.Request) error {
stdin := json.NewEncoder(h.in)
bb := new(bytes.Buffer)
_, err := bb.ReadFrom(req.Body)
if err != nil {
return err
}
err = writeString(err, h.in, "{")
err = writeString(err, h.in, `"body":`)
if err != nil {
return err
}
err = stdin.Encode(bb.String())
err = writeString(err, h.in, ",")
err = writeString(err, h.in, `"headers":`)
if err != nil {
return err
}
err = stdin.Encode(req.Header)
err = writeString(err, h.in, "}")
return err
}
func (h *JSONProtocol) Dispatch(w io.Writer, req *http.Request) error {
err := h.DumpJSON(req)
if err != nil {
return err
}
jout := new(jsonio)
dec := json.NewDecoder(h.out)
if err := dec.Decode(jout); err != nil {
return err
}
if rw, ok := w.(http.ResponseWriter); ok {
// this has to be done for pulling out:
// - status code
// - body
// - headers
for k, vs := range jout.Headers {
for _, v := range vs {
rw.Header().Add(k, v) // on top of any specified on the route
}
}
if jout.StatusCode != 0 {
rw.WriteHeader(jout.StatusCode)
} else {
rw.WriteHeader(200)
}
_, err = io.WriteString(rw, jout.Body) // TODO timeout
if err != nil {
return err
}
} else {
// logs can just copy the full thing in there, headers and all.
err = json.NewEncoder(w).Encode(jout)
if err != nil {
return err
}
}
return nil
}

View File

@@ -0,0 +1,122 @@
package protocol
import (
"bytes"
"encoding/json"
"io"
"io/ioutil"
"net/http"
"net/url"
"reflect"
"testing"
)
type RequestData struct {
A string `json:"a"`
}
type fuckReed struct {
Body RequestData `json:"body"`
}
func TestJSONProtocolDumpJSONRequestWithData(t *testing.T) {
req := &http.Request{
Method: http.MethodPost,
URL: &url.URL{
Scheme: "http",
Host: "localhost:8080",
Path: "/v1/apps",
RawQuery: "something=something&etc=etc",
},
ProtoMajor: 1,
ProtoMinor: 1,
Header: http.Header{
"Host": []string{"localhost:8080"},
"User-Agent": []string{"curl/7.51.0"},
"Content-Type": []string{"application/json"},
},
Host: "localhost:8080",
}
var buf bytes.Buffer
rDataBefore := RequestData{A: "a"}
json.NewEncoder(&buf).Encode(rDataBefore)
req.Body = ioutil.NopCloser(&buf)
r, w := io.Pipe()
proto := JSONProtocol{w, r}
go func() {
err := proto.DumpJSON(req)
if err != nil {
t.Error(err.Error())
}
w.Close()
}()
incomingReq := new(jsonio)
bb := new(bytes.Buffer)
_, err := bb.ReadFrom(r)
if err != nil {
t.Error(err.Error())
}
err = json.Unmarshal(bb.Bytes(), incomingReq)
if err != nil {
t.Error(err.Error())
}
rDataAfter := new(RequestData)
err = json.Unmarshal([]byte(incomingReq.Body), &rDataAfter)
if err != nil {
t.Error(err.Error())
}
if rDataBefore.A != rDataAfter.A {
t.Errorf("Request data assertion mismatch: expected: %s, got %s",
rDataBefore.A, rDataAfter.A)
}
}
func TestJSONProtocolDumpJSONRequestWithoutData(t *testing.T) {
req := &http.Request{
Method: http.MethodPost,
URL: &url.URL{
Scheme: "http",
Host: "localhost:8080",
Path: "/v1/apps",
RawQuery: "something=something&etc=etc",
},
ProtoMajor: 1,
ProtoMinor: 1,
Header: http.Header{
"Host": []string{"localhost:8080"},
"User-Agent": []string{"curl/7.51.0"},
"Content-Type": []string{"application/json"},
},
Host: "localhost:8080",
}
var buf bytes.Buffer
req.Body = ioutil.NopCloser(&buf)
r, w := io.Pipe()
proto := JSONProtocol{w, r}
go func() {
err := proto.DumpJSON(req)
if err != nil {
t.Error(err.Error())
}
w.Close()
}()
incomingReq := new(jsonio)
bb := new(bytes.Buffer)
_, err := bb.ReadFrom(r)
if err != nil {
t.Error(err.Error())
}
err = json.Unmarshal(bb.Bytes(), incomingReq)
if err != nil {
t.Error(err.Error())
}
if ok := reflect.DeepEqual(req.Header, incomingReq.Headers); !ok {
t.Errorf("Request headers assertion mismatch: expected: %s, got %s",
req.Header, incomingReq.Headers)
}
}

View File

@@ -18,6 +18,8 @@ const (
FormatDefault = "default"
// FormatHTTP ...
FormatHTTP = "http"
// FormatJSON ...
FormatJSON = "json"
)
var possibleStatuses = [...]string{"delayed", "queued", "running", "success", "error", "cancelled"}

View File

@@ -95,7 +95,7 @@ func (r *Route) Validate() error {
return ErrRoutesInvalidType
}
if r.Format != FormatDefault && r.Format != FormatHTTP {
if r.Format != FormatDefault && r.Format != FormatHTTP && r.Format != FormatJSON {
return ErrRoutesInvalidFormat
}

View File

@@ -2,7 +2,7 @@
This document will describe the details of how a function works, inputs/outputs, etc.
## Formats
## I/O Formats
### STDIN and Environment Variables
@@ -10,17 +10,19 @@ While wanting to keep things simple, flexible and expandable, we decided to go b
Configuration values, environment information and other things will be passed in through environment variables.
The goals of the input format are the following:
The goals of the I/O format are the following:
* Very easy to use and parse
* Streamable for increasing performance (more than one call per container execution)
* Supports hot for increasing performance (more than one call per container execution)
* Ability to build higher level abstractions on top (ie: Lambda syntax compatible)
The format is still up for discussion and in order to move forward and remain flexible, it's likely we will just allow different input formats and the function creator can decide what they want, on a per function basis. Default being the simplest format to use.
The format is still up for discussion and in order to move forward and remain flexible, it's likely we will just allow different I/O formats and the function creator can decide what they want, on a per function basis. Default being the simplest format to use.
TODO: Put common env vars here, that show up in all formats.
#### Default I/O Format
The default I/O format is simply the request body itself plus some environment variables. For instance, if someone were to post a JSON body, the unmodified body would be sent in via STDIN. The result comes via STDOUT. When call is done, pipes are closed and the container running the function is terminated.
The default format is simply the request body itself plus some environment variables. For instance, if someone were to post a JSON body, the unmodified body would be sent in via STDIN. The result comes via STDOUT. When task is done, pipes are closed and the container running the function is terminated.
Pros:
@@ -28,16 +30,17 @@ Pros:
Cons:
* Not streamable
* Not very efficient resource utilization - one function for one event.
#### HTTP I/O Format
`--format http`
HTTP format could be a good option as it is in very common use obviously, most languages have some semi-easy way to parse it, and it's streamable. The response will look like a HTTP response. The communication is still done via stdin/stdout, but these pipes are never closed unless the container is explicitly terminated. The basic format is:
HTTP format could be a good option as it is in very common use obviously, most languages have some semi-easy way to parse it, and it's supports hot format. The response will look like a HTTP response. The communication is still done via stdin/stdout, but these pipes are never closed unless the container is explicitly terminated. The basic format is:
Request:
```
```text
GET / HTTP/1.1
Content-Length: 5
@@ -45,7 +48,8 @@ world
```
Response:
```
```text
HTTP/1.1 200 OK
Content-Length: 11
@@ -58,7 +62,7 @@ The header keys and values would be populated with information about the functio
Pros:
* Streamable
* Supports streaming
* Common format
Cons:
@@ -66,34 +70,57 @@ Cons:
* Requires a parsing library or fair amount of code to parse headers properly
* Double parsing - headers + body (if body is to be parsed, such as json)
#### JSON I/O Format (not implemented)
#### JSON I/O Format
`--format json`
The idea here is to keep the HTTP base structure, but make it a bit easier to parse by making the `request line` and `headers` a JSON struct.
Eg:
Fn accepts request data of the following format:
```
```json
{
"request_url":"http://....",
"params": {
"blog_name": "yeezy"
"some": "input"
}
}
BLANK LINE
BODY
```
Internally function receives data in following format:
```json
{
"body": "{\"some\":\"input\"}\n",
"headers": {
"yo": ["dawg"]
}
}
```
Function's output format should have following format:
```json
{
"status_code": 200,
"body": "...",
"headeres": {
"A": ["b"]
}
}
```
At client side user will receive HTTP response with HTTP headers, status code and the body from taken from function's response.
Pros:
* Streamable
* Easy to parse headers
* Supports hot format
* Easy to parse
Cons:
* New, unknown format
* Not streamable
### STDERR
## Output
### Output back to client
Typically JSON is the output format and is the default output, but any format can be used.
### Logging
Standard error is reserved for logging, like it was meant to be. Anything you output to STDERR will show up in the logs. And if you use a log
collector like logspout, you can collect those logs in a central location. See [logging](logging.md).

View File

@@ -0,0 +1,8 @@
FROM fnproject/go:dev as build-stage
WORKDIR /function
ADD . /src
RUN cd /src && go build -o func
FROM fnproject/go
WORKDIR /function
COPY --from=build-stage /src/func /function/
ENTRYPOINT ["./func"]

View File

@@ -1,7 +1,7 @@
name: fnproject/hotfn-py
name: fnproject/hot-http-go
version: 0.0.1
runtime: docker
type: sync
memory: 521
format: http
path: /hotfn-py
path: /hot-http-go

View File

@@ -1,9 +1,8 @@
FROM jjanzic/docker-python3-opencv
FROM python:3.6.2
RUN mkdir /code
ADD . /code/
WORKDIR /code
RUN pip3 install -r requirements.txt
WORKDIR /code/
ENTRYPOINT ["python3", "func.py"]

View File

@@ -0,0 +1,7 @@
name: fnproject/hot-http-python
version: 0.0.1
runtime: docker
type: sync
memory: 521
format: http
path: /hot-http-python

View File

@@ -0,0 +1,8 @@
FROM fnproject/go:dev as build-stage
WORKDIR /function
ADD . /src
RUN cd /src && go build -o func
FROM fnproject/go
WORKDIR /function
COPY --from=build-stage /src/func /function/
ENTRYPOINT ["./func"]

View File

@@ -0,0 +1,3 @@
# Go using JSON format
This example uses the `json` input format.

View File

@@ -0,0 +1,68 @@
package main
import (
"bytes"
"encoding/json"
"fmt"
"log"
"net/http"
"os"
"strconv"
)
type Person struct {
Name string `json:"name"`
}
type JSON struct {
Headers http.Header `json:"headers"`
Body string `json:"body,omitempty"`
StatusCode int `json:"status,omitempty"`
}
func main() {
stdin := json.NewDecoder(os.Stdin)
stdout := json.NewEncoder(os.Stdout)
stderr := json.NewEncoder(os.Stderr)
for {
in := &JSON{}
err := stdin.Decode(in)
if err != nil {
log.Fatalf("Unable to decode incoming data: %s", err.Error())
fmt.Fprintf(os.Stderr, err.Error())
}
person := Person{}
stderr.Encode(in.Body)
if len(in.Body) != 0 {
if err := json.NewDecoder(bytes.NewReader([]byte(in.Body))).Decode(&person); err != nil {
log.Fatalf("Unable to decode Person object data: %s", err.Error())
fmt.Fprintf(os.Stderr, err.Error())
}
}
if person.Name == "" {
person.Name = "World"
}
mapResult := map[string]string{"message": fmt.Sprintf("Hello %s", person.Name)}
b, err := json.Marshal(mapResult)
if err != nil {
log.Fatalf("Unable to marshal JSON response body: %s", err.Error())
fmt.Fprintf(os.Stderr, err.Error())
}
h := http.Header{}
h.Set("Content-Type", "application/json")
h.Set("Content-Length", strconv.Itoa(len(b)))
out := &JSON{
StatusCode: http.StatusOK,
Body: string(b),
Headers: h,
}
stderr.Encode(out)
if err := stdout.Encode(out); err != nil {
log.Fatalf("Unable to encode JSON response: %s", err.Error())
fmt.Fprintf(os.Stderr, err.Error())
}
}
}

View File

@@ -0,0 +1,7 @@
name: fnproject/hot-json-go
version: 0.0.1
runtime: docker
type: sync
memory: 256
format: json
path: /hot-json-go

View File

@@ -0,0 +1,3 @@
{
"name": "Johnny"
}

View File

@@ -0,0 +1,26 @@
{
"tests": [
{
"input": {
"body": {
"name": "Johnny"
}
},
"output": {
"body": {
"message": "Hello Johnny"
}
}
},
{
"input": {
"body": ""
},
"output": {
"body": {
"message": "Hello World"
}
}
}
]
}

View File

@@ -1,6 +0,0 @@
name: hotfunction-http
version: 0.0.10
runtime: go
entrypoint: ./func
format: http
path: /hotfn-go

View File

@@ -0,0 +1,8 @@
FROM fnproject/go:dev as build-stage
WORKDIR /function
ADD . /src
RUN cd /src && go build -o func
FROM fnproject/go
WORKDIR /function
COPY --from=build-stage /src/func /function/
ENTRYPOINT ["./func"]

View File

@@ -0,0 +1,68 @@
package main
import (
"bytes"
"encoding/json"
"fmt"
"log"
"net/http"
"os"
"strconv"
)
type Person struct {
Name string `json:"name"`
}
type JSON struct {
Headers http.Header `json:"headers"`
Body string `json:"body,omitempty"`
StatusCode int `json:"status,omitempty"`
}
func main() {
stdin := json.NewDecoder(os.Stdin)
stdout := json.NewEncoder(os.Stdout)
stderr := json.NewEncoder(os.Stderr)
for {
in := &JSON{}
err := stdin.Decode(in)
if err != nil {
log.Fatalf("Unable to decode incoming data: %s", err.Error())
fmt.Fprintf(os.Stderr, err.Error())
}
person := Person{}
stderr.Encode(in.Body)
if len(in.Body) != 0 {
if err := json.NewDecoder(bytes.NewReader([]byte(in.Body))).Decode(&person); err != nil {
log.Fatalf("Unable to decode Person object data: %s", err.Error())
fmt.Fprintf(os.Stderr, err.Error())
}
}
if person.Name == "" {
person.Name = "World"
}
mapResult := map[string]string{"message": fmt.Sprintf("Hello %s", person.Name)}
b, err := json.Marshal(mapResult)
if err != nil {
log.Fatalf("Unable to marshal JSON response body: %s", err.Error())
fmt.Fprintf(os.Stderr, err.Error())
}
h := http.Header{}
h.Set("Content-Type", "application/json")
h.Set("Content-Length", strconv.Itoa(len(b)))
out := &JSON{
StatusCode: http.StatusOK,
Body: string(b),
Headers: h,
}
stderr.Encode(out)
if err := stdout.Encode(out); err != nil {
log.Fatalf("Unable to encode JSON response: %s", err.Error())
fmt.Fprintf(os.Stderr, err.Error())
}
}
}

View File

@@ -0,0 +1,72 @@
package tests
import (
"bytes"
"encoding/json"
"net/url"
"path"
"strconv"
"strings"
"testing"
)
type JSONResponse struct {
Message string `json:"message"`
}
func TestFnFormats(t *testing.T) {
t.Run("test-json-format", func(t *testing.T) {
t.Parallel()
s := SetupDefaultSuite()
// TODO(treeder): put image in fnproject @ dockerhub
image := "denismakogon/test-hot-json-go:0.0.1"
format := "json"
route := "/test-hot-json-go"
CreateApp(t, s.Context, s.Client, s.AppName, map[string]string{})
CreateRoute(t, s.Context, s.Client, s.AppName, route, image, "sync",
format, s.RouteConfig, s.RouteHeaders)
u := url.URL{
Scheme: "http",
Host: Host(),
}
u.Path = path.Join(u.Path, "r", s.AppName, s.RoutePath)
b, _ := json.Marshal(&struct {
Name string `json:"name"`
}{
Name: "Jimmy",
})
content := bytes.NewBuffer(b)
output := &bytes.Buffer{}
headers, err := CallFN(u.String(), content, output, "POST", []string{})
if err != nil {
t.Errorf("Got unexpected error: %v", err)
}
msg := &JSONResponse{}
json.Unmarshal(output.Bytes(), msg)
expectedOutput := "Hello Jimmy"
if !strings.Contains(expectedOutput, msg.Message) {
t.Errorf("Assertion error.\n\tExpected: %v\n\tActual: %v", expectedOutput, output.String())
}
expectedHeaderNames := []string{"Content-Type", "Content-Length"}
expectedHeaderValues := []string{"application/json; charset=utf-8", strconv.Itoa(output.Len())}
for i, name := range expectedHeaderNames {
actual := headers.Get(name)
expected := expectedHeaderValues[i]
if !strings.Contains(expected, actual) {
t.Errorf("HTTP header assertion error for %v."+
"\n\tExpected: %v\n\tActual: %v", name, expected, actual)
}
}
DeleteApp(t, s.Context, s.Client, s.AppName)
})
}

View File

@@ -89,7 +89,7 @@ func assertRouteFields(t *testing.T, routeObject *models.Route, path, image, rou
}
func createRoute(ctx context.Context, fnclient *client.Fn, appName, image, routePath, routeType string, routeConfig map[string]string, headers map[string][]string) (*routes.PostAppsAppRoutesOK, error) {
func createRoute(ctx context.Context, fnclient *client.Fn, appName, image, routePath, routeType, routeFormat string, routeConfig map[string]string, headers map[string][]string) (*routes.PostAppsAppRoutesOK, error) {
cfg := &routes.PostAppsAppRoutesParams{
App: appName,
Body: &models.RouteWrapper{
@@ -99,6 +99,7 @@ func createRoute(ctx context.Context, fnclient *client.Fn, appName, image, route
Image: image,
Path: routePath,
Type: routeType,
Format: routeFormat,
},
},
Context: ctx,
@@ -119,7 +120,7 @@ func createRoute(ctx context.Context, fnclient *client.Fn, appName, image, route
}
func CreateRoute(t *testing.T, ctx context.Context, fnclient *client.Fn, appName, routePath, image, routeType, routeFormat string, routeConfig map[string]string, headers map[string][]string) {
routeResponse, err := createRoute(ctx, fnclient, appName, image, routePath, routeType, routeConfig, headers)
routeResponse, err := createRoute(ctx, fnclient, appName, image, routePath, routeType, routeFormat, routeConfig, headers)
CheckRouteResponseError(t, err)
assertRouteFields(t, routeResponse.Payload.Route, routePath, image, routeType, routeFormat)

View File

@@ -16,7 +16,7 @@ func TestRoutes(t *testing.T) {
t.Parallel()
s := SetupDefaultSuite()
CreateApp(t, s.Context, s.Client, s.AppName, map[string]string{})
_, err := createRoute(s.Context, s.Client, s.AppName, s.RoutePath, s.Image, "",
_, err := createRoute(s.Context, s.Client, s.AppName, s.RoutePath, s.Image, "", s.Format,
s.RouteConfig, s.RouteHeaders)
if err == nil {
t.Errorf("Should fail with Invalid route Type.")
@@ -128,7 +128,8 @@ func TestRoutes(t *testing.T) {
CreateRoute(t, s.Context, s.Client, s.AppName, s.RoutePath, s.Image, s.RouteType,
s.Format, s.RouteConfig, s.RouteHeaders)
_, err := createRoute(s.Context, s.Client, s.AppName, s.Image, s.RoutePath, newRouteType, s.RouteConfig, s.RouteHeaders)
_, err := createRoute(s.Context, s.Client, s.AppName, s.Image, s.RoutePath,
newRouteType, s.Format, s.RouteConfig, s.RouteHeaders)
if err == nil {
t.Errorf("Route duplicate error should appear, but it didn't")
}