From b6b9b55ca905e235b35dcb4be47b3baceaf13b06 Mon Sep 17 00:00:00 2001 From: amykang2020 Date: Thu, 7 Sep 2017 02:00:22 -0700 Subject: [PATCH] apply/make Travis's json-format branch prototype to work with latest restructured master; added StatusCode to JSONOutput server-function contract --- api/agent/protocol/factory.go | 7 ++ api/agent/protocol/json.go | 98 ++++++++++++++++++++ api/models/call.go | 2 + api/models/route.go | 2 +- docs/function-format.md | 53 +++++++---- examples/formats/json/go/.gitignore | 6 ++ examples/formats/json/go/README.md | 3 + examples/formats/json/go/func.go | 69 ++++++++++++++ examples/formats/json/go/sample.payload.json | 3 + examples/formats/json/go/test.json | 26 ++++++ 10 files changed, 250 insertions(+), 19 deletions(-) create mode 100644 api/agent/protocol/json.go create mode 100644 examples/formats/json/go/.gitignore create mode 100644 examples/formats/json/go/README.md create mode 100644 examples/formats/json/go/func.go create mode 100644 examples/formats/json/go/sample.payload.json create mode 100644 examples/formats/json/go/test.json diff --git a/api/agent/protocol/factory.go b/api/agent/protocol/factory.go index 44f0f27da..5fb5bb2e1 100644 --- a/api/agent/protocol/factory.go +++ b/api/agent/protocol/factory.go @@ -36,6 +36,7 @@ type Protocol string const ( Default Protocol = models.FormatDefault HTTP Protocol = models.FormatHTTP + JSON Protocol = models.FormatJSON Empty Protocol = "" ) @@ -45,6 +46,8 @@ func (p *Protocol) UnmarshalJSON(b []byte) error { *p = Default case HTTP: *p = HTTP + case JSON: + *p = JSON default: return errInvalidProtocol } @@ -57,6 +60,8 @@ func (p Protocol) MarshalJSON() ([]byte, error) { return []byte(Default), nil case HTTP: return []byte(HTTP), nil + case JSON: + return []byte(JSON), nil } return nil, errInvalidProtocol } @@ -67,6 +72,8 @@ func New(p Protocol, in io.Writer, out io.Reader) ContainerIO { switch p { case HTTP: return &HTTPProtocol{in, out} + case JSON: + return &JSONProtocol{in, out} case Default, Empty: return &DefaultProtocol{} } diff --git a/api/agent/protocol/json.go b/api/agent/protocol/json.go new file mode 100644 index 000000000..15b5fbc28 --- /dev/null +++ b/api/agent/protocol/json.go @@ -0,0 +1,98 @@ +package protocol + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" +) + +// JSONInput is what's sent into the function +// All HTTP request headers should be set in env +type JSONInput struct { + RequestURL string `json:"request_url"` + CallID string `json:"call_id"` + Method string `json:"method"` + Body string `json:"body"` +} + +// JSONOutput function must return this format +// StatusCode value must be a HTTP status code +type JSONOutput struct { + StatusCode int `json:"status"` + Body string `json:"body"` +} + +// JSONProtocol converts stdin/stdout streams from HTTP into JSON format. +type JSONProtocol struct { + in io.Writer + out io.Reader +} + +func (p *JSONProtocol) IsStreamable() bool { + return true +} + +func (h *JSONProtocol) Dispatch(w io.Writer, req *http.Request) error { + reqURL := req.Header.Get("REQUEST_URL") + method := req.Header.Get("METHOD") + callID := req.Header.Get("CALL_ID") + + // TODO content-length or chunked encoding + var body bytes.Buffer + if req.Body != nil { + var dest io.Writer = &body + + // TODO copy w/ ctx and check err + io.Copy(dest, req.Body) + } + + // convert to JSON func format + jin := &JSONInput{ + RequestURL: reqURL, + Method: method, + CallID: callID, + Body: body.String(), + } + b, err := json.Marshal(jin) + if err != nil { + // this shouldn't happen + return fmt.Errorf("error marshalling JSONInput: %v", err) + } + h.in.Write(b) + + // TODO: put max size on how big the response can be so we don't blow up + jout := &JSONOutput{} + dec := json.NewDecoder(h.out) + if err := dec.Decode(jout); err != nil { + // TODO: how do we get an error back to the client?? + return fmt.Errorf("error unmarshalling JSONOutput: %v", err) + } + + // res := &http.Response{} + // res.Body = strings.NewReader(jout.Body) + // TODO: shouldn't we pass back the full response object or something so we can set some things on it here? + // For instance, user could set response content type or what have you. + //io.Copy(cfg.Stdout, strings.NewReader(jout.Body)) + + if rw, ok := w.(http.ResponseWriter); ok { + b, err = json.Marshal(jout.Body) + if err != nil { + return fmt.Errorf("error unmarshalling JSONOutput.Body: %v", err) + } + rw.WriteHeader(jout.StatusCode) + rw.Write(b) // TODO timeout + } else { + // logs can just copy the full thing in there, headers and all. + b, err = json.Marshal(jout) + if err != nil { + return fmt.Errorf("error unmarshalling JSONOutput: %v", err) + } + + w.Write(b) // TODO timeout + + } + return nil + +} diff --git a/api/models/call.go b/api/models/call.go index 37591a244..9aa078781 100644 --- a/api/models/call.go +++ b/api/models/call.go @@ -18,6 +18,8 @@ const ( FormatDefault = "default" // FormatHTTP ... FormatHTTP = "http" + // FormatJSON ... + FormatJSON = "json" ) var possibleStatuses = [...]string{"delayed", "queued", "running", "success", "error", "cancelled"} diff --git a/api/models/route.go b/api/models/route.go index 74ff51af1..6a90aea3d 100644 --- a/api/models/route.go +++ b/api/models/route.go @@ -95,7 +95,7 @@ func (r *Route) Validate() error { return ErrRoutesInvalidType } - if r.Format != FormatDefault && r.Format != FormatHTTP { + if r.Format != FormatDefault && r.Format != FormatHTTP && r.Format != FormatJSON { return ErrRoutesInvalidFormat } diff --git a/docs/function-format.md b/docs/function-format.md index 651242ddb..2a54d1088 100644 --- a/docs/function-format.md +++ b/docs/function-format.md @@ -2,7 +2,7 @@ This document will describe the details of how a function works, inputs/outputs, etc. -## Formats +## Input Formats ### STDIN and Environment Variables @@ -18,9 +18,11 @@ The goals of the input format are the following: The format is still up for discussion and in order to move forward and remain flexible, it's likely we will just allow different input formats and the function creator can decide what they want, on a per function basis. Default being the simplest format to use. -#### Default I/O Format +TODO: Put common env vars here, that show up in all formats. -The default I/O format is simply the request body itself plus some environment variables. For instance, if someone were to post a JSON body, the unmodified body would be sent in via STDIN. The result comes via STDOUT. When call is done, pipes are closed and the container running the function is terminated. +#### Default Input Format + +The default format is simply the request body itself plus some environment variables. For instance, if someone were to post a JSON body, the unmodified body would be sent in via STDIN. The result comes via STDOUT. When task is done, pipes are closed and the container running the function is terminated. Pros: @@ -30,14 +32,15 @@ Cons: * Not streamable -#### HTTP I/O Format +#### HTTP Input Format `--format http` HTTP format could be a good option as it is in very common use obviously, most languages have some semi-easy way to parse it, and it's streamable. The response will look like a HTTP response. The communication is still done via stdin/stdout, but these pipes are never closed unless the container is explicitly terminated. The basic format is: Request: -``` + +```text GET / HTTP/1.1 Content-Length: 5 @@ -45,7 +48,8 @@ world ``` Response: -``` + +```text HTTP/1.1 200 OK Content-Length: 11 @@ -66,34 +70,47 @@ Cons: * Requires a parsing library or fair amount of code to parse headers properly * Double parsing - headers + body (if body is to be parsed, such as json) -#### JSON I/O Format (not implemented) +#### JSON Input Format `--format json` -The idea here is to keep the HTTP base structure, but make it a bit easier to parse by making the `request line` and `headers` a JSON struct. -Eg: +An easy to parse JSON structure. -``` +```json { - "request_url":"http://....", - "params": { - "blog_name": "yeezy" + "request_url": "http://....", + "call_id": "abc123", + "method": "GET", + "body": { + "some": "input" + } +} +{ + "request_url":"http://....", + "call_id": "edf456", + "method": "GET", + "body": { + "other": "input" } } -BLANK LINE -BODY ``` Pros: * Streamable -* Easy to parse headers +* Easy to parse Cons: -* New, unknown format +* ??? -### STDERR +## Output + +### Output back to client + +Typically JSON is the output format and is the default output, but any format can be used. + +### Logging Standard error is reserved for logging, like it was meant to be. Anything you output to STDERR will show up in the logs. And if you use a log collector like logspout, you can collect those logs in a central location. See [logging](logging.md). diff --git a/examples/formats/json/go/.gitignore b/examples/formats/json/go/.gitignore new file mode 100644 index 000000000..d450e309c --- /dev/null +++ b/examples/formats/json/go/.gitignore @@ -0,0 +1,6 @@ +vendor/ +/go +/app +/__uberscript__ + +func.yaml diff --git a/examples/formats/json/go/README.md b/examples/formats/json/go/README.md new file mode 100644 index 000000000..30a50bb82 --- /dev/null +++ b/examples/formats/json/go/README.md @@ -0,0 +1,3 @@ +# Go using JSON format + +This example uses the `json` input format. diff --git a/examples/formats/json/go/func.go b/examples/formats/json/go/func.go new file mode 100644 index 000000000..feef0fe3a --- /dev/null +++ b/examples/formats/json/go/func.go @@ -0,0 +1,69 @@ +package main + +import ( + "encoding/json" + "fmt" + "log" + "os" +) + +type Person struct { + Name string +} + +type JSONInput struct { + RequestURL string `json:"request_url"` + CallID string `json:"call_id"` + Method string `json:"method"` + Body string `json:"body"` +} + +func (a *JSONInput) String() string { + return fmt.Sprintf("request_url=%s\ncall_id=%s\nmethod=%s\n\nbody=%s", + a.RequestURL, a.CallID, a.Method, a.Body) +} + +type JSONOutput struct { + StatusCode int `json:"status"` + Body string `json:"body"` +} + +func main() { + // p := &Person{Name: "World"} + // json.Unmarshal(os.Stdin).Decode(p) + // mapD := map[string]string{"message": fmt.Sprintf("Hello %s", p.Name)} + // mapB, _ := json.Marshal(mapD) + // fmt.Println(string(mapB)) + + dec := json.NewDecoder(os.Stdin) + enc := json.NewEncoder(os.Stdout) + var loopCounter = 0 + for { + loopCounter++ + log.Println("loopCounter:", loopCounter) + + in := &JSONInput{} + if err := dec.Decode(in); err != nil { + log.Fatalln(err) + return + } + log.Println("JSONInput: ", in) + + person := Person{} + if in.Body != "" { + if err := json.Unmarshal([]byte(in.Body), &person); err != nil { + log.Fatalln(err) + } + } + + log.Println("Person: ", person) + + mapResult := map[string]string{"message": fmt.Sprintf("Hello %s", person.Name)} + out := &JSONOutput{StatusCode: 200} + b, _ := json.Marshal(mapResult) + out.Body = string(b) + if err := enc.Encode(out); err != nil { + log.Fatalln(err) + } + } +} diff --git a/examples/formats/json/go/sample.payload.json b/examples/formats/json/go/sample.payload.json new file mode 100644 index 000000000..0a3c281da --- /dev/null +++ b/examples/formats/json/go/sample.payload.json @@ -0,0 +1,3 @@ +{ + "Name": "Johnny" +} diff --git a/examples/formats/json/go/test.json b/examples/formats/json/go/test.json new file mode 100644 index 000000000..391d9b42f --- /dev/null +++ b/examples/formats/json/go/test.json @@ -0,0 +1,26 @@ +{ + "tests": [ + { + "input": { + "body": { + "name": "Johnny" + } + }, + "output": { + "body": { + "message": "Hello Johnny" + } + } + }, + { + "input": { + "body": "" + }, + "output": { + "body": { + "message": "Hello World" + } + } + } + ] +}