apply/make Travis's json-format branch prototype to work with latest restructured master; added StatusCode to JSONOutput server-function contract

api/agent/protocol/factory.go

@@ -36,6 +36,7 @@ type Protocol string
 const (
 	Default Protocol = models.FormatDefault
 	HTTP    Protocol = models.FormatHTTP
+	JSON    Protocol = models.FormatJSON
 	Empty   Protocol = ""
 )
 
@@ -45,6 +46,8 @@ func (p *Protocol) UnmarshalJSON(b []byte) error {
 		*p = Default
 	case HTTP:
 		*p = HTTP
+	case JSON:
+		*p = JSON 
 	default:
 		return errInvalidProtocol
 	}
@@ -57,6 +60,8 @@ func (p Protocol) MarshalJSON() ([]byte, error) {
 		return []byte(Default), nil
 	case HTTP:
 		return []byte(HTTP), nil
+	case JSON:
+		return []byte(JSON), nil
 	}
 	return nil, errInvalidProtocol
 }
@@ -67,6 +72,8 @@ func New(p Protocol, in io.Writer, out io.Reader) ContainerIO {
 	switch p {
 	case HTTP:
 		return &HTTPProtocol{in, out}
+	case JSON:
+		return &JSONProtocol{in, out}
 	case Default, Empty:
 		return &DefaultProtocol{}
 	}

api/agent/protocol/json.go

@@ -0,0 +1,98 @@
+package protocol
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+)
+
+// JSONInput is what's sent into the function
+// All HTTP request headers should be set in env
+type JSONInput struct {
+	RequestURL string `json:"request_url"`
+	CallID     string `json:"call_id"`
+	Method     string `json:"method"`
+	Body       string `json:"body"`
+}
+
+// JSONOutput function must return this format
+// StatusCode value must be a HTTP status code
+type JSONOutput struct {
+	StatusCode int    `json:"status"`
+	Body       string `json:"body"`
+}
+
+// JSONProtocol converts stdin/stdout streams from HTTP into JSON format.
+type JSONProtocol struct {
+	in  io.Writer
+	out io.Reader
+}
+
+func (p *JSONProtocol) IsStreamable() bool {
+	return true
+}
+
+func (h *JSONProtocol) Dispatch(w io.Writer, req *http.Request) error {
+	reqURL := req.Header.Get("REQUEST_URL")
+	method := req.Header.Get("METHOD")
+	callID := req.Header.Get("CALL_ID")
+
+	// TODO content-length or chunked encoding
+	var body bytes.Buffer
+	if req.Body != nil {
+		var dest io.Writer = &body
+
+		// TODO copy w/ ctx and check err
+		io.Copy(dest, req.Body)
+	}
+
+	// convert to JSON func format
+	jin := &JSONInput{
+		RequestURL: reqURL,
+		Method:     method,
+		CallID:     callID,
+		Body:       body.String(),
+	}
+	b, err := json.Marshal(jin)
+	if err != nil {
+		// this shouldn't happen
+		return fmt.Errorf("error marshalling JSONInput: %v", err)
+	}
+	h.in.Write(b)
+
+	// TODO: put max size on how big the response can be so we don't blow up
+	jout := &JSONOutput{}
+	dec := json.NewDecoder(h.out)
+	if err := dec.Decode(jout); err != nil {
+		// TODO: how do we get an error back to the client??
+		return fmt.Errorf("error unmarshalling JSONOutput: %v", err)
+	}
+
+	// res := &http.Response{}
+	// res.Body = strings.NewReader(jout.Body)
+	// TODO: shouldn't we pass back the full response object or something so we can set some things on it here?
+	// For instance, user could set response content type or what have you.
+	//io.Copy(cfg.Stdout, strings.NewReader(jout.Body))
+
+	if rw, ok := w.(http.ResponseWriter); ok {
+		b, err = json.Marshal(jout.Body)
+		if err != nil {
+			return fmt.Errorf("error unmarshalling JSONOutput.Body: %v", err)
+		}
+		rw.WriteHeader(jout.StatusCode)
+		rw.Write(b) // TODO timeout
+	} else {
+		// logs can just copy the full thing in there, headers and all.
+		b, err = json.Marshal(jout)
+		if err != nil {
+			return fmt.Errorf("error unmarshalling JSONOutput: %v", err)
+		}
+
+		w.Write(b) // TODO timeout
+
+	}
+	return nil
+
+}

api/models/call.go

@@ -18,6 +18,8 @@ const (
 	FormatDefault = "default"
 	// FormatHTTP ...
 	FormatHTTP = "http"
+	// FormatJSON ...
+	FormatJSON = "json"
 )
 
 var possibleStatuses = [...]string{"delayed", "queued", "running", "success", "error", "cancelled"}

api/models/route.go

@@ -95,7 +95,7 @@ func (r *Route) Validate() error {
 		return ErrRoutesInvalidType
 	}
 
-	if r.Format != FormatDefault && r.Format != FormatHTTP {
+	if r.Format != FormatDefault && r.Format != FormatHTTP && r.Format != FormatJSON {
 		return ErrRoutesInvalidFormat
 	}
 

docs/function-format.md

@@ -2,7 +2,7 @@
 
 This document will describe the details of how a function works, inputs/outputs, etc.
 
-## Formats
+## Input Formats
 
 ### STDIN and Environment Variables
 
@@ -18,9 +18,11 @@ The goals of the input format are the following:
 
 The format is still up for discussion and in order to move forward and remain flexible, it's likely we will just allow different input formats and the function creator can decide what they want, on a per function basis. Default being the simplest format to use.
 
-#### Default I/O Format
+TODO: Put common env vars here, that show up in all formats.
 
-The default I/O format is simply the request body itself plus some environment variables. For instance, if someone were to post a JSON body, the unmodified body would be sent in via STDIN. The result comes via STDOUT. When call is done, pipes are closed and the container running the function is terminated.
+#### Default Input Format
+
+The default format is simply the request body itself plus some environment variables. For instance, if someone were to post a JSON body, the unmodified body would be sent in via STDIN. The result comes via STDOUT. When task is done, pipes are closed and the container running the function is terminated.
 
 Pros:
 
@@ -30,14 +32,15 @@ Cons:
 
 * Not streamable
 
-#### HTTP I/O Format
+#### HTTP Input Format
 
 `--format http`
 
 HTTP format could be a good option as it is in very common use obviously, most languages have some semi-easy way to parse it, and it's streamable. The response will look like a HTTP response. The communication is still done via stdin/stdout, but these pipes are never closed unless the container is explicitly terminated. The basic format is:
 
 Request:
-```
+
+```text
 GET / HTTP/1.1
 Content-Length: 5
 
@@ -45,7 +48,8 @@ world
 ```
 
 Response:
-```
+
+```text
 HTTP/1.1 200 OK
 Content-Length: 11
 
@@ -66,34 +70,47 @@ Cons:
 * Requires a parsing library or fair amount of code to parse headers properly
 * Double parsing - headers + body (if body is to be parsed, such as json)
 
-#### JSON I/O Format (not implemented)
+#### JSON Input Format
 
 `--format json`
 
-The idea here is to keep the HTTP base structure, but make it a bit easier to parse by making the `request line` and `headers` a JSON struct.
-Eg:
+An easy to parse JSON structure.
 
-```
+```json
 {
-  "request_url":"http://....",
-  "params": {
-    "blog_name": "yeezy"
+  "request_url": "http://....",
+  "call_id": "abc123",
+  "method": "GET",
+  "body": {
+    "some": "input"
+  }
+}
+{
+  "request_url":"http://....",
+  "call_id": "edf456",
+  "method": "GET",
+  "body": {
+    "other": "input"
   }
 }
-BLANK LINE
-BODY
 ```
 
 Pros:
 
 * Streamable
-* Easy to parse headers
+* Easy to parse
 
 Cons:
 
-* New, unknown format
+* ???
 
-### STDERR
+## Output
+
+### Output back to client
+
+Typically JSON is the output format and is the default output, but any format can be used.
+
+### Logging
 
 Standard error is reserved for logging, like it was meant to be. Anything you output to STDERR will show up in the logs. And if you use a log
 collector like logspout, you can collect those logs in a central location. See [logging](logging.md).

examples/formats/json/go/.gitignore

@@ -0,0 +1,6 @@
+vendor/
+/go
+/app
+/__uberscript__
+
+func.yaml

examples/formats/json/go/README.md

@@ -0,0 +1,3 @@
+# Go using JSON format
+
+This example uses the `json` input format.

examples/formats/json/go/func.go

@@ -0,0 +1,69 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"log"
+	"os"
+)
+
+type Person struct {
+	Name string
+}
+
+type JSONInput struct {
+	RequestURL string `json:"request_url"`
+	CallID     string `json:"call_id"`
+	Method     string `json:"method"`
+	Body       string `json:"body"`
+}
+
+func (a *JSONInput) String() string {
+	return fmt.Sprintf("request_url=%s\ncall_id=%s\nmethod=%s\n\nbody=%s",
+		a.RequestURL, a.CallID, a.Method, a.Body)
+}
+
+type JSONOutput struct {
+	StatusCode int `json:"status"`
+	Body string `json:"body"`
+}
+
+func main() {
+	// p := &Person{Name: "World"}
+	// json.Unmarshal(os.Stdin).Decode(p)
+	// mapD := map[string]string{"message": fmt.Sprintf("Hello %s", p.Name)}
+	// mapB, _ := json.Marshal(mapD)
+	// fmt.Println(string(mapB))
+
+	dec := json.NewDecoder(os.Stdin)
+	enc := json.NewEncoder(os.Stdout)
+	var loopCounter = 0
+	for {
+		loopCounter++
+		log.Println("loopCounter:", loopCounter)
+
+		in := &JSONInput{}
+		if err := dec.Decode(in); err != nil {
+			log.Fatalln(err)
+			return
+		}
+		log.Println("JSONInput: ", in)
+
+		person := Person{}
+                if in.Body != "" {
+			if err := json.Unmarshal([]byte(in.Body), &person); err != nil {
+				log.Fatalln(err)
+			}
+                }
+
+		log.Println("Person: ", person)
+
+		mapResult := map[string]string{"message": fmt.Sprintf("Hello %s", person.Name)}
+		out := &JSONOutput{StatusCode: 200}
+		b, _ := json.Marshal(mapResult)
+		out.Body = string(b)
+		if err := enc.Encode(out); err != nil {
+			log.Fatalln(err)
+		}
+	}
+}

examples/formats/json/go/sample.payload.json

@@ -0,0 +1,3 @@
+{
+    "Name": "Johnny"
+}

examples/formats/json/go/test.json

@@ -0,0 +1,26 @@
+{
+    "tests": [
+        {
+            "input": {
+                "body": {
+                    "name": "Johnny"
+                }
+            },
+            "output": {
+                "body": {
+                    "message": "Hello Johnny"
+                }
+            }
+        },
+        {
+            "input": {
+                "body": ""
+            },
+            "output": {
+                "body": {
+                    "message": "Hello World"
+                }
+            }
+        }
+    ]
+}