fn: I/O related improvements (#809)

*) I/O protocol parse issues should shutdown the container as the container
goes to inconsistent state between calls. (eg. next call may receive previous
calls left overs.)
*) Move ghost read/write code into io_utils in common.
*) Clean unused error from docker Wait()
*) We can catch one case in JSON, if there's remaining unparsed data in
decoder buffer, we can shut the container
*) stdout/stderr when container is not handling a request are now blocked if freezer is also enabled.
*) if a fatal err is set for slot, we do not requeue it and proceed to shutdown
*) added a test function for a few cases with freezer strict behavior

api/agent/protocol/factory.go

@@ -13,6 +13,8 @@ import (
 
 var errInvalidProtocol = errors.New("Invalid Protocol")
 
+var ErrExcessData = errors.New("Excess data in stream")
+
 type errorProto struct {
 	error
 }

api/agent/protocol/json.go

@@ -100,7 +100,8 @@ func (h *JSONProtocol) Dispatch(ctx context.Context, ci CallInfo, w io.Writer) e
 
 	_, span = trace.StartSpan(ctx, "dispatch_json_read_response")
 	var jout jsonOut
-	err = json.NewDecoder(h.out).Decode(&jout)
+	decoder := json.NewDecoder(h.out)
+	err = decoder.Decode(&jout)
 	span.End()
 	if err != nil {
 		return models.NewAPIError(http.StatusBadGateway, fmt.Errorf("invalid json response from function err: %v", err))
@@ -112,7 +113,8 @@ func (h *JSONProtocol) Dispatch(ctx context.Context, ci CallInfo, w io.Writer) e
 	rw, ok := w.(http.ResponseWriter)
 	if !ok {
 		// logs can just copy the full thing in there, headers and all.
-		return json.NewEncoder(w).Encode(jout)
+		err := json.NewEncoder(w).Encode(jout)
+		return h.isExcessData(err, decoder)
 	}
 
 	// this has to be done for pulling out:
@@ -141,5 +143,16 @@ func (h *JSONProtocol) Dispatch(ctx context.Context, ci CallInfo, w io.Writer) e
 	}
 
 	_, err = io.WriteString(rw, jout.Body)
+	return h.isExcessData(err, decoder)
+}
+
+func (h *JSONProtocol) isExcessData(err error, decoder *json.Decoder) error {
+	if err == nil {
+		// Now check for excess output, if this is the case, we can be certain that the next request will fail.
+		tmp, ok := decoder.Buffered().(*bytes.Reader)
+		if ok && tmp.Len() > 0 {
+			return ErrExcessData
+		}
+	}
 	return err
 }