clean up the logging code

add limit writecloser, add closer method so we can flush logs properly,
buffer logs and stuff

it builds it works amirite

api/runner/func_logger.go

@@ -1,18 +1,18 @@
 package runner
 
 import (
-	"bufio"
+	"bytes"
-	"fmt"
+	"context"
+	"errors"
 	"io"
 
-	"context"
 	"github.com/Sirupsen/logrus"
 	"gitlab-odx.oracle.com/odx/functions/api/models"
 	"gitlab-odx.oracle.com/odx/functions/api/runner/common"
 )
 
 type FuncLogger interface {
-	Writer(ctx context.Context, appName, path, image, reqID string) io.Writer
+	Writer(ctx context.Context, appName, path, image, reqID string) io.WriteCloser
 }
 
 // FuncLogger reads STDERR output from a container and outputs it in a parsed structured log format, see: https://github.com/treeder/functions/issues/76
@@ -24,30 +24,61 @@ func NewFuncLogger(logDB models.FnLog) FuncLogger {
 	return &DefaultFuncLogger{logDB}
 }
 
-func (l *DefaultFuncLogger) persistLog(ctx context.Context, log logrus.FieldLogger, reqID, logText string) {
+type writer struct {
-	err := l.logDB.InsertLog(ctx, reqID, logText)
+	bytes.Buffer
-	if err != nil {
+
-		log.WithError(err).Println(fmt.Sprintf(
+	db      models.FnLog
-			"Unable to persist log for call %v. Error: %v", reqID, err))
+	ctx     context.Context
+	reqID   string
+	appName string
+	image   string
+	path    string
+}
+
+func (w *writer) Close() error {
+	return w.db.InsertLog(context.TODO(), w.reqID, w.String())
+}
+
+func (w *writer) Write(b []byte) (int, error) {
+	n, err := w.Buffer.Write(b)
+
+	// for now, also write to stderr so we can debug quick ;)
+	// TODO this should be a separate FuncLogger but time is running short !
+	//log := common.Logger(w.ctx)
+	//log = log.WithFields(logrus.Fields{"user_log": true, "app_name": w.appName,
+	//"path": w.path, "image": w.image, "call_id": w.reqID})
+	//log.Println(string(b))
+
+	return n, err
+}
+
+// overrides Write, keeps Close
+type limitWriter struct {
+	n, max int
+	io.WriteCloser
+}
+
+func newLimitWriter(max int, w io.WriteCloser) io.WriteCloser {
+	return &limitWriter{max: max, WriteCloser: w}
+}
+
+func (l *limitWriter) Write(b []byte) (int, error) {
+	if l.n > l.max {
+		return 0, errors.New("max log size exceeded, truncating log")
 	}
+	n, err := l.WriteCloser.Write(b)
+	l.n += n
+	return n, err
 }
 
-func (l *DefaultFuncLogger) Writer(ctx context.Context, appName, path, image, reqID string) io.Writer {
+func (l *DefaultFuncLogger) Writer(ctx context.Context, appName, path, image, reqID string) io.WriteCloser {
-	r, w := io.Pipe()
+	const MB = 1 * 1024 * 1024
-
+	return newLimitWriter(MB, &writer{
-	go func(reader io.Reader) {
+		db:      l.logDB,
-		log := common.Logger(ctx)
+		ctx:     ctx,
-		log = log.WithFields(logrus.Fields{"user_log": true, "app_name": appName,
+		appName: appName,
-			"path": path, "image": image, "call_id": reqID})
+		path:    path,
-
+		image:   image,
-		var res string
+		reqID:   reqID,
-		errMsg := "-------Unable to get full log, it's too big-------"
+	})
-		fmt.Fscanf(reader, "%v", &res)
-		if len(res) >= bufio.MaxScanTokenSize {
-			res = res[0:bufio.MaxScanTokenSize - len(errMsg)] + errMsg
-		}
-
-		l.persistLog(ctx, log, reqID, res)
-	}(r)
-	return w
 }

api/runner/runner.go

@@ -173,6 +173,7 @@ func (r *Runner) run(ctx context.Context, cfg *task.Config) (drivers.RunResult,
 	}
 
 	cfg.Stderr = r.flog.Writer(ctx, cfg.AppName, cfg.Path, cfg.Image, cfg.ID)
+	defer cfg.Stderr.Close() // TODO we should prob log this err but hey
 	if cfg.Stdout == nil {
 		cfg.Stdout = cfg.Stderr
 	}

api/runner/task/task.go

@@ -24,7 +24,7 @@ type Config struct {
 
 	Stdin  io.Reader
 	Stdout io.Writer
-	Stderr io.Writer
+	Stderr io.WriteCloser // closer for flushy poo
 }
 
 // Request stores the task to be executed, It holds in itself the channel to