Merge branch 'func_logs2' into 'master'

Func logs feature

See merge request !66

Makefile

@@ -1,5 +1,5 @@
 # Just builds
-.PHONY: all test dep build
+.PHONY: all test dep build test-log-datastore
 
 dep:
 	dep ensure
@@ -13,6 +13,9 @@ test:
 test-datastore:
 	cd api/datastore && go test -v ./...
 
+test-log-datastore:
+	cd api/logs && go test -v ./...
+
 test-build-arm:
 	GOARCH=arm GOARM=5 $(MAKE) build
 	GOARCH=arm GOARM=6 $(MAKE) build

api/datastore/bolt/bolt.go

@@ -53,7 +53,8 @@ func New(url *url.URL) (models.Datastore, error) {
 	extrasBucketName := []byte(bucketPrefix + "extras") // todo: think of a better name
 	callsBucketName := []byte(bucketPrefix + "calls")
 	err = db.Update(func(tx *bolt.Tx) error {
-		for _, name := range [][]byte{routesBucketName, appsBucketName, logsBucketName, extrasBucketName, callsBucketName} {
+		for _, name := range [][]byte{routesBucketName, appsBucketName, logsBucketName,
+			extrasBucketName, callsBucketName} {
 			_, err := tx.CreateBucketIfNotExists(name)
 			if err != nil {
 				log.WithError(err).WithFields(logrus.Fields{"name": name}).Error("create bucket")

api/datastore/internal/datastoretest/test.go

@@ -10,8 +10,6 @@ import (
 	"gitlab-odx.oracle.com/odx/functions/api/models"
 
 	"net/http"
-	"net/url"
-	"os"
 	"reflect"
 	"time"
 
@@ -30,14 +28,6 @@ func setLogBuffer() *bytes.Buffer {
 	return &buf
 }
 
-func GetContainerHostIP() string {
-	dockerHost := os.Getenv("DOCKER_HOST")
-	if dockerHost == "" {
-		return "127.0.0.1"
-	}
-	parts, _ := url.Parse(dockerHost)
-	return parts.Hostname()
-}
 
 func Test(t *testing.T, ds models.Datastore) {
 	buf := setLogBuffer()

api/datastore/internal/datastoreutil/shared.go

@@ -15,6 +15,13 @@ type RowScanner interface {
 	Scan(dest ...interface{}) error
 }
 
+func ScanLog(scanner RowScanner, log *models.FnCallLog) error {
+	return scanner.Scan(
+		&log.CallID,
+		&log.Log,
+	)
+}
+
 func ScanRoute(scanner RowScanner, route *models.Route) error {
 	var headerStr string
 	var configStr string

api/datastore/mock.go

@@ -15,10 +15,10 @@ type mock struct {
 }
 
 func NewMock() models.Datastore {
-	return NewMockInit(nil, nil, nil)
+	return NewMockInit(nil, nil, nil, nil)
 }
 
-func NewMockInit(apps models.Apps, routes models.Routes, calls models.FnCalls) models.Datastore {
+func NewMockInit(apps models.Apps, routes models.Routes, calls models.FnCalls, logs []*models.FnCallLog) models.Datastore {
 	if apps == nil {
 		apps = models.Apps{}
 	}
@@ -28,6 +28,9 @@ func NewMockInit(apps models.Apps, routes models.Routes, calls models.FnCalls) m
 	if calls == nil {
 		calls = models.FnCalls{}
 	}
+	if logs == nil {
+		logs = []*models.FnCallLog{}
+	}
 	return datastoreutil.NewValidator(&mock{apps, routes, calls, make(map[string][]byte)})
 }
 

api/datastore/mysql/mysql.go

@@ -63,7 +63,8 @@ type MySQLDatastore struct {
 New creates a new MySQL Datastore.
 */
 func New(url *url.URL) (models.Datastore, error) {
-	tables := []string{routesTableCreate, appsTableCreate, extrasTableCreate, callTableCreate}
+	tables := []string{routesTableCreate, appsTableCreate,
+		extrasTableCreate, callTableCreate}
 	dialect := "mysql"
 	sqlDatastore := &MySQLDatastore{}
 	dataSourceName := fmt.Sprintf("%s@%s%s", url.User.String(), url.Host, url.Path)

api/datastore/postgres/postgres.go

@@ -53,6 +53,7 @@ const callsTableCreate = `CREATE TABLE IF NOT EXISTS calls (
 
 const callSelector = `SELECT id, created_at, started_at, completed_at, status, app_name, path FROM calls`
 
+
 type PostgresDatastore struct {
 	db *sql.DB
 }

api/datastore/redis/redis.go

@@ -317,11 +317,6 @@ func applyCallFilter(call *models.FnCall, filter *models.CallFilter) bool {
 }
 
 func (ds *RedisDataStore) InsertTask(ctx context.Context, task *models.Task) error {
-	_, err := ds.conn.Do("HEXISTS", "calls", task.ID)
-	if err != nil {
-		return err
-	}
-
 	taskBytes, err := json.Marshal(task)
 	if err != nil {
 		return err

api/logs/bolt.go

@@ -0,0 +1,128 @@
+package logs
+
+import (
+	"encoding/json"
+	"net/url"
+	"os"
+	"path/filepath"
+	"time"
+
+	"context"
+
+	"github.com/Sirupsen/logrus"
+	"github.com/boltdb/bolt"
+	"gitlab-odx.oracle.com/odx/functions/api/models"
+)
+
+
+type BoltLogDatastore struct {
+	callLogsBucket []byte
+	db           *bolt.DB
+	log          logrus.FieldLogger
+	datastore    models.Datastore
+}
+
+
+func NewBolt(url *url.URL) (models.FnLog, error) {
+	dir := filepath.Dir(url.Path)
+	log := logrus.WithFields(logrus.Fields{"logdb": url.Scheme, "dir": dir})
+	err := os.MkdirAll(dir, 0755)
+	if err != nil {
+		log.WithError(err).Errorln("Could not create data directory for log.db")
+		return nil, err
+	}
+	log.WithFields(logrus.Fields{"path": url.Path}).Debug("Creating bolt log.db")
+	db, err := bolt.Open(url.Path, 0655, &bolt.Options{Timeout: 1 * time.Second})
+	if err != nil {
+		log.WithError(err).Errorln("Error on bolt.Open")
+		return nil, err
+	}
+	// I don't think we need a prefix here do we? Made it blank. If we do, we should call the query param "prefix" instead of bucket.
+	bucketPrefix := ""
+	if url.Query()["bucket"] != nil {
+		bucketPrefix = url.Query()["bucket"][0]
+	}
+	callLogsBucketName := []byte(bucketPrefix + "call_logs")
+	err = db.Update(func(tx *bolt.Tx) error {
+		for _, name := range [][]byte{callLogsBucketName} {
+			_, err := tx.CreateBucketIfNotExists(name)
+			if err != nil {
+				log.WithError(err).WithFields(logrus.Fields{"name": name}).Error("create bucket")
+				return err
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		log.WithError(err).Errorln("Error creating bolt buckets")
+		return nil, err
+	}
+
+	fnl := &BoltLogDatastore{
+		callLogsBucket: callLogsBucketName,
+		db:           db,
+		log:          log,
+	}
+	log.WithFields(logrus.Fields{"prefix": bucketPrefix, "file": url.Path}).Debug("BoltDB initialized")
+
+	return NewValidator(fnl), nil
+}
+
+func (fnl *BoltLogDatastore) InsertLog(ctx context.Context, callID string, callLog string) error {
+	log := &models.FnCallLog{
+		CallID: callID,
+		Log: callLog,
+	}
+	id := []byte(callID)
+	err := fnl.db.Update(
+		func(tx *bolt.Tx) error {
+			bIm := tx.Bucket(fnl.callLogsBucket)
+			buf, err := json.Marshal(log)
+			if err != nil {
+				return err
+			}
+			err = bIm.Put(id, buf)
+			if err != nil {
+				return err
+			}
+			return nil
+		})
+
+	return err
+}
+
+func (fnl *BoltLogDatastore) GetLog(ctx context.Context, callID string) (*models.FnCallLog, error) {
+	var res *models.FnCallLog
+	err := fnl.db.View(func(tx *bolt.Tx) error {
+		b := tx.Bucket(fnl.callLogsBucket)
+		v := b.Get([]byte(callID))
+		if v != nil {
+			fnCall := &models.FnCallLog{}
+			err := json.Unmarshal(v, fnCall)
+			if err != nil {
+				return nil
+			}
+			res = fnCall
+		} else {
+			return models.ErrCallLogNotFound
+		}
+		return nil
+	})
+	return res, err
+}
+
+func (fnl *BoltLogDatastore) DeleteLog(ctx context.Context, callID string) error {
+	_, err := fnl.GetLog(ctx, callID)
+	//means object does not exist
+	if err != nil {
+		return nil
+	}
+
+	id := []byte(callID)
+	err = fnl.db.Update(func(tx *bolt.Tx) error {
+		bIm := tx.Bucket(fnl.callLogsBucket)
+		err := bIm.Delete(id)
+		return err
+	})
+	return err
+}

api/logs/bolt_test.go

@@ -0,0 +1,34 @@
+package logs
+
+import (
+	"net/url"
+	"os"
+	"testing"
+
+	"gitlab-odx.oracle.com/odx/functions/api/datastore/bolt"
+	logTesting "gitlab-odx.oracle.com/odx/functions/api/logs/testing"
+)
+
+const tmpLogDb = "/tmp/func_test_log.db"
+const tmpDatastore = "/tmp/func_test_datastore.db"
+
+
+func TestDatastore(t *testing.T) {
+	os.Remove(tmpLogDb)
+	os.Remove(tmpDatastore)
+	uLog, err := url.Parse("bolt://" + tmpLogDb)
+	if err != nil {
+		t.Fatalf("failed to parse url: %v", err)
+	}
+	uDatastore, err := url.Parse("bolt://" + tmpDatastore)
+
+	fnl, err := NewBolt(uLog)
+	if err != nil {
+		t.Fatalf("failed to create bolt log datastore: %v", err)
+	}
+	ds, err := bolt.New(uDatastore)
+	if err != nil {
+		t.Fatalf("failed to create bolt datastore: %v", err)
+	}
+	logTesting.Test(t, fnl, ds)
+}

api/logs/log.go

@@ -0,0 +1,22 @@
+package logs
+
+import (
+	"fmt"
+	"net/url"
+	"github.com/Sirupsen/logrus"
+	"gitlab-odx.oracle.com/odx/functions/api/models"
+)
+
+func New(dbURL string) (models.FnLog, error) {
+	u, err := url.Parse(dbURL)
+	if err != nil {
+		logrus.WithError(err).WithFields(logrus.Fields{"url": dbURL}).Fatal("bad DB URL")
+	}
+	logrus.WithFields(logrus.Fields{"db": u.Scheme}).Debug("creating new datastore")
+	switch u.Scheme {
+	case "bolt":
+		return NewBolt(u)
+	default:
+		return nil, fmt.Errorf("db type not supported %v", u.Scheme)
+	}
+}

api/logs/mock.go

@@ -0,0 +1,47 @@
+package logs
+
+import (
+	"context"
+	"gitlab-odx.oracle.com/odx/functions/api/models"
+	"github.com/pkg/errors"
+)
+
+type mock struct {
+	Logs map[string]*models.FnCallLog
+	ds models.Datastore
+}
+
+func NewMock() models.FnLog {
+	return NewMockInit(nil)
+}
+
+func NewMockInit(logs map[string]*models.FnCallLog) models.FnLog {
+	if logs == nil {
+		logs = map[string]*models.FnCallLog{}
+	}
+	fnl := NewValidator(&mock{logs, nil})
+	return fnl
+}
+
+func (m *mock) SetDatastore(ctx context.Context, ds models.Datastore) {
+	m.ds = ds
+}
+
+func (m *mock) InsertLog(ctx context.Context, callID string, callLog string) error {
+	m.Logs[callID] = &models.FnCallLog{CallID: callID, Log:callLog}
+	return nil
+}
+
+func (m *mock) GetLog(ctx context.Context, callID string) (*models.FnCallLog, error) {
+	logEntry := m.Logs[callID]
+	if logEntry == nil {
+		return nil, errors.New("Call log not found")
+	}
+
+	return m.Logs[callID], nil
+}
+
+func (m *mock) DeleteLog(ctx context.Context, callID string) error {
+	delete(m.Logs, callID)
+	return nil
+}

api/logs/testing/test.go

@@ -0,0 +1,91 @@
+package testing
+
+import (
+	"testing"
+	"time"
+	"context"
+	"strings"
+
+	"gitlab-odx.oracle.com/odx/functions/api/models"
+	"github.com/go-openapi/strfmt"
+	"gitlab-odx.oracle.com/odx/functions/api/id"
+)
+
+
+var testApp = &models.App{
+	Name: "Test",
+}
+
+var testRoute = &models.Route{
+	AppName: testApp.Name,
+	Path:    "/test",
+	Image:   "funcy/hello",
+	Type:    "sync",
+	Format:  "http",
+}
+
+func SetUpTestTask() *models.Task {
+	task := &models.Task{}
+	task.CreatedAt = strfmt.DateTime(time.Now())
+	task.Status = "success"
+	task.StartedAt = strfmt.DateTime(time.Now())
+	task.CompletedAt = strfmt.DateTime(time.Now())
+	task.AppName = testApp.Name
+	task.Path = testRoute.Path
+	return task
+}
+
+func Test(t *testing.T, fnl models.FnLog, ds models.Datastore) {
+	ctx := context.Background()
+	task := SetUpTestTask()
+
+	t.Run("call-log-insert", func(t *testing.T) {
+		task.ID = id.New().String()
+		err := ds.InsertTask(ctx, task)
+		if err != nil {
+			t.Fatalf("Test InsertTask(ctx, &task): unexpected error `%v`", err)
+		}
+		err = fnl.InsertLog(ctx, task.ID, "test")
+		if err != nil {
+			t.Fatalf("Test InsertLog(ctx, task.ID, logText): unexpected error during inserting log `%v`", err)
+		}
+	})
+	t.Run("call-log-insert-get", func(t *testing.T) {
+		task.ID = id.New().String()
+		err := ds.InsertTask(ctx, task)
+		logText := "test"
+		if err != nil {
+			t.Fatalf("Test InsertTask(ctx, &task): unexpected error `%v`", err)
+		}
+		err = fnl.InsertLog(ctx, task.ID, logText)
+		if err != nil {
+			t.Fatalf("Test InsertLog(ctx, task.ID, logText): unexpected error during inserting log `%v`", err)
+		}
+		logEntry, err := fnl.GetLog(ctx, task.ID)
+		if !strings.Contains(logEntry.Log, logText) {
+			t.Fatalf("Test GetLog(ctx, task.ID, logText): unexpected error, log mismatch. " +
+				"Expected: `%v`. Got `%v`.", logText, logEntry.Log)
+		}
+	})
+	t.Run("call-log-insert-get-delete", func(t *testing.T) {
+		task.ID = id.New().String()
+		err := ds.InsertTask(ctx, task)
+		logText := "test"
+		if err != nil {
+			t.Fatalf("Test InsertTask(ctx, &task): unexpected error `%v`", err)
+		}
+		err = fnl.InsertLog(ctx, task.ID, logText)
+		if err != nil {
+			t.Fatalf("Test InsertLog(ctx, task.ID, logText): unexpected error during inserting log `%v`", err)
+		}
+		logEntry, err := fnl.GetLog(ctx, task.ID)
+		if !strings.Contains(logEntry.Log, logText) {
+			t.Fatalf("Test GetLog(ctx, task.ID, logText): unexpected error, log mismatch. " +
+				"Expected: `%v`. Got `%v`.", logText, logEntry.Log)
+		}
+		err = fnl.DeleteLog(ctx, task.ID)
+		if err != nil {
+			t.Fatalf("Test DeleteLog(ctx, task.ID): unexpected error during deleting log `%v`", err)
+		}
+	})
+}

api/logs/validator.go

@@ -0,0 +1,35 @@
+package logs
+
+import (
+	"context"
+
+	"gitlab-odx.oracle.com/odx/functions/api/models"
+)
+
+type FnLog interface {
+
+	InsertLog(ctx context.Context, callID string, callLog string) error
+	GetLog(ctx context.Context, callID string) (*models.FnCallLog, error)
+	DeleteLog(ctx context.Context, callID string) error
+}
+
+type validator struct {
+	fnl FnLog
+}
+
+func NewValidator(fnl FnLog) models.FnLog {
+	return &validator{fnl}
+}
+
+
+func (v *validator) InsertLog(ctx context.Context, callID string, callLog string) error {
+	return v.fnl.InsertLog(ctx, callID, callLog)
+}
+
+func (v *validator) GetLog(ctx context.Context, callID string) (*models.FnCallLog, error) {
+	return v.fnl.GetLog(ctx, callID)
+}
+
+func (v *validator) DeleteLog(ctx context.Context, callID string) error {
+	return v.fnl.DeleteLog(ctx, callID)
+}

api/models/app.go

@@ -22,7 +22,6 @@ var (
 	ErrAppsUpdate             = errors.New("Could not update app")
 	ErrDeleteAppsWithRoutes   = errors.New("Cannot remove apps with routes")
 	ErrUsableImage            = errors.New("Image not found")
-	ErrCallNotFound           = errors.New("Call not found")
 	ErrTaskInvalidAppAndRoute = errors.New("Unable to get call for given app and route")
 )
 

api/models/logs.go

@@ -0,0 +1,12 @@
+package models
+
+import (
+	"context"
+)
+
+type FnLog interface {
+
+	InsertLog(ctx context.Context, callID string, callLog string) error
+	GetLog(ctx context.Context, callID string) (*FnCallLog, error)
+	DeleteLog(ctx context.Context, callID string) error
+}

api/models/task.go

@@ -4,6 +4,7 @@ package models
 // Editing this file might prove futile when you re-run the swagger generate command
 
 import (
+	apierrors "errors"
 	"encoding/json"
 
 	strfmt "github.com/go-openapi/strfmt"
@@ -28,6 +29,12 @@ const (
 	FormatHTTP = "http"
 )
 
+var (
+	ErrCallNotFound     = apierrors.New("Call not found")
+	ErrCallLogNotFound  = apierrors.New("Call log not found")
+	ErrCallLogRemoving  = apierrors.New("Could not remove call log")
+)
+
 type FnCall struct {
 	IDStatus
 	CompletedAt strfmt.DateTime `json:"completed_at,omitempty"`
@@ -37,6 +44,12 @@ type FnCall struct {
 	Path        string          `json:"path"`
 }
 
+type FnCallLog struct {
+	CallID string `json:"call_id"`
+	Log string `json:"log"`
+}
+
+
 func (fnCall *FnCall) FromTask(task *Task) *FnCall {
 	return &FnCall{
 		CreatedAt:   task.CreatedAt,

api/runner/async_runner.go

@@ -135,7 +135,7 @@ func startAsyncRunners(ctx context.Context, url string, rnr *Runner, ds models.D
 			go func() {
 				defer wg.Done()
 				// Process Task
-				_, err := rnr.RunTrackedTask(task, ctx, getCfg(task), ds)
+				_, err := rnr.RunTrackedTask(task, ctx, getCfg(task))
 				if err != nil {
 					log.WithError(err).Error("Cannot run task")
 				}

api/runner/async_runner_test.go

@@ -20,6 +20,7 @@ import (
 	"gitlab-odx.oracle.com/odx/functions/api/mqs"
 	"gitlab-odx.oracle.com/odx/functions/api/runner/drivers"
 	"gitlab-odx.oracle.com/odx/functions/api/runner/task"
+	"gitlab-odx.oracle.com/odx/functions/api/logs"
 )
 
 func setLogBuffer() *bytes.Buffer {
@@ -193,7 +194,9 @@ func TestTasksrvURL(t *testing.T) {
 
 func testRunner(t *testing.T) (*Runner, context.CancelFunc) {
 	ctx, cancel := context.WithCancel(context.Background())
-	r, err := New(ctx, NewFuncLogger(), NewMetricLogger())
+	ds := datastore.NewMock()
+	fnl := logs.NewMock()
+	r, err := New(ctx, NewFuncLogger(fnl), NewMetricLogger(), ds)
 	if err != nil {
 		t.Fatal("Test: failed to create new runner")
 	}

api/runner/func_logger.go

@@ -2,40 +2,52 @@ package runner
 
 import (
 	"bufio"
+	"fmt"
 	"io"
 
 	"context"
 	"github.com/Sirupsen/logrus"
+	"gitlab-odx.oracle.com/odx/functions/api/models"
 	"gitlab-odx.oracle.com/odx/functions/api/runner/common"
 )
 
 type FuncLogger interface {
-	Writer(context.Context, string, string, string, string) io.Writer
+	Writer(ctx context.Context, appName, path, image, reqID string) io.Writer
 }
 
-// FuncLogger reads STDERR output from a container and outputs it in a parseable structured log format, see: https://github.com/treeder/functions/issues/76
+// FuncLogger reads STDERR output from a container and outputs it in a parsed structured log format, see: https://github.com/treeder/functions/issues/76
 type DefaultFuncLogger struct {
+	logDB models.FnLog
 }
 
-func NewFuncLogger() FuncLogger {
+func NewFuncLogger(logDB models.FnLog) FuncLogger {
-	return &DefaultFuncLogger{}
+	return &DefaultFuncLogger{logDB}
+}
+
+func (l *DefaultFuncLogger) persistLog(ctx context.Context, log logrus.FieldLogger, reqID, logText string) {
+	err := l.logDB.InsertLog(ctx, reqID, logText)
+	if err != nil {
+		log.WithError(err).Println(fmt.Sprintf(
+			"Unable to persist log for call %v. Error: %v", reqID, err))
+	}
 }
 
 func (l *DefaultFuncLogger) Writer(ctx context.Context, appName, path, image, reqID string) io.Writer {
 	r, w := io.Pipe()
 
-	log := common.Logger(ctx)
-	log = log.WithFields(logrus.Fields{"user_log": true, "app_name": appName, "path": path, "image": image, "call_id": reqID})
-
 	go func(reader io.Reader) {
-		scanner := bufio.NewScanner(reader)
+		log := common.Logger(ctx)
-		for scanner.Scan() {
+		log = log.WithFields(logrus.Fields{"user_log": true, "app_name": appName,
-			log.Println(scanner.Text())
+			"path": path, "image": image, "call_id": reqID})
-		}
-		if err := scanner.Err(); err != nil {
-			log.WithError(err).Println("There was an error with the scanner in attached container")
-		}
-	}(r)
 
+		var res string
+		errMsg := "-------Unable to get full log, it's too big-------"
+		fmt.Fscanf(reader, "%v", &res)
+		if len(res) >= bufio.MaxScanTokenSize {
+			res = res[0:bufio.MaxScanTokenSize - len(errMsg)] + errMsg
+		}
+
+		l.persistLog(ctx, log, reqID, res)
+	}(r)
 	return w
 }

api/runner/runner.go

@@ -14,6 +14,7 @@ import (
 	"time"
 
 	"github.com/Sirupsen/logrus"
+	"gitlab-odx.oracle.com/odx/functions/api/models"
 	"gitlab-odx.oracle.com/odx/functions/api/runner/common"
 	"gitlab-odx.oracle.com/odx/functions/api/runner/drivers"
 	"gitlab-odx.oracle.com/odx/functions/api/runner/drivers/docker"
@@ -33,6 +34,7 @@ type Runner struct {
 	usedMem      int64
 	usedMemMutex sync.RWMutex
 	hcmgr        htfnmgr
+	datastore    models.Datastore
 
 	stats
 }
@@ -48,7 +50,7 @@ const (
 	DefaultIdleTimeout = 30 * time.Second
 )
 
-func New(ctx context.Context, flog FuncLogger, mlog MetricLogger) (*Runner, error) {
+func New(ctx context.Context, flog FuncLogger, mlog MetricLogger, ds models.Datastore) (*Runner, error) {
 	// TODO: Is this really required for the container drivers? Can we remove it?
 	env := common.NewEnvironment(func(e *common.Environment) {})
 
@@ -65,6 +67,7 @@ func New(ctx context.Context, flog FuncLogger, mlog MetricLogger) (*Runner, erro
 		mlog:         mlog,
 		availableMem: getAvailableMemory(),
 		usedMem:      0,
+		datastore:    ds,
 	}
 
 	go r.queueHandler(ctx)

api/runner/runner_test.go

@@ -8,8 +8,10 @@ import (
 	"testing"
 	"time"
 
+	"gitlab-odx.oracle.com/odx/functions/api/datastore"
 	"gitlab-odx.oracle.com/odx/functions/api/models"
 	"gitlab-odx.oracle.com/odx/functions/api/runner/task"
+	"gitlab-odx.oracle.com/odx/functions/api/logs"
 )
 
 func TestRunnerHello(t *testing.T) {
@@ -17,11 +19,14 @@ func TestRunnerHello(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
-	runner, err := New(ctx, NewFuncLogger(), NewMetricLogger())
+	ds := datastore.NewMock()
+	fnl := logs.NewMock()
+	runner, err := New(ctx, NewFuncLogger(fnl), NewMetricLogger(), ds)
 	if err != nil {
 		t.Fatalf("Test error during New() - %s", err)
 	}
 
+
 	for i, test := range []struct {
 		route          *models.Route
 		payload        string
@@ -71,7 +76,9 @@ func TestRunnerError(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
-	runner, err := New(ctx, NewFuncLogger(), NewMetricLogger())
+	ds := datastore.NewMock()
+	fnl := logs.NewMock()
+	runner, err := New(ctx, NewFuncLogger(fnl), NewMetricLogger(), ds)
 	if err != nil {
 		t.Fatalf("Test error during New() - %s", err)
 	}

api/runner/worker.go

@@ -60,7 +60,7 @@ import (
 //                                           (internal clock)
 
 // RunTrackedTask is just a wrapper for shared logic for async/sync runners
-func (rnr *Runner) RunTrackedTask(newTask *models.Task, ctx context.Context, cfg *task.Config, ds models.Datastore) (drivers.RunResult, error) {
+func (rnr *Runner) RunTrackedTask(newTask *models.Task, ctx context.Context, cfg *task.Config) (drivers.RunResult, error) {
 	startedAt := strfmt.DateTime(time.Now())
 	newTask.StartedAt = startedAt
 
@@ -78,7 +78,7 @@ func (rnr *Runner) RunTrackedTask(newTask *models.Task, ctx context.Context, cfg
 	newTask.CompletedAt = completedAt
 	newTask.Status = status
 
-	if err := ds.InsertTask(ctx, newTask); err != nil {
+	if err := rnr.datastore.InsertTask(ctx, newTask); err != nil {
 		// TODO we should just log this error not return it to user? just issue storing task status but task is run
 		logrus.WithError(err).Error("error inserting task into datastore")
 	}

api/server/apps_test.go

@@ -12,6 +12,7 @@ import (
 	"gitlab-odx.oracle.com/odx/functions/api/datastore"
 	"gitlab-odx.oracle.com/odx/functions/api/models"
 	"gitlab-odx.oracle.com/odx/functions/api/mqs"
+	"gitlab-odx.oracle.com/odx/functions/api/logs"
 )
 
 func setLogBuffer() *bytes.Buffer {
@@ -28,25 +29,26 @@ func TestAppCreate(t *testing.T) {
 	buf := setLogBuffer()
 	for i, test := range []struct {
 		mock          models.Datastore
+		logDB         models.FnLog
 		path          string
 		body          string
 		expectedCode  int
 		expectedError error
 	}{
 		// errors
-		{datastore.NewMock(), "/v1/apps", ``, http.StatusBadRequest, models.ErrInvalidJSON},
+		{datastore.NewMock(), logs.NewMock(),"/v1/apps", ``, http.StatusBadRequest, models.ErrInvalidJSON},
-		{datastore.NewMock(), "/v1/apps", `{}`, http.StatusBadRequest, models.ErrAppsMissingNew},
+		{datastore.NewMock(), logs.NewMock(),"/v1/apps", `{}`, http.StatusBadRequest, models.ErrAppsMissingNew},
-		{datastore.NewMock(), "/v1/apps", `{ "name": "Test" }`, http.StatusBadRequest, models.ErrAppsMissingNew},
+		{datastore.NewMock(), logs.NewMock(),"/v1/apps", `{ "name": "Test" }`, http.StatusBadRequest, models.ErrAppsMissingNew},
-		{datastore.NewMock(), "/v1/apps", `{ "app": { "name": "" } }`, http.StatusInternalServerError, models.ErrAppsValidationMissingName},
+		{datastore.NewMock(), logs.NewMock(),"/v1/apps", `{ "app": { "name": "" } }`, http.StatusInternalServerError, models.ErrAppsValidationMissingName},
-		{datastore.NewMock(), "/v1/apps", `{ "app": { "name": "1234567890123456789012345678901" } }`, http.StatusInternalServerError, models.ErrAppsValidationTooLongName},
+		{datastore.NewMock(), logs.NewMock(),"/v1/apps", `{ "app": { "name": "1234567890123456789012345678901" } }`, http.StatusInternalServerError, models.ErrAppsValidationTooLongName},
-		{datastore.NewMock(), "/v1/apps", `{ "app": { "name": "&&%@!#$#@$" } }`, http.StatusInternalServerError, models.ErrAppsValidationInvalidName},
+		{datastore.NewMock(), logs.NewMock(),"/v1/apps", `{ "app": { "name": "&&%@!#$#@$" } }`, http.StatusInternalServerError, models.ErrAppsValidationInvalidName},
-		{datastore.NewMock(), "/v1/apps", `{ "app": { "name": "&&%@!#$#@$" } }`, http.StatusInternalServerError, models.ErrAppsValidationInvalidName},
+		{datastore.NewMock(), logs.NewMock(),"/v1/apps", `{ "app": { "name": "&&%@!#$#@$" } }`, http.StatusInternalServerError, models.ErrAppsValidationInvalidName},
 
 		// success
-		{datastore.NewMock(), "/v1/apps", `{ "app": { "name": "teste" } }`, http.StatusOK, nil},
+		{datastore.NewMock(), logs.NewMock(),"/v1/apps", `{ "app": { "name": "teste" } }`, http.StatusOK, nil},
 	} {
 		rnr, cancel := testRunner(t)
-		srv := testServer(test.mock, &mqs.Mock{}, rnr)
+		srv := testServer(test.mock, &mqs.Mock{}, test.logDB, rnr)
 		router := srv.Router
 
 		body := bytes.NewBuffer([]byte(test.body))
@@ -76,20 +78,21 @@ func TestAppDelete(t *testing.T) {
 
 	for i, test := range []struct {
 		ds            models.Datastore
+		logDB         models.FnLog
 		path          string
 		body          string
 		expectedCode  int
 		expectedError error
 	}{
-		{datastore.NewMock(), "/v1/apps/myapp", "", http.StatusNotFound, nil},
+		{datastore.NewMock(), logs.NewMock(),"/v1/apps/myapp", "", http.StatusNotFound, nil},
 		{datastore.NewMockInit(
 			[]*models.App{{
 				Name: "myapp",
-			}}, nil, nil,
+			}}, nil, nil, nil,
-		), "/v1/apps/myapp", "", http.StatusOK, nil},
+		), logs.NewMock(),"/v1/apps/myapp", "", http.StatusOK, nil},
 	} {
 		rnr, cancel := testRunner(t)
-		srv := testServer(test.ds, &mqs.Mock{}, rnr)
+		srv := testServer(test.ds, &mqs.Mock{}, test.logDB, rnr)
 
 		_, rec := routerRequest(t, srv.Router, "DELETE", test.path, nil)
 
@@ -117,7 +120,9 @@ func TestAppList(t *testing.T) {
 
 	rnr, cancel := testRunner(t)
 	defer cancel()
-	srv := testServer(datastore.NewMock(), &mqs.Mock{}, rnr)
+	ds := datastore.NewMock()
+	fnl := logs.NewMock()
+	srv := testServer(ds, &mqs.Mock{}, fnl, rnr)
 
 	for i, test := range []struct {
 		path          string
@@ -152,7 +157,9 @@ func TestAppGet(t *testing.T) {
 
 	rnr, cancel := testRunner(t)
 	defer cancel()
-	srv := testServer(datastore.NewMock(), &mqs.Mock{}, rnr)
+	ds := datastore.NewMock()
+	fnl := logs.NewMock()
+	srv := testServer(ds, &mqs.Mock{}, fnl, rnr)
 
 	for i, test := range []struct {
 		path          string
@@ -187,30 +194,31 @@ func TestAppUpdate(t *testing.T) {
 
 	for i, test := range []struct {
 		mock          models.Datastore
+		logDB         models.FnLog
 		path          string
 		body          string
 		expectedCode  int
 		expectedError error
 	}{
 		// errors
-		{datastore.NewMock(), "/v1/apps/myapp", ``, http.StatusBadRequest, models.ErrInvalidJSON},
+		{datastore.NewMock(), logs.NewMock(),"/v1/apps/myapp", ``, http.StatusBadRequest, models.ErrInvalidJSON},
 
 		// success
 		{datastore.NewMockInit(
 			[]*models.App{{
 				Name: "myapp",
-			}}, nil, nil,
+			}}, nil, nil, nil,
-		), "/v1/apps/myapp", `{ "app": { "config": { "test": "1" } } }`, http.StatusOK, nil},
+		), logs.NewMock(),"/v1/apps/myapp", `{ "app": { "config": { "test": "1" } } }`, http.StatusOK, nil},
 
 		// Addresses #380
 		{datastore.NewMockInit(
 			[]*models.App{{
 				Name: "myapp",
-			}}, nil, nil,
+			}}, nil, nil, nil,
-		), "/v1/apps/myapp", `{ "app": { "name": "othername" } }`, http.StatusBadRequest, nil},
+		), logs.NewMock(),"/v1/apps/myapp", `{ "app": { "name": "othername" } }`, http.StatusBadRequest, nil},
 	} {
 		rnr, cancel := testRunner(t)
-		srv := testServer(test.mock, &mqs.Mock{}, rnr)
+		srv := testServer(test.mock, &mqs.Mock{}, test.logDB, rnr)
 
 		body := bytes.NewBuffer([]byte(test.body))
 		_, rec := routerRequest(t, srv.Router, "PATCH", test.path, body)

api/server/call_logs.go

@@ -0,0 +1,46 @@
+package server
+
+import (
+	"context"
+	"net/http"
+
+	"github.com/gin-gonic/gin"
+	"gitlab-odx.oracle.com/odx/functions/api"
+)
+
+func (s *Server) handleCallLogGet(c *gin.Context) {
+	ctx := c.MustGet("ctx").(context.Context)
+
+	callID := c.Param(api.Call)
+	_, err := s.Datastore.GetTask(ctx, callID)
+	if err != nil {
+		handleErrorResponse(c, err)
+		return
+	}
+
+	callObj, err := s.LogDB.GetLog(ctx, callID)
+	if err != nil {
+		handleErrorResponse(c, err)
+		return
+	}
+
+	c.JSON(http.StatusOK, fnCallLogResponse{"Successfully loaded call", callObj})
+}
+
+
+func (s *Server) handleCallLogDelete(c *gin.Context) {
+	ctx := c.MustGet("ctx").(context.Context)
+
+	callID := c.Param(api.Call)
+	_, err := s.Datastore.GetTask(ctx, callID)
+	if err != nil {
+		handleErrorResponse(c, err)
+		return
+	}
+	err = s.LogDB.DeleteLog(ctx, callID)
+	if err != nil {
+		handleErrorResponse(c, err)
+		return
+	}
+	c.JSON(http.StatusAccepted, gin.H{"message": "Log delete accepted"})
+}

api/server/error_response.go

@@ -21,6 +21,7 @@ var errStatusCode = map[error]int{
 	models.ErrRoutesNotFound:      http.StatusNotFound,
 	models.ErrRoutesAlreadyExists: http.StatusConflict,
 	models.ErrCallNotFound:        http.StatusNotFound,
+	models.ErrCallLogNotFound:     http.StatusNotFound,
 }
 
 func handleErrorResponse(c *gin.Context, err error) {

api/server/init.go

@@ -24,6 +24,7 @@ func init() {
 	viper.SetDefault(EnvLogLevel, "info")
 	viper.SetDefault(EnvMQURL, fmt.Sprintf("bolt://%s/data/worker_mq.db", cwd))
 	viper.SetDefault(EnvDBURL, fmt.Sprintf("bolt://%s/data/bolt.db?bucket=funcs", cwd))
+	viper.SetDefault(EnvLOGDBURL, fmt.Sprintf("bolt://%s/data/log.db?bucket=funcs", cwd))
 	viper.SetDefault(EnvPort, 8080)
 	viper.SetDefault(EnvAPIURL, fmt.Sprintf("http://127.0.0.1:%d", viper.GetInt(EnvPort)))
 	viper.AutomaticEnv() // picks up env vars automatically

api/server/routes_test.go

@@ -9,6 +9,7 @@ import (
 	"gitlab-odx.oracle.com/odx/functions/api/datastore"
 	"gitlab-odx.oracle.com/odx/functions/api/models"
 	"gitlab-odx.oracle.com/odx/functions/api/mqs"
+	"gitlab-odx.oracle.com/odx/functions/api/logs"
 )
 
 func TestRouteCreate(t *testing.T) {
@@ -16,26 +17,27 @@ func TestRouteCreate(t *testing.T) {
 
 	for i, test := range []struct {
 		mock          models.Datastore
+		logDB         models.FnLog
 		path          string
 		body          string
 		expectedCode  int
 		expectedError error
 	}{
 		// errors
-		{datastore.NewMock(), "/v1/apps/a/routes", ``, http.StatusBadRequest, models.ErrInvalidJSON},
+		{datastore.NewMock(), logs.NewMock(),"/v1/apps/a/routes", ``, http.StatusBadRequest, models.ErrInvalidJSON},
-		{datastore.NewMock(), "/v1/apps/a/routes", `{ }`, http.StatusBadRequest, models.ErrRoutesMissingNew},
+		{datastore.NewMock(), logs.NewMock(),"/v1/apps/a/routes", `{ }`, http.StatusBadRequest, models.ErrRoutesMissingNew},
-		{datastore.NewMock(), "/v1/apps/a/routes", `{ "path": "/myroute" }`, http.StatusBadRequest, models.ErrRoutesMissingNew},
+		{datastore.NewMock(), logs.NewMock(),"/v1/apps/a/routes", `{ "path": "/myroute" }`, http.StatusBadRequest, models.ErrRoutesMissingNew},
-		{datastore.NewMock(), "/v1/apps/a/routes", `{ "route": { } }`, http.StatusBadRequest, models.ErrRoutesValidationMissingPath},
+		{datastore.NewMock(), logs.NewMock(),"/v1/apps/a/routes", `{ "route": { } }`, http.StatusBadRequest, models.ErrRoutesValidationMissingPath},
-		{datastore.NewMock(), "/v1/apps/a/routes", `{ "route": { "path": "/myroute" } }`, http.StatusBadRequest, models.ErrRoutesValidationMissingImage},
+		{datastore.NewMock(), logs.NewMock(),"/v1/apps/a/routes", `{ "route": { "path": "/myroute" } }`, http.StatusBadRequest, models.ErrRoutesValidationMissingImage},
-		{datastore.NewMock(), "/v1/apps/a/routes", `{ "route": { "image": "funcy/hello" } }`, http.StatusBadRequest, models.ErrRoutesValidationMissingPath},
+		{datastore.NewMock(), logs.NewMock(),"/v1/apps/a/routes", `{ "route": { "image": "funcy/hello" } }`, http.StatusBadRequest, models.ErrRoutesValidationMissingPath},
-		{datastore.NewMock(), "/v1/apps/a/routes", `{ "route": { "image": "funcy/hello", "path": "myroute" } }`, http.StatusBadRequest, models.ErrRoutesValidationInvalidPath},
+		{datastore.NewMock(), logs.NewMock(),"/v1/apps/a/routes", `{ "route": { "image": "funcy/hello", "path": "myroute" } }`, http.StatusBadRequest, models.ErrRoutesValidationInvalidPath},
-		{datastore.NewMock(), "/v1/apps/$/routes", `{ "route": { "image": "funcy/hello", "path": "/myroute" } }`, http.StatusInternalServerError, models.ErrAppsValidationInvalidName},
+		{datastore.NewMock(), logs.NewMock(),"/v1/apps/$/routes", `{ "route": { "image": "funcy/hello", "path": "/myroute" } }`, http.StatusInternalServerError, models.ErrAppsValidationInvalidName},
 
 		// success
-		{datastore.NewMock(), "/v1/apps/a/routes", `{ "route": { "image": "funcy/hello", "path": "/myroute" } }`, http.StatusOK, nil},
+		{datastore.NewMock(), logs.NewMock(),"/v1/apps/a/routes", `{ "route": { "image": "funcy/hello", "path": "/myroute" } }`, http.StatusOK, nil},
 	} {
 		rnr, cancel := testRunner(t)
-		srv := testServer(test.mock, &mqs.Mock{}, rnr)
+		srv := testServer(test.mock, &mqs.Mock{}, test.logDB, rnr)
 
 		body := bytes.NewBuffer([]byte(test.body))
 		_, rec := routerRequest(t, srv.Router, "POST", test.path, body)
@@ -67,20 +69,21 @@ func TestRouteDelete(t *testing.T) {
 
 	for i, test := range []struct {
 		ds            models.Datastore
+		logDB         models.FnLog
 		path          string
 		body          string
 		expectedCode  int
 		expectedError error
 	}{
-		{datastore.NewMock(), "/v1/apps/a/routes/missing", "", http.StatusNotFound, nil},
+		{datastore.NewMock(), logs.NewMock(),"/v1/apps/a/routes/missing", "", http.StatusNotFound, nil},
 		{datastore.NewMockInit(nil,
 			[]*models.Route{
 				{Path: "/myroute", AppName: "a"},
-			}, nil,
+			}, nil, nil,
-		), "/v1/apps/a/routes/myroute", "", http.StatusOK, nil},
+		), logs.NewMock(),"/v1/apps/a/routes/myroute", "", http.StatusOK, nil},
 	} {
 		rnr, cancel := testRunner(t)
-		srv := testServer(test.ds, &mqs.Mock{}, rnr)
+		srv := testServer(test.ds, &mqs.Mock{}, test.logDB, rnr)
 		_, rec := routerRequest(t, srv.Router, "DELETE", test.path, nil)
 
 		if rec.Code != test.expectedCode {
@@ -107,7 +110,11 @@ func TestRouteList(t *testing.T) {
 
 	rnr, cancel := testRunner(t)
 	defer cancel()
-	srv := testServer(datastore.NewMock(), &mqs.Mock{}, rnr)
+
+	ds := datastore.NewMock()
+	fnl := logs.NewMock()
+
+	srv := testServer(ds, &mqs.Mock{}, fnl, rnr)
 
 	for i, test := range []struct {
 		path          string
@@ -143,7 +150,10 @@ func TestRouteGet(t *testing.T) {
 	rnr, cancel := testRunner(t)
 	defer cancel()
 
-	srv := testServer(datastore.NewMock(), &mqs.Mock{}, rnr)
+	ds := datastore.NewMock()
+	fnl := logs.NewMock()
+
+	srv := testServer(ds, &mqs.Mock{}, fnl, rnr)
 
 	for i, test := range []struct {
 		path          string
@@ -178,16 +188,17 @@ func TestRouteUpdate(t *testing.T) {
 
 	for i, test := range []struct {
 		ds            models.Datastore
+		logDB         models.FnLog
 		path          string
 		body          string
 		expectedCode  int
 		expectedError error
 	}{
 		// errors
-		{datastore.NewMock(), "/v1/apps/a/routes/myroute/do", ``, http.StatusBadRequest, models.ErrInvalidJSON},
+		{datastore.NewMock(), logs.NewMock(),"/v1/apps/a/routes/myroute/do", ``, http.StatusBadRequest, models.ErrInvalidJSON},
-		{datastore.NewMock(), "/v1/apps/a/routes/myroute/do", `{}`, http.StatusBadRequest, models.ErrRoutesMissingNew},
+		{datastore.NewMock(), logs.NewMock(),"/v1/apps/a/routes/myroute/do", `{}`, http.StatusBadRequest, models.ErrRoutesMissingNew},
-		{datastore.NewMock(), "/v1/apps/a/routes/myroute/do", `{ "route": { "type": "invalid-type" } }`, http.StatusBadRequest, nil},
+		{datastore.NewMock(), logs.NewMock(),"/v1/apps/a/routes/myroute/do", `{ "route": { "type": "invalid-type" } }`, http.StatusBadRequest, nil},
-		{datastore.NewMock(), "/v1/apps/a/routes/myroute/do", `{ "route": { "format": "invalid-format" } }`, http.StatusBadRequest, nil},
+		{datastore.NewMock(), logs.NewMock(),"/v1/apps/a/routes/myroute/do", `{ "route": { "format": "invalid-format" } }`, http.StatusBadRequest, nil},
 
 		// success
 		{datastore.NewMockInit(nil,
@@ -196,8 +207,8 @@ func TestRouteUpdate(t *testing.T) {
 					AppName: "a",
 					Path:    "/myroute/do",
 				},
-			}, nil,
+			}, nil, nil,
-		), "/v1/apps/a/routes/myroute/do", `{ "route": { "image": "funcy/hello" } }`, http.StatusOK, nil},
+		), logs.NewMock(),"/v1/apps/a/routes/myroute/do", `{ "route": { "image": "funcy/hello" } }`, http.StatusOK, nil},
 
 		// Addresses #381
 		{datastore.NewMockInit(nil,
@@ -206,11 +217,12 @@ func TestRouteUpdate(t *testing.T) {
 					AppName: "a",
 					Path:    "/myroute/do",
 				},
-			}, nil,
+			}, nil, nil,
-		), "/v1/apps/a/routes/myroute/do", `{ "route": { "path": "/otherpath" } }`, http.StatusBadRequest, nil},
+		), logs.NewMock(),"/v1/apps/a/routes/myroute/do", `{ "route": { "path": "/otherpath" } }`, http.StatusBadRequest, nil},
 	} {
 		rnr, cancel := testRunner(t)
-		srv := testServer(test.ds, &mqs.Mock{}, rnr)
+
+		srv := testServer(test.ds, &mqs.Mock{}, test.logDB, rnr)
 
 		body := bytes.NewBuffer([]byte(test.body))
 

api/server/runner.go

@@ -245,7 +245,7 @@ func (s *Server) serve(ctx context.Context, c *gin.Context, appName string, foun
 		c.JSON(http.StatusAccepted, map[string]string{"call_id": newTask.ID})
 
 	default:
-		result, err := s.Runner.RunTrackedTask(newTask, ctx, cfg, s.Datastore)
+		result, err := s.Runner.RunTrackedTask(newTask, ctx, cfg)
 		if result != nil {
 			waitTime := result.StartTime().Sub(cfg.ReceivedTime)
 			c.Header("XXX-FXLB-WAIT", waitTime.String())

api/server/runner_async_test.go

@@ -45,7 +45,7 @@ func TestRouteRunnerAsyncExecution(t *testing.T) {
 			{Type: "async", Path: "/myroute", AppName: "myapp", Image: "funcy/hello", Config: map[string]string{"test": "true"}},
 			{Type: "async", Path: "/myerror", AppName: "myapp", Image: "funcy/error", Config: map[string]string{"test": "true"}},
 			{Type: "async", Path: "/myroute/:param", AppName: "myapp", Image: "funcy/hello", Config: map[string]string{"test": "true"}},
-		}, nil,
+		}, nil, nil,
 	)
 	mq := &mqs.Mock{}
 

api/server/runner_test.go

@@ -11,11 +11,14 @@ import (
 	"gitlab-odx.oracle.com/odx/functions/api/models"
 	"gitlab-odx.oracle.com/odx/functions/api/mqs"
 	"gitlab-odx.oracle.com/odx/functions/api/runner"
+	"gitlab-odx.oracle.com/odx/functions/api/logs"
 )
 
 func testRunner(t *testing.T) (*runner.Runner, context.CancelFunc) {
 	ctx, cancel := context.WithCancel(context.Background())
-	r, err := runner.New(ctx, runner.NewFuncLogger(), runner.NewMetricLogger())
+	ds := datastore.NewMock()
+	fnl := logs.NewMock()
+	r, err := runner.New(ctx, runner.NewFuncLogger(fnl), runner.NewMetricLogger(), ds)
 	if err != nil {
 		t.Fatal("Test: failed to create new runner")
 	}
@@ -24,15 +27,15 @@ func testRunner(t *testing.T) (*runner.Runner, context.CancelFunc) {
 
 func TestRouteRunnerGet(t *testing.T) {
 	buf := setLogBuffer()
-
 	rnr, cancel := testRunner(t)
 	defer cancel()
-
+	ds := datastore.NewMockInit(
-	srv := testServer(datastore.NewMockInit(
 		[]*models.App{
 			{Name: "myapp", Config: models.Config{}},
-		}, nil, nil,
+		}, nil, nil, nil,
-	), &mqs.Mock{}, rnr)
+	)
+	logDB := logs.NewMock()
+	srv := testServer(ds, &mqs.Mock{}, logDB, rnr)
 
 	for i, test := range []struct {
 		path          string
@@ -70,11 +73,13 @@ func TestRouteRunnerPost(t *testing.T) {
 	rnr, cancel := testRunner(t)
 	defer cancel()
 
-	srv := testServer(datastore.NewMockInit(
+	ds := datastore.NewMockInit(
 		[]*models.App{
 			{Name: "myapp", Config: models.Config{}},
-		}, nil, nil,
+		}, nil, nil, nil,
-	), &mqs.Mock{}, rnr)
+	)
+	fnl := logs.NewMock()
+	srv := testServer(ds, &mqs.Mock{}, fnl, rnr)
 
 	for i, test := range []struct {
 		path          string
@@ -114,15 +119,20 @@ func TestRouteRunnerExecution(t *testing.T) {
 	rnr, cancelrnr := testRunner(t)
 	defer cancelrnr()
 
-	srv := testServer(datastore.NewMockInit(
+	ds := datastore.NewMockInit(
 		[]*models.App{
 			{Name: "myapp", Config: models.Config{}},
 		},
 		[]*models.Route{
 			{Path: "/myroute", AppName: "myapp", Image: "funcy/hello", Headers: map[string][]string{"X-Function": {"Test"}}},
 			{Path: "/myerror", AppName: "myapp", Image: "funcy/error", Headers: map[string][]string{"X-Function": {"Test"}}},
-		}, nil,
+		}, nil, nil,
-	), &mqs.Mock{}, rnr)
+	)
+
+
+	fnl := logs.NewMock()
+	srv := testServer(ds, &mqs.Mock{}, fnl, rnr)
+
 
 	for i, test := range []struct {
 		path            string
@@ -167,14 +177,16 @@ func TestRouteRunnerTimeout(t *testing.T) {
 	rnr, cancelrnr := testRunner(t)
 	defer cancelrnr()
 
-	srv := testServer(datastore.NewMockInit(
+	ds := datastore.NewMockInit(
 		[]*models.App{
 			{Name: "myapp", Config: models.Config{}},
 		},
 		[]*models.Route{
 			{Path: "/sleeper", AppName: "myapp", Image: "funcy/sleeper", Timeout: 1},
-		}, nil,
+		}, nil, nil,
-	), &mqs.Mock{}, rnr)
+	)
+	fnl := logs.NewMock()
+	srv := testServer(ds, &mqs.Mock{}, fnl, rnr)
 	
 	for i, test := range []struct {
 		path            string

api/server/server.go

@@ -24,12 +24,14 @@ import (
 	"gitlab-odx.oracle.com/odx/functions/api/runner"
 	"gitlab-odx.oracle.com/odx/functions/api/runner/common"
 	"gitlab-odx.oracle.com/odx/functions/api/server/internal/routecache"
+	"gitlab-odx.oracle.com/odx/functions/api/logs"
 )
 
 const (
 	EnvLogLevel = "log_level"
 	EnvMQURL    = "mq_url"
 	EnvDBURL    = "db_url"
+	EnvLOGDBURL    = "logstore_url"
 	EnvPort     = "port" // be careful, Gin expects this variable to be "port"
 	EnvAPIURL   = "api_url"
 )
@@ -40,6 +42,7 @@ type Server struct {
 	Router       *gin.Engine
 	MQ           models.MessageQueue
 	Enqueue      models.Enqueue
+	LogDB        models.FnLog
 
 	apiURL string
 
@@ -67,17 +70,22 @@ func NewFromEnv(ctx context.Context) *Server {
 		logrus.WithError(err).Fatal("Error initializing message queue.")
 	}
 
+	logDB, err := logs.New(viper.GetString(EnvLOGDBURL))
+	if err != nil {
+		logrus.WithError(err).Fatal("Error initializing logs store.")
+	}
+
 	apiURL := viper.GetString(EnvAPIURL)
 
-	return New(ctx, ds, mq, apiURL)
+	return New(ctx, ds, mq, logDB, apiURL)
 }
 
 // New creates a new Functions server with the passed in datastore, message queue and API URL
-func New(ctx context.Context, ds models.Datastore, mq models.MessageQueue, apiURL string, opts ...ServerOption) *Server {
+func New(ctx context.Context, ds models.Datastore, mq models.MessageQueue, logDB models.FnLog, apiURL string, opts ...ServerOption) *Server {
 	metricLogger := runner.NewMetricLogger()
-	funcLogger := runner.NewFuncLogger()
+	funcLogger := runner.NewFuncLogger(logDB)
 
-	rnr, err := runner.New(ctx, funcLogger, metricLogger)
+	rnr, err := runner.New(ctx, funcLogger, metricLogger, ds)
 	if err != nil {
 		logrus.WithError(err).Fatalln("Failed to create a runner")
 		return nil
@@ -89,6 +97,7 @@ func New(ctx context.Context, ds models.Datastore, mq models.MessageQueue, apiUR
 		Datastore: ds,
 		MQ:        mq,
 		hotroutes: routecache.New(cacheSize),
+		LogDB:     logDB,
 		Enqueue:   DefaultEnqueue,
 		apiURL:    apiURL,
 	}
@@ -302,6 +311,8 @@ func (s *Server) bindHandlers(ctx context.Context) {
 		v1.GET("/routes", s.handleRouteList)
 
 		v1.GET("/calls/:call", s.handleCallGet)
+		v1.GET("/calls/:call/log", s.handleCallLogGet)
+		v1.DELETE("/calls/:call/log", s.handleCallLogDelete)
 
 		apps := v1.Group("/apps/:app")
 		{
@@ -356,3 +367,8 @@ type fnCallsResponse struct {
 	Message string         `json:"message"`
 	Calls   models.FnCalls `json:"calls"`
 }
+
+type fnCallLogResponse struct {
+	Message string         `json:"message"`
+	Log *models.FnCallLog   `json:"log"`
+}

api/server/server_test.go

@@ -17,17 +17,21 @@ import (
 	"gitlab-odx.oracle.com/odx/functions/api/mqs"
 	"gitlab-odx.oracle.com/odx/functions/api/runner"
 	"gitlab-odx.oracle.com/odx/functions/api/server/internal/routecache"
+	"gitlab-odx.oracle.com/odx/functions/api/logs"
 )
 
-var tmpBolt = "/tmp/func_test_bolt.db"
+var tmpDatastoreBolt = "/tmp/func_test_bolt_datastore.db"
+var tmpLogBolt = "/tmp/func_test_bolt_log.db"
 
-func testServer(ds models.Datastore, mq models.MessageQueue, rnr *runner.Runner) *Server {
+
+func testServer(ds models.Datastore, mq models.MessageQueue, logDB models.FnLog, rnr *runner.Runner) *Server {
 	ctx := context.Background()
 
 	s := &Server{
 		Runner:    rnr,
 		Router:    gin.New(),
 		Datastore: ds,
+		LogDB:     nil,
 		MQ:        mq,
 		Enqueue:   DefaultEnqueue,
 		hotroutes: routecache.New(2),
@@ -79,26 +83,33 @@ func getErrorResponse(t *testing.T, rec *httptest.ResponseRecorder) models.Error
 	return errResp
 }
 
-func prepareBolt(t *testing.T) (models.Datastore, func()) {
+func prepareBolt(ctx context.Context, t *testing.T) (models.Datastore, models.FnLog, func()) {
-	os.Remove(tmpBolt)
+	os.Remove(tmpDatastoreBolt)
-	ds, err := datastore.New("bolt://" + tmpBolt)
+	os.Remove(tmpLogBolt)
+	ds, err := datastore.New("bolt://" + tmpDatastoreBolt)
 	if err != nil {
-		t.Fatal("Error when creating datastore: %s", err)
+		t.Fatalf("Error when creating datastore: %s", err)
 	}
-	return ds, func() {
+	logDB, err := logs.New("bolt://" + tmpLogBolt)
-		os.Remove(tmpBolt)
+	if err != nil {
+		t.Fatalf("Error when creating log store: %s", err)
+	}
+	return ds,logDB, func() {
+		os.Remove(tmpDatastoreBolt)
+		os.Remove(tmpLogBolt)
 	}
 }
 
 func TestFullStack(t *testing.T) {
+	ctx := context.Background()
 	buf := setLogBuffer()
-	ds, closeBolt := prepareBolt(t)
+	ds, logDB, closeBolt := prepareBolt(ctx, t)
 	defer closeBolt()
 
 	rnr, rnrcancel := testRunner(t)
 	defer rnrcancel()
 
-	srv := testServer(ds, &mqs.Mock{}, rnr)
+	srv := testServer(ds, &mqs.Mock{}, logDB, rnr)
 	srv.hotroutes = routecache.New(2)
 
 	for _, test := range []struct {

docs/operating/logs/README.md

@@ -0,0 +1,19 @@
+
+# Function logs
+
+We currently support the following function logs stores and they are passed in via the `LOGSTORE_URL` environment variable. For example:
+Maximum size of single log entry: 4Mb
+
+
+```sh
+docker run -e "LOGSTORE_URL=bolt:///functions/logs/bolt.db" ...
+```
+
+## [Bolt](https://github.com/boltdb/bolt) (default)
+
+URL: `bolt:///functions/logs/bolt.db`
+
+Bolt is an embedded database which stores to disk. If you want to use this, be sure you don't lose the data directory by mounting
+the directory on your host. eg: `docker run -v $PWD/data:/functions/data -e LOGSTORE_URL=bolt:///functions/data/bolt.db ...`
+
+[More on BoltDB](../databases/boltdb.md)

docs/swagger.yml

@@ -317,6 +317,52 @@ paths:
           schema:
             $ref: '#/definitions/Error'
 
+  /calls/{call}/log:
+    get:
+      summary: Get call logs
+      description: Get call logs
+      tags:
+        - Call
+        - Log
+      parameters:
+        - name: call
+          description: Call ID.
+          required: true
+          type: string
+          in: path
+      responses:
+        200:
+          description: Log found
+          schema:
+            $ref:  '#/definitions/LogWrapper'
+        404:
+          description: Log not found.
+          schema:
+            $ref: '#/definitions/Error'
+    delete:
+      summary: Delete call log entry
+      description: Delete call log entry
+      tags:
+         - Call
+         - Log
+      parameters:
+        - name: call
+          description: Call ID.
+          required: true
+          type: string
+          in: path
+      responses:
+        202:
+          description: Log delete request accepted
+        404:
+          description: Does not exist.
+          schema:
+            $ref: '#/definitions/Error'
+        default:
+          description: Unexpected error
+          schema:
+            $ref: '#/definitions/Error'
+
   /calls/{call}:
     get:
       summary: Get call information
@@ -519,6 +565,25 @@ definitions:
         $ref: '#/definitions/Call'
         description: "Call object."
 
+
+  LogWrapper:
+    type: object
+    required:
+      - log
+    properties:
+      log:
+        $ref: '#/definitions/Log'
+        description: "Call log entry."
+
+  Log:
+    type: object
+    properties:
+      call_id:
+        type: string
+        description: Call UUID ID
+      log:
+        type: string # maybe bytes, long logs wouldn't fit into string type
+
   Call:
     type: object
     properties:

examples/tutorial/logging/Dockerfile

@@ -0,0 +1,8 @@
+FROM funcy/go:dev as build-stage
+WORKDIR /function
+ADD . /src
+RUN cd /src && go build -o func
+FROM funcy/go
+WORKDIR /function
+COPY --from=build-stage /src/func /function/
+ENTRYPOINT ["./func"]

examples/tutorial/logging/func.yaml

@@ -0,0 +1,5 @@
+name: funcy/stderr-logging
+version: 0.0.1
+runtime: go
+entrypoint: ./func
+path: /stderr-logging

examples/tutorial/logging/main.go

@@ -0,0 +1,29 @@
+package main
+
+import (
+	"fmt"
+	"encoding/json"
+	"os"
+	"math/rand"
+)
+
+const lBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
+
+type OutputSize struct {
+	Size int `json:"size"`
+}
+
+
+func RandStringBytes(n int) string {
+	b := make([]byte, n)
+	for i := range b {
+		b[i] = lBytes[rand.Intn(len(lBytes))]
+	}
+	return string(b)
+}
+
+func main() {
+	out := &OutputSize{Size: 64 * 1024}
+	json.NewDecoder(os.Stdin).Decode(out)
+	fmt.Fprintln(os.Stderr, RandStringBytes(out.Size))
+}

examples/tutorial/logging/sample.payload.json

@@ -0,0 +1,3 @@
+{
+    "size": 8
+}