alihan/fn-serverless
1
0
Fork 0
mirror of https://github.com/fnproject/fn.git synced 2022-10-28 21:29:17 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'func_logs2' into 'master'

Browse Source 
Func logs feature

See merge request !66
This commit is contained in:
Travis Reeder
2017-06-20 11:51:26 -07:00
parent 89ccbcbfd9 8a3edb8309
commit 8800ecc5c2
39 changed files with 783 additions and 140 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
Makefile
View File

@@ -1,5 +1,5 @@
# Just builds
.PHONY: all test dep build
.PHONY: all test dep build test-log-datastore
dep:
dep ensure
@@ -13,11 +13,14 @@ test:
test-datastore:
cd api/datastore && go test -v ./...
test-log-datastore:
cd api/logs && go test -v ./...
test-build-arm:
GOARCH=arm GOARM=5 $(MAKE) build
GOARCH=arm GOARM=6 $(MAKE) build
GOARCH=arm GOARM=7 $(MAKE) build
GOARCH=arm64 $(MAKE) build
GOARCH=arm GOARM=5 $(MAKE) build
GOARCH=arm GOARM=6 $(MAKE) build
GOARCH=arm GOARM=7 $(MAKE) build
GOARCH=arm64 $(MAKE) build
run: build
GIN_MODE=debug ./functions

23
api/datastore/bolt/bolt.go
View File

@@ -19,11 +19,11 @@ import (
)
type BoltDatastore struct {
routesBucket []byte
appsBucket []byte
logsBucket []byte
extrasBucket []byte
callsBucket []byte
routesBucket []byte
appsBucket []byte
logsBucket []byte
extrasBucket []byte
callsBucket []byte
db *bolt.DB
log logrus.FieldLogger
}
@@ -53,7 +53,8 @@ func New(url *url.URL) (models.Datastore, error) {
extrasBucketName := []byte(bucketPrefix + "extras") // todo: think of a better name
callsBucketName := []byte(bucketPrefix + "calls")
err = db.Update(func(tx *bolt.Tx) error {
for _, name := range [][]byte{routesBucketName, appsBucketName, logsBucketName, extrasBucketName, callsBucketName} {
for _, name := range [][]byte{routesBucketName, appsBucketName, logsBucketName,
extrasBucketName, callsBucketName} {
_, err := tx.CreateBucketIfNotExists(name)
if err != nil {
log.WithError(err).WithFields(logrus.Fields{"name": name}).Error("create bucket")
@@ -68,11 +69,11 @@ func New(url *url.URL) (models.Datastore, error) {
}
ds := &BoltDatastore{
routesBucket: routesBucketName,
appsBucket: appsBucketName,
logsBucket: logsBucketName,
extrasBucket: extrasBucketName,
callsBucket: callsBucketName,
routesBucket: routesBucketName,
appsBucket: appsBucketName,
logsBucket: logsBucketName,
extrasBucket: extrasBucketName,
callsBucket: callsBucketName,
db: db,
log: log,
}

10
api/datastore/internal/datastoretest/test.go
View File

@@ -10,8 +10,6 @@ import (
"gitlab-odx.oracle.com/odx/functions/api/models"
"net/http"
"net/url"
"os"
"reflect"
"time"
@@ -30,14 +28,6 @@ func setLogBuffer() *bytes.Buffer {
return &buf
}
func GetContainerHostIP() string {
dockerHost := os.Getenv("DOCKER_HOST")
if dockerHost == "" {
return "127.0.0.1"
}
parts, _ := url.Parse(dockerHost)
return parts.Hostname()
}
func Test(t *testing.T, ds models.Datastore) {
buf := setLogBuffer()

7
api/datastore/internal/datastoreutil/shared.go
View File

@@ -15,6 +15,13 @@ type RowScanner interface {
Scan(dest ...interface{}) error
}
func ScanLog(scanner RowScanner, log *models.FnCallLog) error {
return scanner.Scan(
&log.CallID,
&log.Log,
)
}
func ScanRoute(scanner RowScanner, route *models.Route) error {
var headerStr string
var configStr string

7
api/datastore/mock.go
View File

@@ -15,10 +15,10 @@ type mock struct {
}
func NewMock() models.Datastore {
return NewMockInit(nil, nil, nil)
return NewMockInit(nil, nil, nil, nil)
}
func NewMockInit(apps models.Apps, routes models.Routes, calls models.FnCalls) models.Datastore {
func NewMockInit(apps models.Apps, routes models.Routes, calls models.FnCalls, logs []*models.FnCallLog) models.Datastore {
if apps == nil {
apps = models.Apps{}
}
@@ -28,6 +28,9 @@ func NewMockInit(apps models.Apps, routes models.Routes, calls models.FnCalls) m
if calls == nil {
calls = models.FnCalls{}
}
if logs == nil {
logs = []*models.FnCallLog{}
}
return datastoreutil.NewValidator(&mock{apps, routes, calls, make(map[string][]byte)})
}

3
api/datastore/mysql/mysql.go
View File

@@ -63,7 +63,8 @@ type MySQLDatastore struct {
New creates a new MySQL Datastore.
*/
func New(url *url.URL) (models.Datastore, error) {
tables := []string{routesTableCreate, appsTableCreate, extrasTableCreate, callTableCreate}
tables := []string{routesTableCreate, appsTableCreate,
extrasTableCreate, callTableCreate}
dialect := "mysql"
sqlDatastore := &MySQLDatastore{}
dataSourceName := fmt.Sprintf("%s@%s%s", url.User.String(), url.Host, url.Path)

1
api/datastore/postgres/postgres.go
View File

@@ -53,6 +53,7 @@ const callsTableCreate = `CREATE TABLE IF NOT EXISTS calls (
const callSelector = `SELECT id, created_at, started_at, completed_at, status, app_name, path FROM calls`
type PostgresDatastore struct {
db *sql.DB
}

5
api/datastore/redis/redis.go
View File

@@ -317,11 +317,6 @@ func applyCallFilter(call *models.FnCall, filter *models.CallFilter) bool {
}
func (ds *RedisDataStore) InsertTask(ctx context.Context, task *models.Task) error {
_, err := ds.conn.Do("HEXISTS", "calls", task.ID)
if err != nil {
return err
}
taskBytes, err := json.Marshal(task)
if err != nil {
return err

128
api/logs/bolt.go Normal file
View File

@@ -0,0 +1,128 @@
package logs
import (
"encoding/json"
"net/url"
"os"
"path/filepath"
"time"
"context"
"github.com/Sirupsen/logrus"
"github.com/boltdb/bolt"
"gitlab-odx.oracle.com/odx/functions/api/models"
)
type BoltLogDatastore struct {
callLogsBucket []byte
db *bolt.DB
log logrus.FieldLogger
datastore models.Datastore
}
func NewBolt(url *url.URL) (models.FnLog, error) {
dir := filepath.Dir(url.Path)
log := logrus.WithFields(logrus.Fields{"logdb": url.Scheme, "dir": dir})
err := os.MkdirAll(dir, 0755)
if err != nil {
log.WithError(err).Errorln("Could not create data directory for log.db")
return nil, err
}
log.WithFields(logrus.Fields{"path": url.Path}).Debug("Creating bolt log.db")
db, err := bolt.Open(url.Path, 0655, &bolt.Options{Timeout: 1 * time.Second})
if err != nil {
log.WithError(err).Errorln("Error on bolt.Open")
return nil, err
}
// I don't think we need a prefix here do we? Made it blank. If we do, we should call the query param "prefix" instead of bucket.
bucketPrefix := ""
if url.Query()["bucket"] != nil {
bucketPrefix = url.Query()["bucket"][0]
}
callLogsBucketName := []byte(bucketPrefix + "call_logs")
err = db.Update(func(tx *bolt.Tx) error {
for _, name := range [][]byte{callLogsBucketName} {
_, err := tx.CreateBucketIfNotExists(name)
if err != nil {
log.WithError(err).WithFields(logrus.Fields{"name": name}).Error("create bucket")
return err
}
}
return nil
})
if err != nil {
log.WithError(err).Errorln("Error creating bolt buckets")
return nil, err
}
fnl := &BoltLogDatastore{
callLogsBucket: callLogsBucketName,
db: db,
log: log,
}
log.WithFields(logrus.Fields{"prefix": bucketPrefix, "file": url.Path}).Debug("BoltDB initialized")
return NewValidator(fnl), nil
}
func (fnl *BoltLogDatastore) InsertLog(ctx context.Context, callID string, callLog string) error {
log := &models.FnCallLog{
CallID: callID,
Log: callLog,
}
id := []byte(callID)
err := fnl.db.Update(
func(tx *bolt.Tx) error {
bIm := tx.Bucket(fnl.callLogsBucket)
buf, err := json.Marshal(log)
if err != nil {
return err
}
err = bIm.Put(id, buf)
if err != nil {
return err
}
return nil
})
return err
}
func (fnl *BoltLogDatastore) GetLog(ctx context.Context, callID string) (*models.FnCallLog, error) {
var res *models.FnCallLog
err := fnl.db.View(func(tx *bolt.Tx) error {
b := tx.Bucket(fnl.callLogsBucket)
v := b.Get([]byte(callID))
if v != nil {
fnCall := &models.FnCallLog{}
err := json.Unmarshal(v, fnCall)
if err != nil {
return nil
}
res = fnCall
} else {
return models.ErrCallLogNotFound
}
return nil
})
return res, err
}
func (fnl *BoltLogDatastore) DeleteLog(ctx context.Context, callID string) error {
_, err := fnl.GetLog(ctx, callID)
//means object does not exist
if err != nil {
return nil
}
id := []byte(callID)
err = fnl.db.Update(func(tx *bolt.Tx) error {
bIm := tx.Bucket(fnl.callLogsBucket)
err := bIm.Delete(id)
return err
})
return err
}

34
api/logs/bolt_test.go Normal file
View File

@@ -0,0 +1,34 @@
package logs
import (
"net/url"
"os"
"testing"
"gitlab-odx.oracle.com/odx/functions/api/datastore/bolt"
logTesting "gitlab-odx.oracle.com/odx/functions/api/logs/testing"
)
const tmpLogDb = "/tmp/func_test_log.db"
const tmpDatastore = "/tmp/func_test_datastore.db"
func TestDatastore(t *testing.T) {
os.Remove(tmpLogDb)
os.Remove(tmpDatastore)
uLog, err := url.Parse("bolt://" + tmpLogDb)
if err != nil {
t.Fatalf("failed to parse url: %v", err)
}
uDatastore, err := url.Parse("bolt://" + tmpDatastore)
fnl, err := NewBolt(uLog)
if err != nil {
t.Fatalf("failed to create bolt log datastore: %v", err)
}
ds, err := bolt.New(uDatastore)
if err != nil {
t.Fatalf("failed to create bolt datastore: %v", err)
}
logTesting.Test(t, fnl, ds)
}

22
api/logs/log.go Normal file
View File

@@ -0,0 +1,22 @@
package logs
import (
"fmt"
"net/url"
"github.com/Sirupsen/logrus"
"gitlab-odx.oracle.com/odx/functions/api/models"
)
func New(dbURL string) (models.FnLog, error) {
u, err := url.Parse(dbURL)
if err != nil {
logrus.WithError(err).WithFields(logrus.Fields{"url": dbURL}).Fatal("bad DB URL")
}
logrus.WithFields(logrus.Fields{"db": u.Scheme}).Debug("creating new datastore")
switch u.Scheme {
case "bolt":
return NewBolt(u)
default:
return nil, fmt.Errorf("db type not supported %v", u.Scheme)
}
}

47
api/logs/mock.go Normal file
View File

@@ -0,0 +1,47 @@
package logs
import (
"context"
"gitlab-odx.oracle.com/odx/functions/api/models"
"github.com/pkg/errors"
)
type mock struct {
Logs map[string]*models.FnCallLog
ds models.Datastore
}
func NewMock() models.FnLog {
return NewMockInit(nil)
}
func NewMockInit(logs map[string]*models.FnCallLog) models.FnLog {
if logs == nil {
logs = map[string]*models.FnCallLog{}
}
fnl := NewValidator(&mock{logs, nil})
return fnl
}
func (m *mock) SetDatastore(ctx context.Context, ds models.Datastore) {
m.ds = ds
}
func (m *mock) InsertLog(ctx context.Context, callID string, callLog string) error {
m.Logs[callID] = &models.FnCallLog{CallID: callID, Log:callLog}
return nil
}
func (m *mock) GetLog(ctx context.Context, callID string) (*models.FnCallLog, error) {
logEntry := m.Logs[callID]
if logEntry == nil {
return nil, errors.New("Call log not found")
}
return m.Logs[callID], nil
}
func (m *mock) DeleteLog(ctx context.Context, callID string) error {
delete(m.Logs, callID)
return nil
}

91
api/logs/testing/test.go Normal file
View File

@@ -0,0 +1,91 @@
package testing
import (
"testing"
"time"
"context"
"strings"
"gitlab-odx.oracle.com/odx/functions/api/models"
"github.com/go-openapi/strfmt"
"gitlab-odx.oracle.com/odx/functions/api/id"
)
var testApp = &models.App{
Name: "Test",
}
var testRoute = &models.Route{
AppName: testApp.Name,
Path: "/test",
Image: "funcy/hello",
Type: "sync",
Format: "http",
}
func SetUpTestTask() *models.Task {
task := &models.Task{}
task.CreatedAt = strfmt.DateTime(time.Now())
task.Status = "success"
task.StartedAt = strfmt.DateTime(time.Now())
task.CompletedAt = strfmt.DateTime(time.Now())
task.AppName = testApp.Name
task.Path = testRoute.Path
return task
}
func Test(t *testing.T, fnl models.FnLog, ds models.Datastore) {
ctx := context.Background()
task := SetUpTestTask()
t.Run("call-log-insert", func(t *testing.T) {
task.ID = id.New().String()
err := ds.InsertTask(ctx, task)
if err != nil {
t.Fatalf("Test InsertTask(ctx, &task): unexpected error `%v`", err)
}
err = fnl.InsertLog(ctx, task.ID, "test")
if err != nil {
t.Fatalf("Test InsertLog(ctx, task.ID, logText): unexpected error during inserting log `%v`", err)
}
})
t.Run("call-log-insert-get", func(t *testing.T) {
task.ID = id.New().String()
err := ds.InsertTask(ctx, task)
logText := "test"
if err != nil {
t.Fatalf("Test InsertTask(ctx, &task): unexpected error `%v`", err)
}
err = fnl.InsertLog(ctx, task.ID, logText)
if err != nil {
t.Fatalf("Test InsertLog(ctx, task.ID, logText): unexpected error during inserting log `%v`", err)
}
logEntry, err := fnl.GetLog(ctx, task.ID)
if !strings.Contains(logEntry.Log, logText) {
t.Fatalf("Test GetLog(ctx, task.ID, logText): unexpected error, log mismatch. " +
"Expected: `%v`. Got `%v`.", logText, logEntry.Log)
}
})
t.Run("call-log-insert-get-delete", func(t *testing.T) {
task.ID = id.New().String()
err := ds.InsertTask(ctx, task)
logText := "test"
if err != nil {
t.Fatalf("Test InsertTask(ctx, &task): unexpected error `%v`", err)
}
err = fnl.InsertLog(ctx, task.ID, logText)
if err != nil {
t.Fatalf("Test InsertLog(ctx, task.ID, logText): unexpected error during inserting log `%v`", err)
}
logEntry, err := fnl.GetLog(ctx, task.ID)
if !strings.Contains(logEntry.Log, logText) {
t.Fatalf("Test GetLog(ctx, task.ID, logText): unexpected error, log mismatch. " +
"Expected: `%v`. Got `%v`.", logText, logEntry.Log)
}
err = fnl.DeleteLog(ctx, task.ID)
if err != nil {
t.Fatalf("Test DeleteLog(ctx, task.ID): unexpected error during deleting log `%v`", err)
}
})
}

35
api/logs/validator.go Normal file
View File

@@ -0,0 +1,35 @@
package logs
import (
"context"
"gitlab-odx.oracle.com/odx/functions/api/models"
)
type FnLog interface {
InsertLog(ctx context.Context, callID string, callLog string) error
GetLog(ctx context.Context, callID string) (*models.FnCallLog, error)
DeleteLog(ctx context.Context, callID string) error
}
type validator struct {
fnl FnLog
}
func NewValidator(fnl FnLog) models.FnLog {
return &validator{fnl}
}
func (v *validator) InsertLog(ctx context.Context, callID string, callLog string) error {
return v.fnl.InsertLog(ctx, callID, callLog)
}
func (v *validator) GetLog(ctx context.Context, callID string) (*models.FnCallLog, error) {
return v.fnl.GetLog(ctx, callID)
}
func (v *validator) DeleteLog(ctx context.Context, callID string) error {
return v.fnl.DeleteLog(ctx, callID)
}

1
api/models/app.go
View File

@@ -22,7 +22,6 @@ var (
ErrAppsUpdate = errors.New("Could not update app")
ErrDeleteAppsWithRoutes = errors.New("Cannot remove apps with routes")
ErrUsableImage = errors.New("Image not found")
ErrCallNotFound = errors.New("Call not found")
ErrTaskInvalidAppAndRoute = errors.New("Unable to get call for given app and route")
)

12
api/models/logs.go Normal file
View File

@@ -0,0 +1,12 @@
package models
import (
"context"
)
type FnLog interface {
InsertLog(ctx context.Context, callID string, callLog string) error
GetLog(ctx context.Context, callID string) (*FnCallLog, error)
DeleteLog(ctx context.Context, callID string) error
}

13
api/models/task.go
View File

@@ -4,6 +4,7 @@ package models
// Editing this file might prove futile when you re-run the swagger generate command
import (
apierrors "errors"
"encoding/json"
strfmt "github.com/go-openapi/strfmt"
@@ -28,6 +29,12 @@ const (
FormatHTTP = "http"
)
var (
ErrCallNotFound = apierrors.New("Call not found")
ErrCallLogNotFound = apierrors.New("Call log not found")
ErrCallLogRemoving = apierrors.New("Could not remove call log")
)
type FnCall struct {
IDStatus
CompletedAt strfmt.DateTime `json:"completed_at,omitempty"`
@@ -37,6 +44,12 @@ type FnCall struct {
Path string `json:"path"`
}
type FnCallLog struct {
CallID string `json:"call_id"`
Log string `json:"log"`
}
func (fnCall *FnCall) FromTask(task *Task) *FnCall {
return &FnCall{
CreatedAt: task.CreatedAt,

2
api/runner/async_runner.go
View File

@@ -135,7 +135,7 @@ func startAsyncRunners(ctx context.Context, url string, rnr *Runner, ds models.D
go func() {
defer wg.Done()
// Process Task
_, err := rnr.RunTrackedTask(task, ctx, getCfg(task), ds)
_, err := rnr.RunTrackedTask(task, ctx, getCfg(task))
if err != nil {
log.WithError(err).Error("Cannot run task")
}

5
api/runner/async_runner_test.go
View File

@@ -20,6 +20,7 @@ import (
"gitlab-odx.oracle.com/odx/functions/api/mqs"
"gitlab-odx.oracle.com/odx/functions/api/runner/drivers"
"gitlab-odx.oracle.com/odx/functions/api/runner/task"
"gitlab-odx.oracle.com/odx/functions/api/logs"
)
func setLogBuffer() *bytes.Buffer {
@@ -193,7 +194,9 @@ func TestTasksrvURL(t *testing.T) {
func testRunner(t *testing.T) (*Runner, context.CancelFunc) {
ctx, cancel := context.WithCancel(context.Background())
r, err := New(ctx, NewFuncLogger(), NewMetricLogger())
ds := datastore.NewMock()
fnl := logs.NewMock()
r, err := New(ctx, NewFuncLogger(fnl), NewMetricLogger(), ds)
if err != nil {
t.Fatal("Test: failed to create new runner")
}

42
api/runner/func_logger.go
View File

@@ -2,40 +2,52 @@ package runner
import (
"bufio"
"fmt"
"io"
"context"
"github.com/Sirupsen/logrus"
"gitlab-odx.oracle.com/odx/functions/api/models"
"gitlab-odx.oracle.com/odx/functions/api/runner/common"
)
type FuncLogger interface {
Writer(context.Context, string, string, string, string) io.Writer
Writer(ctx context.Context, appName, path, image, reqID string) io.Writer
}
// FuncLogger reads STDERR output from a container and outputs it in a parseable structured log format, see: https://github.com/treeder/functions/issues/76
// FuncLogger reads STDERR output from a container and outputs it in a parsed structured log format, see: https://github.com/treeder/functions/issues/76
type DefaultFuncLogger struct {
logDB models.FnLog
}
func NewFuncLogger() FuncLogger {
return &DefaultFuncLogger{}
func NewFuncLogger(logDB models.FnLog) FuncLogger {
return &DefaultFuncLogger{logDB}
}
func (l *DefaultFuncLogger) persistLog(ctx context.Context, log logrus.FieldLogger, reqID, logText string) {
err := l.logDB.InsertLog(ctx, reqID, logText)
if err != nil {
log.WithError(err).Println(fmt.Sprintf(
"Unable to persist log for call %v. Error: %v", reqID, err))
}
}
func (l *DefaultFuncLogger) Writer(ctx context.Context, appName, path, image, reqID string) io.Writer {
r, w := io.Pipe()
log := common.Logger(ctx)
log = log.WithFields(logrus.Fields{"user_log": true, "app_name": appName, "path": path, "image": image, "call_id": reqID})
go func(reader io.Reader) {
scanner := bufio.NewScanner(reader)
for scanner.Scan() {
log.Println(scanner.Text())
}
if err := scanner.Err(); err != nil {
log.WithError(err).Println("There was an error with the scanner in attached container")
}
}(r)
log := common.Logger(ctx)
log = log.WithFields(logrus.Fields{"user_log": true, "app_name": appName,
"path": path, "image": image, "call_id": reqID})
var res string
errMsg := "-------Unable to get full log, it's too big-------"
fmt.Fscanf(reader, "%v", &res)
if len(res) >= bufio.MaxScanTokenSize {
res = res[0:bufio.MaxScanTokenSize - len(errMsg)] + errMsg
}
l.persistLog(ctx, log, reqID, res)
}(r)
return w
}

5
api/runner/runner.go
View File

@@ -14,6 +14,7 @@ import (
"time"
"github.com/Sirupsen/logrus"
"gitlab-odx.oracle.com/odx/functions/api/models"
"gitlab-odx.oracle.com/odx/functions/api/runner/common"
"gitlab-odx.oracle.com/odx/functions/api/runner/drivers"
"gitlab-odx.oracle.com/odx/functions/api/runner/drivers/docker"
@@ -33,6 +34,7 @@ type Runner struct {
usedMem int64
usedMemMutex sync.RWMutex
hcmgr htfnmgr
datastore models.Datastore
stats
}
@@ -48,7 +50,7 @@ const (
DefaultIdleTimeout = 30 * time.Second
)
func New(ctx context.Context, flog FuncLogger, mlog MetricLogger) (*Runner, error) {
func New(ctx context.Context, flog FuncLogger, mlog MetricLogger, ds models.Datastore) (*Runner, error) {
// TODO: Is this really required for the container drivers? Can we remove it?
env := common.NewEnvironment(func(e *common.Environment) {})
@@ -65,6 +67,7 @@ func New(ctx context.Context, flog FuncLogger, mlog MetricLogger) (*Runner, erro
mlog: mlog,
availableMem: getAvailableMemory(),
usedMem: 0,
datastore: ds,
}
go r.queueHandler(ctx)

11
api/runner/runner_test.go
View File

@@ -8,8 +8,10 @@ import (
"testing"
"time"
"gitlab-odx.oracle.com/odx/functions/api/datastore"
"gitlab-odx.oracle.com/odx/functions/api/models"
"gitlab-odx.oracle.com/odx/functions/api/runner/task"
"gitlab-odx.oracle.com/odx/functions/api/logs"
)
func TestRunnerHello(t *testing.T) {
@@ -17,11 +19,14 @@ func TestRunnerHello(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
runner, err := New(ctx, NewFuncLogger(), NewMetricLogger())
ds := datastore.NewMock()
fnl := logs.NewMock()
runner, err := New(ctx, NewFuncLogger(fnl), NewMetricLogger(), ds)
if err != nil {
t.Fatalf("Test error during New() - %s", err)
}
for i, test := range []struct {
route *models.Route
payload string
@@ -71,7 +76,9 @@ func TestRunnerError(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
runner, err := New(ctx, NewFuncLogger(), NewMetricLogger())
ds := datastore.NewMock()
fnl := logs.NewMock()
runner, err := New(ctx, NewFuncLogger(fnl), NewMetricLogger(), ds)
if err != nil {
t.Fatalf("Test error during New() - %s", err)
}

4
api/runner/worker.go
View File

@@ -60,7 +60,7 @@ import (
// (internal clock)
// RunTrackedTask is just a wrapper for shared logic for async/sync runners
func (rnr *Runner) RunTrackedTask(newTask *models.Task, ctx context.Context, cfg *task.Config, ds models.Datastore) (drivers.RunResult, error) {
func (rnr *Runner) RunTrackedTask(newTask *models.Task, ctx context.Context, cfg *task.Config) (drivers.RunResult, error) {
startedAt := strfmt.DateTime(time.Now())
newTask.StartedAt = startedAt
@@ -78,7 +78,7 @@ func (rnr *Runner) RunTrackedTask(newTask *models.Task, ctx context.Context, cfg
newTask.CompletedAt = completedAt
newTask.Status = status
if err := ds.InsertTask(ctx, newTask); err != nil {
if err := rnr.datastore.InsertTask(ctx, newTask); err != nil {
// TODO we should just log this error not return it to user? just issue storing task status but task is run
logrus.WithError(err).Error("error inserting task into datastore")
}

50
api/server/apps_test.go
View File

@@ -12,6 +12,7 @@ import (
"gitlab-odx.oracle.com/odx/functions/api/datastore"
"gitlab-odx.oracle.com/odx/functions/api/models"
"gitlab-odx.oracle.com/odx/functions/api/mqs"
"gitlab-odx.oracle.com/odx/functions/api/logs"
)
func setLogBuffer() *bytes.Buffer {
@@ -28,25 +29,26 @@ func TestAppCreate(t *testing.T) {
buf := setLogBuffer()
for i, test := range []struct {
mock models.Datastore
logDB models.FnLog
path string
body string
expectedCode int
expectedError error
}{
// errors
{datastore.NewMock(), "/v1/apps", ``, http.StatusBadRequest, models.ErrInvalidJSON},
{datastore.NewMock(), "/v1/apps", `{}`, http.StatusBadRequest, models.ErrAppsMissingNew},
{datastore.NewMock(), "/v1/apps", `{ "name": "Test" }`, http.StatusBadRequest, models.ErrAppsMissingNew},
{datastore.NewMock(), "/v1/apps", `{ "app": { "name": "" } }`, http.StatusInternalServerError, models.ErrAppsValidationMissingName},
{datastore.NewMock(), "/v1/apps", `{ "app": { "name": "1234567890123456789012345678901" } }`, http.StatusInternalServerError, models.ErrAppsValidationTooLongName},
{datastore.NewMock(), "/v1/apps", `{ "app": { "name": "&&%@!#$#@$" } }`, http.StatusInternalServerError, models.ErrAppsValidationInvalidName},
{datastore.NewMock(), "/v1/apps", `{ "app": { "name": "&&%@!#$#@$" } }`, http.StatusInternalServerError, models.ErrAppsValidationInvalidName},
{datastore.NewMock(), logs.NewMock(),"/v1/apps", ``, http.StatusBadRequest, models.ErrInvalidJSON},
{datastore.NewMock(), logs.NewMock(),"/v1/apps", `{}`, http.StatusBadRequest, models.ErrAppsMissingNew},
{datastore.NewMock(), logs.NewMock(),"/v1/apps", `{ "name": "Test" }`, http.StatusBadRequest, models.ErrAppsMissingNew},
{datastore.NewMock(), logs.NewMock(),"/v1/apps", `{ "app": { "name": "" } }`, http.StatusInternalServerError, models.ErrAppsValidationMissingName},
{datastore.NewMock(), logs.NewMock(),"/v1/apps", `{ "app": { "name": "1234567890123456789012345678901" } }`, http.StatusInternalServerError, models.ErrAppsValidationTooLongName},
{datastore.NewMock(), logs.NewMock(),"/v1/apps", `{ "app": { "name": "&&%@!#$#@$" } }`, http.StatusInternalServerError, models.ErrAppsValidationInvalidName},
{datastore.NewMock(), logs.NewMock(),"/v1/apps", `{ "app": { "name": "&&%@!#$#@$" } }`, http.StatusInternalServerError, models.ErrAppsValidationInvalidName},
// success
{datastore.NewMock(), "/v1/apps", `{ "app": { "name": "teste" } }`, http.StatusOK, nil},
{datastore.NewMock(), logs.NewMock(),"/v1/apps", `{ "app": { "name": "teste" } }`, http.StatusOK, nil},
} {
rnr, cancel := testRunner(t)
srv := testServer(test.mock, &mqs.Mock{}, rnr)
srv := testServer(test.mock, &mqs.Mock{}, test.logDB, rnr)
router := srv.Router
body := bytes.NewBuffer([]byte(test.body))
@@ -76,20 +78,21 @@ func TestAppDelete(t *testing.T) {
for i, test := range []struct {
ds models.Datastore
logDB models.FnLog
path string
body string
expectedCode int
expectedError error
}{
{datastore.NewMock(), "/v1/apps/myapp", "", http.StatusNotFound, nil},
{datastore.NewMock(), logs.NewMock(),"/v1/apps/myapp", "", http.StatusNotFound, nil},
{datastore.NewMockInit(
[]*models.App{{
Name: "myapp",
}}, nil, nil,
), "/v1/apps/myapp", "", http.StatusOK, nil},
}}, nil, nil, nil,
), logs.NewMock(),"/v1/apps/myapp", "", http.StatusOK, nil},
} {
rnr, cancel := testRunner(t)
srv := testServer(test.ds, &mqs.Mock{}, rnr)
srv := testServer(test.ds, &mqs.Mock{}, test.logDB, rnr)
_, rec := routerRequest(t, srv.Router, "DELETE", test.path, nil)
@@ -117,7 +120,9 @@ func TestAppList(t *testing.T) {
rnr, cancel := testRunner(t)
defer cancel()
srv := testServer(datastore.NewMock(), &mqs.Mock{}, rnr)
ds := datastore.NewMock()
fnl := logs.NewMock()
srv := testServer(ds, &mqs.Mock{}, fnl, rnr)
for i, test := range []struct {
path string
@@ -152,7 +157,9 @@ func TestAppGet(t *testing.T) {
rnr, cancel := testRunner(t)
defer cancel()
srv := testServer(datastore.NewMock(), &mqs.Mock{}, rnr)
ds := datastore.NewMock()
fnl := logs.NewMock()
srv := testServer(ds, &mqs.Mock{}, fnl, rnr)
for i, test := range []struct {
path string
@@ -187,30 +194,31 @@ func TestAppUpdate(t *testing.T) {
for i, test := range []struct {
mock models.Datastore
logDB models.FnLog
path string
body string
expectedCode int
expectedError error
}{
// errors
{datastore.NewMock(), "/v1/apps/myapp", ``, http.StatusBadRequest, models.ErrInvalidJSON},
{datastore.NewMock(), logs.NewMock(),"/v1/apps/myapp", ``, http.StatusBadRequest, models.ErrInvalidJSON},
// success
{datastore.NewMockInit(
[]*models.App{{
Name: "myapp",
}}, nil, nil,
), "/v1/apps/myapp", `{ "app": { "config": { "test": "1" } } }`, http.StatusOK, nil},
}}, nil, nil, nil,
), logs.NewMock(),"/v1/apps/myapp", `{ "app": { "config": { "test": "1" } } }`, http.StatusOK, nil},
// Addresses #380
{datastore.NewMockInit(
[]*models.App{{
Name: "myapp",
}}, nil, nil,
), "/v1/apps/myapp", `{ "app": { "name": "othername" } }`, http.StatusBadRequest, nil},
}}, nil, nil, nil,
), logs.NewMock(),"/v1/apps/myapp", `{ "app": { "name": "othername" } }`, http.StatusBadRequest, nil},
} {
rnr, cancel := testRunner(t)
srv := testServer(test.mock, &mqs.Mock{}, rnr)
srv := testServer(test.mock, &mqs.Mock{}, test.logDB, rnr)
body := bytes.NewBuffer([]byte(test.body))
_, rec := routerRequest(t, srv.Router, "PATCH", test.path, body)

46
api/server/call_logs.go Normal file
View File

@@ -0,0 +1,46 @@
package server
import (
"context"
"net/http"
"github.com/gin-gonic/gin"
"gitlab-odx.oracle.com/odx/functions/api"
)
func (s *Server) handleCallLogGet(c *gin.Context) {
ctx := c.MustGet("ctx").(context.Context)
callID := c.Param(api.Call)
_, err := s.Datastore.GetTask(ctx, callID)
if err != nil {
handleErrorResponse(c, err)
return
}
callObj, err := s.LogDB.GetLog(ctx, callID)
if err != nil {
handleErrorResponse(c, err)
return
}
c.JSON(http.StatusOK, fnCallLogResponse{"Successfully loaded call", callObj})
}
func (s *Server) handleCallLogDelete(c *gin.Context) {
ctx := c.MustGet("ctx").(context.Context)
callID := c.Param(api.Call)
_, err := s.Datastore.GetTask(ctx, callID)
if err != nil {
handleErrorResponse(c, err)
return
}
err = s.LogDB.DeleteLog(ctx, callID)
if err != nil {
handleErrorResponse(c, err)
return
}
c.JSON(http.StatusAccepted, gin.H{"message": "Log delete accepted"})
}

1
api/server/error_response.go
View File

@@ -21,6 +21,7 @@ var errStatusCode = map[error]int{
models.ErrRoutesNotFound: http.StatusNotFound,
models.ErrRoutesAlreadyExists: http.StatusConflict,
models.ErrCallNotFound: http.StatusNotFound,
models.ErrCallLogNotFound: http.StatusNotFound,
}
func handleErrorResponse(c *gin.Context, err error) {

1
api/server/init.go
View File

@@ -24,6 +24,7 @@ func init() {
viper.SetDefault(EnvLogLevel, "info")
viper.SetDefault(EnvMQURL, fmt.Sprintf("bolt://%s/data/worker_mq.db", cwd))
viper.SetDefault(EnvDBURL, fmt.Sprintf("bolt://%s/data/bolt.db?bucket=funcs", cwd))
viper.SetDefault(EnvLOGDBURL, fmt.Sprintf("bolt://%s/data/log.db?bucket=funcs", cwd))
viper.SetDefault(EnvPort, 8080)
viper.SetDefault(EnvAPIURL, fmt.Sprintf("http://127.0.0.1:%d", viper.GetInt(EnvPort)))
viper.AutomaticEnv() // picks up env vars automatically

62
api/server/routes_test.go
View File

@@ -9,6 +9,7 @@ import (
"gitlab-odx.oracle.com/odx/functions/api/datastore"
"gitlab-odx.oracle.com/odx/functions/api/models"
"gitlab-odx.oracle.com/odx/functions/api/mqs"
"gitlab-odx.oracle.com/odx/functions/api/logs"
)
func TestRouteCreate(t *testing.T) {
@@ -16,26 +17,27 @@ func TestRouteCreate(t *testing.T) {
for i, test := range []struct {
mock models.Datastore
logDB models.FnLog
path string
body string
expectedCode int
expectedError error
}{
// errors
{datastore.NewMock(), "/v1/apps/a/routes", ``, http.StatusBadRequest, models.ErrInvalidJSON},
{datastore.NewMock(), "/v1/apps/a/routes", `{ }`, http.StatusBadRequest, models.ErrRoutesMissingNew},
{datastore.NewMock(), "/v1/apps/a/routes", `{ "path": "/myroute" }`, http.StatusBadRequest, models.ErrRoutesMissingNew},
{datastore.NewMock(), "/v1/apps/a/routes", `{ "route": { } }`, http.StatusBadRequest, models.ErrRoutesValidationMissingPath},
{datastore.NewMock(), "/v1/apps/a/routes", `{ "route": { "path": "/myroute" } }`, http.StatusBadRequest, models.ErrRoutesValidationMissingImage},
{datastore.NewMock(), "/v1/apps/a/routes", `{ "route": { "image": "funcy/hello" } }`, http.StatusBadRequest, models.ErrRoutesValidationMissingPath},
{datastore.NewMock(), "/v1/apps/a/routes", `{ "route": { "image": "funcy/hello", "path": "myroute" } }`, http.StatusBadRequest, models.ErrRoutesValidationInvalidPath},
{datastore.NewMock(), "/v1/apps/$/routes", `{ "route": { "image": "funcy/hello", "path": "/myroute" } }`, http.StatusInternalServerError, models.ErrAppsValidationInvalidName},
{datastore.NewMock(), logs.NewMock(),"/v1/apps/a/routes", ``, http.StatusBadRequest, models.ErrInvalidJSON},
{datastore.NewMock(), logs.NewMock(),"/v1/apps/a/routes", `{ }`, http.StatusBadRequest, models.ErrRoutesMissingNew},
{datastore.NewMock(), logs.NewMock(),"/v1/apps/a/routes", `{ "path": "/myroute" }`, http.StatusBadRequest, models.ErrRoutesMissingNew},
{datastore.NewMock(), logs.NewMock(),"/v1/apps/a/routes", `{ "route": { } }`, http.StatusBadRequest, models.ErrRoutesValidationMissingPath},
{datastore.NewMock(), logs.NewMock(),"/v1/apps/a/routes", `{ "route": { "path": "/myroute" } }`, http.StatusBadRequest, models.ErrRoutesValidationMissingImage},
{datastore.NewMock(), logs.NewMock(),"/v1/apps/a/routes", `{ "route": { "image": "funcy/hello" } }`, http.StatusBadRequest, models.ErrRoutesValidationMissingPath},
{datastore.NewMock(), logs.NewMock(),"/v1/apps/a/routes", `{ "route": { "image": "funcy/hello", "path": "myroute" } }`, http.StatusBadRequest, models.ErrRoutesValidationInvalidPath},
{datastore.NewMock(), logs.NewMock(),"/v1/apps/$/routes", `{ "route": { "image": "funcy/hello", "path": "/myroute" } }`, http.StatusInternalServerError, models.ErrAppsValidationInvalidName},
// success
{datastore.NewMock(), "/v1/apps/a/routes", `{ "route": { "image": "funcy/hello", "path": "/myroute" } }`, http.StatusOK, nil},
{datastore.NewMock(), logs.NewMock(),"/v1/apps/a/routes", `{ "route": { "image": "funcy/hello", "path": "/myroute" } }`, http.StatusOK, nil},
} {
rnr, cancel := testRunner(t)
srv := testServer(test.mock, &mqs.Mock{}, rnr)
srv := testServer(test.mock, &mqs.Mock{}, test.logDB, rnr)
body := bytes.NewBuffer([]byte(test.body))
_, rec := routerRequest(t, srv.Router, "POST", test.path, body)
@@ -67,20 +69,21 @@ func TestRouteDelete(t *testing.T) {
for i, test := range []struct {
ds models.Datastore
logDB models.FnLog
path string
body string
expectedCode int
expectedError error
}{
{datastore.NewMock(), "/v1/apps/a/routes/missing", "", http.StatusNotFound, nil},
{datastore.NewMock(), logs.NewMock(),"/v1/apps/a/routes/missing", "", http.StatusNotFound, nil},
{datastore.NewMockInit(nil,
[]*models.Route{
{Path: "/myroute", AppName: "a"},
}, nil,
), "/v1/apps/a/routes/myroute", "", http.StatusOK, nil},
}, nil, nil,
), logs.NewMock(),"/v1/apps/a/routes/myroute", "", http.StatusOK, nil},
} {
rnr, cancel := testRunner(t)
srv := testServer(test.ds, &mqs.Mock{}, rnr)
srv := testServer(test.ds, &mqs.Mock{}, test.logDB, rnr)
_, rec := routerRequest(t, srv.Router, "DELETE", test.path, nil)
if rec.Code != test.expectedCode {
@@ -107,7 +110,11 @@ func TestRouteList(t *testing.T) {
rnr, cancel := testRunner(t)
defer cancel()
srv := testServer(datastore.NewMock(), &mqs.Mock{}, rnr)
ds := datastore.NewMock()
fnl := logs.NewMock()
srv := testServer(ds, &mqs.Mock{}, fnl, rnr)
for i, test := range []struct {
path string
@@ -143,7 +150,10 @@ func TestRouteGet(t *testing.T) {
rnr, cancel := testRunner(t)
defer cancel()
srv := testServer(datastore.NewMock(), &mqs.Mock{}, rnr)
ds := datastore.NewMock()
fnl := logs.NewMock()
srv := testServer(ds, &mqs.Mock{}, fnl, rnr)
for i, test := range []struct {
path string
@@ -178,16 +188,17 @@ func TestRouteUpdate(t *testing.T) {
for i, test := range []struct {
ds models.Datastore
logDB models.FnLog
path string
body string
expectedCode int
expectedError error
}{
// errors
{datastore.NewMock(), "/v1/apps/a/routes/myroute/do", ``, http.StatusBadRequest, models.ErrInvalidJSON},
{datastore.NewMock(), "/v1/apps/a/routes/myroute/do", `{}`, http.StatusBadRequest, models.ErrRoutesMissingNew},
{datastore.NewMock(), "/v1/apps/a/routes/myroute/do", `{ "route": { "type": "invalid-type" } }`, http.StatusBadRequest, nil},
{datastore.NewMock(), "/v1/apps/a/routes/myroute/do", `{ "route": { "format": "invalid-format" } }`, http.StatusBadRequest, nil},
{datastore.NewMock(), logs.NewMock(),"/v1/apps/a/routes/myroute/do", ``, http.StatusBadRequest, models.ErrInvalidJSON},
{datastore.NewMock(), logs.NewMock(),"/v1/apps/a/routes/myroute/do", `{}`, http.StatusBadRequest, models.ErrRoutesMissingNew},
{datastore.NewMock(), logs.NewMock(),"/v1/apps/a/routes/myroute/do", `{ "route": { "type": "invalid-type" } }`, http.StatusBadRequest, nil},
{datastore.NewMock(), logs.NewMock(),"/v1/apps/a/routes/myroute/do", `{ "route": { "format": "invalid-format" } }`, http.StatusBadRequest, nil},
// success
{datastore.NewMockInit(nil,
@@ -196,8 +207,8 @@ func TestRouteUpdate(t *testing.T) {
AppName: "a",
Path: "/myroute/do",
},
}, nil,
), "/v1/apps/a/routes/myroute/do", `{ "route": { "image": "funcy/hello" } }`, http.StatusOK, nil},
}, nil, nil,
), logs.NewMock(),"/v1/apps/a/routes/myroute/do", `{ "route": { "image": "funcy/hello" } }`, http.StatusOK, nil},
// Addresses #381
{datastore.NewMockInit(nil,
@@ -206,11 +217,12 @@ func TestRouteUpdate(t *testing.T) {
AppName: "a",
Path: "/myroute/do",
},
}, nil,
), "/v1/apps/a/routes/myroute/do", `{ "route": { "path": "/otherpath" } }`, http.StatusBadRequest, nil},
}, nil, nil,
), logs.NewMock(),"/v1/apps/a/routes/myroute/do", `{ "route": { "path": "/otherpath" } }`, http.StatusBadRequest, nil},
} {
rnr, cancel := testRunner(t)
srv := testServer(test.ds, &mqs.Mock{}, rnr)
srv := testServer(test.ds, &mqs.Mock{}, test.logDB, rnr)
body := bytes.NewBuffer([]byte(test.body))

2
api/server/runner.go
View File

@@ -245,7 +245,7 @@ func (s *Server) serve(ctx context.Context, c *gin.Context, appName string, foun
c.JSON(http.StatusAccepted, map[string]string{"call_id": newTask.ID})
default:
result, err := s.Runner.RunTrackedTask(newTask, ctx, cfg, s.Datastore)
result, err := s.Runner.RunTrackedTask(newTask, ctx, cfg)
if result != nil {
waitTime := result.StartTime().Sub(cfg.ReceivedTime)
c.Header("XXX-FXLB-WAIT", waitTime.String())

2
api/server/runner_async_test.go
View File

@@ -45,7 +45,7 @@ func TestRouteRunnerAsyncExecution(t *testing.T) {
{Type: "async", Path: "/myroute", AppName: "myapp", Image: "funcy/hello", Config: map[string]string{"test": "true"}},
{Type: "async", Path: "/myerror", AppName: "myapp", Image: "funcy/error", Config: map[string]string{"test": "true"}},
{Type: "async", Path: "/myroute/:param", AppName: "myapp", Image: "funcy/hello", Config: map[string]string{"test": "true"}},
}, nil,
}, nil, nil,
)
mq := &mqs.Mock{}

42
api/server/runner_test.go
View File

@@ -11,11 +11,14 @@ import (
"gitlab-odx.oracle.com/odx/functions/api/models"
"gitlab-odx.oracle.com/odx/functions/api/mqs"
"gitlab-odx.oracle.com/odx/functions/api/runner"
"gitlab-odx.oracle.com/odx/functions/api/logs"
)
func testRunner(t *testing.T) (*runner.Runner, context.CancelFunc) {
ctx, cancel := context.WithCancel(context.Background())
r, err := runner.New(ctx, runner.NewFuncLogger(), runner.NewMetricLogger())
ds := datastore.NewMock()
fnl := logs.NewMock()
r, err := runner.New(ctx, runner.NewFuncLogger(fnl), runner.NewMetricLogger(), ds)
if err != nil {
t.Fatal("Test: failed to create new runner")
}
@@ -24,15 +27,15 @@ func testRunner(t *testing.T) (*runner.Runner, context.CancelFunc) {
func TestRouteRunnerGet(t *testing.T) {
buf := setLogBuffer()
rnr, cancel := testRunner(t)
defer cancel()
srv := testServer(datastore.NewMockInit(
ds := datastore.NewMockInit(
[]*models.App{
{Name: "myapp", Config: models.Config{}},
}, nil, nil,
), &mqs.Mock{}, rnr)
}, nil, nil, nil,
)
logDB := logs.NewMock()
srv := testServer(ds, &mqs.Mock{}, logDB, rnr)
for i, test := range []struct {
path string
@@ -70,11 +73,13 @@ func TestRouteRunnerPost(t *testing.T) {
rnr, cancel := testRunner(t)
defer cancel()
srv := testServer(datastore.NewMockInit(
ds := datastore.NewMockInit(
[]*models.App{
{Name: "myapp", Config: models.Config{}},
}, nil, nil,
), &mqs.Mock{}, rnr)
}, nil, nil, nil,
)
fnl := logs.NewMock()
srv := testServer(ds, &mqs.Mock{}, fnl, rnr)
for i, test := range []struct {
path string
@@ -114,15 +119,20 @@ func TestRouteRunnerExecution(t *testing.T) {
rnr, cancelrnr := testRunner(t)
defer cancelrnr()
srv := testServer(datastore.NewMockInit(
ds := datastore.NewMockInit(
[]*models.App{
{Name: "myapp", Config: models.Config{}},
},
[]*models.Route{
{Path: "/myroute", AppName: "myapp", Image: "funcy/hello", Headers: map[string][]string{"X-Function": {"Test"}}},
{Path: "/myerror", AppName: "myapp", Image: "funcy/error", Headers: map[string][]string{"X-Function": {"Test"}}},
}, nil,
), &mqs.Mock{}, rnr)
}, nil, nil,
)
fnl := logs.NewMock()
srv := testServer(ds, &mqs.Mock{}, fnl, rnr)
for i, test := range []struct {
path string
@@ -167,14 +177,16 @@ func TestRouteRunnerTimeout(t *testing.T) {
rnr, cancelrnr := testRunner(t)
defer cancelrnr()
srv := testServer(datastore.NewMockInit(
ds := datastore.NewMockInit(
[]*models.App{
{Name: "myapp", Config: models.Config{}},
},
[]*models.Route{
{Path: "/sleeper", AppName: "myapp", Image: "funcy/sleeper", Timeout: 1},
}, nil,
), &mqs.Mock{}, rnr)
}, nil, nil,
)
fnl := logs.NewMock()
srv := testServer(ds, &mqs.Mock{}, fnl, rnr)
for i, test := range []struct {
path string

34
api/server/server.go
View File

@@ -24,22 +24,25 @@ import (
"gitlab-odx.oracle.com/odx/functions/api/runner"
"gitlab-odx.oracle.com/odx/functions/api/runner/common"
"gitlab-odx.oracle.com/odx/functions/api/server/internal/routecache"
"gitlab-odx.oracle.com/odx/functions/api/logs"
)
const (
EnvLogLevel = "log_level"
EnvMQURL = "mq_url"
EnvDBURL = "db_url"
EnvLOGDBURL = "logstore_url"
EnvPort = "port" // be careful, Gin expects this variable to be "port"
EnvAPIURL = "api_url"
)
type Server struct {
Datastore models.Datastore
Runner *runner.Runner
Router *gin.Engine
MQ models.MessageQueue
Enqueue models.Enqueue
Datastore models.Datastore
Runner *runner.Runner
Router *gin.Engine
MQ models.MessageQueue
Enqueue models.Enqueue
LogDB models.FnLog
apiURL string
@@ -67,17 +70,22 @@ func NewFromEnv(ctx context.Context) *Server {
logrus.WithError(err).Fatal("Error initializing message queue.")
}
logDB, err := logs.New(viper.GetString(EnvLOGDBURL))
if err != nil {
logrus.WithError(err).Fatal("Error initializing logs store.")
}
apiURL := viper.GetString(EnvAPIURL)
return New(ctx, ds, mq, apiURL)
return New(ctx, ds, mq, logDB, apiURL)
}
// New creates a new Functions server with the passed in datastore, message queue and API URL
func New(ctx context.Context, ds models.Datastore, mq models.MessageQueue, apiURL string, opts ...ServerOption) *Server {
func New(ctx context.Context, ds models.Datastore, mq models.MessageQueue, logDB models.FnLog, apiURL string, opts ...ServerOption) *Server {
metricLogger := runner.NewMetricLogger()
funcLogger := runner.NewFuncLogger()
funcLogger := runner.NewFuncLogger(logDB)
rnr, err := runner.New(ctx, funcLogger, metricLogger)
rnr, err := runner.New(ctx, funcLogger, metricLogger, ds)
if err != nil {
logrus.WithError(err).Fatalln("Failed to create a runner")
return nil
@@ -89,6 +97,7 @@ func New(ctx context.Context, ds models.Datastore, mq models.MessageQueue, apiUR
Datastore: ds,
MQ: mq,
hotroutes: routecache.New(cacheSize),
LogDB: logDB,
Enqueue: DefaultEnqueue,
apiURL: apiURL,
}
@@ -302,6 +311,8 @@ func (s *Server) bindHandlers(ctx context.Context) {
v1.GET("/routes", s.handleRouteList)
v1.GET("/calls/:call", s.handleCallGet)
v1.GET("/calls/:call/log", s.handleCallLogGet)
v1.DELETE("/calls/:call/log", s.handleCallLogDelete)
apps := v1.Group("/apps/:app")
{
@@ -356,3 +367,8 @@ type fnCallsResponse struct {
Message string `json:"message"`
Calls models.FnCalls `json:"calls"`
}
type fnCallLogResponse struct {
Message string `json:"message"`
Log *models.FnCallLog `json:"log"`
}

31
api/server/server_test.go
View File

@@ -17,17 +17,21 @@ import (
"gitlab-odx.oracle.com/odx/functions/api/mqs"
"gitlab-odx.oracle.com/odx/functions/api/runner"
"gitlab-odx.oracle.com/odx/functions/api/server/internal/routecache"
"gitlab-odx.oracle.com/odx/functions/api/logs"
)
var tmpBolt = "/tmp/func_test_bolt.db"
var tmpDatastoreBolt = "/tmp/func_test_bolt_datastore.db"
var tmpLogBolt = "/tmp/func_test_bolt_log.db"
func testServer(ds models.Datastore, mq models.MessageQueue, rnr *runner.Runner) *Server {
func testServer(ds models.Datastore, mq models.MessageQueue, logDB models.FnLog, rnr *runner.Runner) *Server {
ctx := context.Background()
s := &Server{
Runner: rnr,
Router: gin.New(),
Datastore: ds,
LogDB: nil,
MQ: mq,
Enqueue: DefaultEnqueue,
hotroutes: routecache.New(2),
@@ -79,26 +83,33 @@ func getErrorResponse(t *testing.T, rec *httptest.ResponseRecorder) models.Error
return errResp
}
func prepareBolt(t *testing.T) (models.Datastore, func()) {
os.Remove(tmpBolt)
ds, err := datastore.New("bolt://" + tmpBolt)
func prepareBolt(ctx context.Context, t *testing.T) (models.Datastore, models.FnLog, func()) {
os.Remove(tmpDatastoreBolt)
os.Remove(tmpLogBolt)
ds, err := datastore.New("bolt://" + tmpDatastoreBolt)
if err != nil {
t.Fatal("Error when creating datastore: %s", err)
t.Fatalf("Error when creating datastore: %s", err)
}
return ds, func() {
os.Remove(tmpBolt)
logDB, err := logs.New("bolt://" + tmpLogBolt)
if err != nil {
t.Fatalf("Error when creating log store: %s", err)
}
return ds,logDB, func() {
os.Remove(tmpDatastoreBolt)
os.Remove(tmpLogBolt)
}
}
func TestFullStack(t *testing.T) {
ctx := context.Background()
buf := setLogBuffer()
ds, closeBolt := prepareBolt(t)
ds, logDB, closeBolt := prepareBolt(ctx, t)
defer closeBolt()
rnr, rnrcancel := testRunner(t)
defer rnrcancel()
srv := testServer(ds, &mqs.Mock{}, rnr)
srv := testServer(ds, &mqs.Mock{}, logDB, rnr)
srv.hotroutes = routecache.New(2)
for _, test := range []struct {

19
docs/operating/logs/README.md Normal file
View File

@@ -0,0 +1,19 @@
# Function logs
We currently support the following function logs stores and they are passed in via the `LOGSTORE_URL` environment variable. For example:
Maximum size of single log entry: 4Mb
```sh
docker run -e "LOGSTORE_URL=bolt:///functions/logs/bolt.db" ...
```
## [Bolt](https://github.com/boltdb/bolt) (default)
URL: `bolt:///functions/logs/bolt.db`
Bolt is an embedded database which stores to disk. If you want to use this, be sure you don't lose the data directory by mounting
the directory on your host. eg: `docker run -v $PWD/data:/functions/data -e LOGSTORE_URL=bolt:///functions/data/bolt.db ...`
[More on BoltDB](../databases/boltdb.md)

65
docs/swagger.yml
View File

@@ -317,6 +317,52 @@ paths:
schema:
$ref: '#/definitions/Error'
/calls/{call}/log:
get:
summary: Get call logs
description: Get call logs
tags:
- Call
- Log
parameters:
- name: call
description: Call ID.
required: true
type: string
in: path
responses:
200:
description: Log found
schema:
$ref: '#/definitions/LogWrapper'
404:
description: Log not found.
schema:
$ref: '#/definitions/Error'
delete:
summary: Delete call log entry
description: Delete call log entry
tags:
- Call
- Log
parameters:
- name: call
description: Call ID.
required: true
type: string
in: path
responses:
202:
description: Log delete request accepted
404:
description: Does not exist.
schema:
$ref: '#/definitions/Error'
default:
description: Unexpected error
schema:
$ref: '#/definitions/Error'
/calls/{call}:
get:
summary: Get call information
@@ -519,6 +565,25 @@ definitions:
$ref: '#/definitions/Call'
description: "Call object."
LogWrapper:
type: object
required:
- log
properties:
log:
$ref: '#/definitions/Log'
description: "Call log entry."
Log:
type: object
properties:
call_id:
type: string
description: Call UUID ID
log:
type: string # maybe bytes, long logs wouldn't fit into string type
Call:
type: object
properties:

8
examples/tutorial/logging/Dockerfile Normal file
View File

@@ -0,0 +1,8 @@
FROM funcy/go:dev as build-stage
WORKDIR /function
ADD . /src
RUN cd /src && go build -o func
FROM funcy/go
WORKDIR /function
COPY --from=build-stage /src/func /function/
ENTRYPOINT ["./func"]

5
examples/tutorial/logging/func.yaml Normal file
View File

@@ -0,0 +1,5 @@
name: funcy/stderr-logging
version: 0.0.1
runtime: go
entrypoint: ./func
path: /stderr-logging

29
examples/tutorial/logging/main.go Normal file
View File

@@ -0,0 +1,29 @@
package main
import (
"fmt"
"encoding/json"
"os"
"math/rand"
)
const lBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
type OutputSize struct {
Size int `json:"size"`
}
func RandStringBytes(n int) string {
b := make([]byte, n)
for i := range b {
b[i] = lBytes[rand.Intn(len(lBytes))]
}
return string(b)
}
func main() {
out := &OutputSize{Size: 64 * 1024}
json.NewDecoder(os.Stdin).Decode(out)
fmt.Fprintln(os.Stderr, RandStringBytes(out.Size))
}

3
examples/tutorial/logging/sample.payload.json Normal file
View File

@@ -0,0 +1,3 @@
{
"size": 8
}