alihan/fn-serverless
1
0
Fork 0
mirror of https://github.com/fnproject/fn.git synced 2022-10-28 21:29:17 +03:00
Code Issues Packages Projects Releases Wiki Activity

datastore no longer implements logstore (#1013)

Browse Source 
* datastore no longer implements logstore

the underlying implementation of our sql store implements both the datastore
and the logstore interface, however going forward we are likely to encounter
datastore implementers that would mock out the logstore interface and not use
its methods - signalling a poor interface. this remedies that, now they are 2
completely separate things, which our sqlstore happens to implement both of.

related to some recent changes around wrapping, this keeps the imposed metrics
and validation wrapping of a servers logstore and datastore, just moving it
into New instead of in the opts - this is so that a user can have the
underlying datastore in order to set the logstore to it, since wrapping it in
a validator/metrics would render it no longer a logstore implementer (i.e.
validate datastore doesn't implement the logstore interface), we need to do
this after setting the logstore to the datastore if one wasn't provided
explicitly.

* splits logstore and datastore metrics & validation logic
* `make test` should be `make full-test` always. got rid of that so that
nobody else has to wait for CI to blow up on them after the tests pass locally
ever again.

* fix new tests
This commit is contained in:
Reed Allman
2018-06-04 00:08:16 -07:00
committed by GitHub
parent e0425abd19
commit 00c29b8bf3
17 changed files with 250 additions and 127 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
api/server/hybrid.go
View File

@@ -165,7 +165,7 @@ func (s *Server) handleRunnerFinish(c *gin.Context) {
// TODO this needs UpdateCall functionality to work for async and should only work if:
// running->error|timeout|success
// TODO all async will fail here :( all sync will work fine :) -- *feeling conflicted*
if err := s.datastore.InsertCall(ctx, &call); err != nil {
if err := s.logstore.InsertCall(ctx, &call); err != nil {
common.Logger(ctx).WithError(err).Error("error inserting call into datastore")
// note: Not returning err here since the job could have already finished successfully.
}

29
api/server/runner_test.go
View File

@@ -42,6 +42,7 @@ func envTweaker(name, value string) func() {
func testRunner(_ *testing.T, args ...interface{}) (agent.Agent, context.CancelFunc) {
ds := datastore.NewMock()
ls := logs.NewMock()
var mq models.MessageQueue = &mqs.Mock{}
for _, a := range args {
switch arg := a.(type) {
@@ -49,9 +50,11 @@ func testRunner(_ *testing.T, args ...interface{}) (agent.Agent, context.CancelF
ds = arg
case models.MessageQueue:
mq = arg
case models.LogStore:
ls = arg
}
}
r := agent.New(agent.NewDirectDataAccess(ds, ds, mq))
r := agent.New(agent.NewDirectDataAccess(ds, ls, mq))
return r, func() { r.Close() }
}
@@ -169,11 +172,12 @@ func TestRouteRunnerExecEmptyBody(t *testing.T) {
{Path: "/hotjson", AppID: app.ID, Image: rImg, Type: "sync", Format: "json", Memory: 64, Timeout: 10, IdleTimeout: 20, Headers: rHdr, Config: rCfg},
},
)
ls := logs.NewMock()
rnr, cancelrnr := testRunner(t, ds)
rnr, cancelrnr := testRunner(t, ds, ls)
defer cancelrnr()
srv := testServer(ds, &mqs.Mock{}, ds, rnr, ServerTypeFull)
srv := testServer(ds, &mqs.Mock{}, ls, rnr, ServerTypeFull)
expHeaders := map[string][]string{"X-Function": {"Test"}}
emptyBody := `{"echoContent": "_TRX_ID_", "isDebug": true, "isEmptyBody": true}`
@@ -258,11 +262,12 @@ func TestRouteRunnerExecution(t *testing.T) {
{Path: "/mybigoutputjson", AppID: app.ID, Image: rImg, Type: "sync", Format: "json", Memory: 64, Timeout: 10, IdleTimeout: 20, Headers: rHdr, Config: rCfg},
},
)
ls := logs.NewMock()
rnr, cancelrnr := testRunner(t, ds)
rnr, cancelrnr := testRunner(t, ds, ls)
defer cancelrnr()
srv := testServer(ds, &mqs.Mock{}, ds, rnr, ServerTypeFull)
srv := testServer(ds, &mqs.Mock{}, ls, rnr, ServerTypeFull)
expHeaders := map[string][]string{"X-Function": {"Test"}, "Content-Type": {"application/json; charset=utf-8"}}
expCTHeaders := map[string][]string{"X-Function": {"Test"}, "Content-Type": {"foo/bar"}}
@@ -373,7 +378,7 @@ func TestRouteRunnerExecution(t *testing.T) {
for i, test := range testCases {
if test.expectedLogsSubStr != nil {
if !checkLogs(t, i, ds, callIds[i], test.expectedLogsSubStr) {
if !checkLogs(t, i, ls, callIds[i], test.expectedLogsSubStr) {
isFailure = true
}
}
@@ -403,7 +408,7 @@ func getDockerId(respBytes []byte) (string, error) {
return id, nil
}
func checkLogs(t *testing.T, tnum int, ds models.Datastore, callID string, expected []string) bool {
func checkLogs(t *testing.T, tnum int, ds models.LogStore, callID string, expected []string) bool {
logReader, err := ds.GetLog(context.Background(), "myapp", callID)
if err != nil {
@@ -460,7 +465,7 @@ func TestFailedEnqueue(t *testing.T) {
err := errors.New("Unable to push task to queue")
mq := &errorMQ{err, http.StatusInternalServerError}
fnl := logs.NewMock()
rnr, cancelrnr := testRunner(t, ds, mq)
rnr, cancelrnr := testRunner(t, ds, mq, fnl)
defer cancelrnr()
srv := testServer(ds, mq, fnl, rnr, ServerTypeFull)
@@ -511,10 +516,10 @@ func TestRouteRunnerTimeout(t *testing.T) {
},
)
rnr, cancelrnr := testRunner(t, ds)
fnl := logs.NewMock()
rnr, cancelrnr := testRunner(t, ds, fnl)
defer cancelrnr()
fnl := logs.NewMock()
srv := testServer(ds, &mqs.Mock{}, fnl, rnr, ServerTypeFull)
for i, test := range []struct {
@@ -581,10 +586,10 @@ func TestRouteRunnerMinimalConcurrentHotSync(t *testing.T) {
},
)
rnr, cancelrnr := testRunner(t, ds)
fnl := logs.NewMock()
rnr, cancelrnr := testRunner(t, ds, fnl)
defer cancelrnr()
fnl := logs.NewMock()
srv := testServer(ds, &mqs.Mock{}, fnl, rnr, ServerTypeFull)
for i, test := range []struct {

26
api/server/server.go
View File

@@ -173,8 +173,10 @@ func NewFromEnv(ctx context.Context, opts ...ServerOption) *Server {
if nodeType != ServerTypeAPI {
opts = append(opts, WithAgentFromEnv())
} else {
// NOTE: ensures logstore is set or there will be troubles
opts = append(opts, WithLogstoreFromDatastore())
}
return New(ctx, opts...)
}
@@ -330,7 +332,7 @@ func WithNodeCertAuthority(ca string) ServerOption {
func WithDatastore(ds models.Datastore) ServerOption {
return func(ctx context.Context, s *Server) error {
s.datastore = datastore.Wrap(ds)
s.datastore = ds
return nil
}
}
@@ -370,7 +372,11 @@ func WithLogstoreFromDatastore() ServerOption {
return errors.New("Need a datastore in order to use it as a logstore")
}
if s.logstore == nil {
s.logstore = s.datastore
if ls, ok := s.datastore.(models.LogStore); ok {
s.logstore = ls
} else {
return errors.New("datastore must implement logstore interface")
}
}
return nil
}
@@ -380,9 +386,12 @@ func WithLogstoreFromDatastore() ServerOption {
func WithFullAgent() ServerOption {
return func(ctx context.Context, s *Server) error {
s.nodeType = ServerTypeFull
// ensure logstore is set (TODO compat only?)
if s.logstore == nil {
s.logstore = s.datastore
WithLogstoreFromDatastore()(ctx, s)
}
if s.datastore == nil || s.logstore == nil || s.mq == nil {
return errors.New("Full nodes must configure FN_DB_URL, FN_LOG_URL, FN_MQ_URL.")
}
@@ -464,14 +473,7 @@ func WithAgentFromEnv() ServerOption {
return errors.New("LBAgent creation failed")
}
default:
s.nodeType = ServerTypeFull
if s.logstore == nil { // TODO seems weird?
s.logstore = s.datastore
}
if s.datastore == nil || s.logstore == nil || s.mq == nil {
return errors.New("Full nodes must configure FN_DB_URL, FN_LOG_URL, FN_MQ_URL.")
}
s.agent = agent.New(agent.NewCachedDataAccess(agent.NewDirectDataAccess(s.datastore, s.logstore, s.mq)))
WithFullAgent()(ctx, s)
}
return nil
}
@@ -543,7 +545,9 @@ func New(ctx context.Context, opts ...ServerOption) *Server {
s.appListeners = new(appListeners)
s.routeListeners = new(routeListeners)
s.datastore = datastore.Wrap(s.datastore)
s.datastore = fnext.NewDatastore(s.datastore, s.appListeners, s.routeListeners)
s.logstore = logs.Wrap(s.logstore)
return s
}

12
api/server/server_test.go
View File

@@ -7,6 +7,7 @@ import (
"io"
"net/http"
"net/http/httptest"
"net/url"
"os"
"strconv"
"strings"
@@ -14,6 +15,7 @@ import (
"github.com/fnproject/fn/api/agent"
"github.com/fnproject/fn/api/datastore"
"github.com/fnproject/fn/api/datastore/sql"
"github.com/fnproject/fn/api/id"
"github.com/fnproject/fn/api/logs"
"github.com/fnproject/fn/api/models"
@@ -94,11 +96,17 @@ func getErrorResponse(t *testing.T, rec *httptest.ResponseRecorder) *models.Erro
func prepareDB(ctx context.Context, t *testing.T) (models.Datastore, models.LogStore, func()) {
os.Remove(tmpDatastoreTests)
ds, err := datastore.New(ctx, "sqlite3://"+tmpDatastoreTests)
uri, err := url.Parse("sqlite3://" + tmpDatastoreTests)
if err != nil {
t.Fatal(err)
}
ss, err := sql.New(ctx, uri)
if err != nil {
t.Fatalf("Error when creating datastore: %s", err)
}
logDB := ds
logDB := logs.Wrap(ss)
ds := datastore.Wrap(ss)
return ds, logDB, func() {
os.Remove(tmpDatastoreTests)
}