mirror of
https://github.com/fnproject/fn.git
synced 2022-10-28 21:29:17 +03:00
merge datastores into sqlx package
replace default bolt option with sqlite3 option. the story here is that we just need a working out of the box solution, and sqlite3 is just fine for that (actually, likely better than bolt). with sqlite3 supplanting bolt, we mostly have sql databases. so remove redis and then we just have one package that has a `sql` implementation of the `models.Datastore` and lean on sqlx to do query rewriting. this does mean queries have to be formed a certain way and likely have to be ANSI-SQL (no special features) but we weren't using them anyway and our base api is basically done and we can easily extend this api as needed to only implement certain methods in certain backends if we need to get cute. * remove bolt & redis datastores (can still use as mqs) * make sql queries work on all 3 (maybe?) * remove bolt log store and use sqlite3 * shove the FnLog shit into the datastore shit for now (free pg/mysql logs... just for demos, etc, not prod) * fix up the docs to remove bolt references * add sqlite3, sqlx dep * fix up tests & mock stuff, make validator less insane * remove put & get in datastore layer as nobody is using. this passes tests which at least seem like they test all the different backends. if we trust our tests then this seems to work great. (tests `make docker-test-run-with-*` work now too)
This commit is contained in:
Reed Allman
126
api/logs/bolt.go
126
api/logs/bolt.go
@@ -1,126 +0,0 @@
|
||||
package logs
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"context"
|
||||
|
||||
"github.com/Sirupsen/logrus"
|
||||
"github.com/boltdb/bolt"
|
||||
"gitlab-odx.oracle.com/odx/functions/api/models"
|
||||
)
|
||||
|
||||
type BoltLogDatastore struct {
|
||||
callLogsBucket []byte
|
||||
db *bolt.DB
|
||||
log logrus.FieldLogger
|
||||
datastore models.Datastore
|
||||
}
|
||||
|
||||
func NewBolt(url *url.URL) (models.FnLog, error) {
|
||||
dir := filepath.Dir(url.Path)
|
||||
log := logrus.WithFields(logrus.Fields{"logdb": url.Scheme, "dir": dir})
|
||||
err := os.MkdirAll(dir, 0755)
|
||||
if err != nil {
|
||||
log.WithError(err).Errorln("Could not create data directory for log.db")
|
||||
return nil, err
|
||||
}
|
||||
log.WithFields(logrus.Fields{"path": url.Path}).Debug("Creating bolt log.db")
|
||||
db, err := bolt.Open(url.Path, 0655, &bolt.Options{Timeout: 1 * time.Second})
|
||||
if err != nil {
|
||||
log.WithError(err).Errorln("Error on bolt.Open")
|
||||
return nil, err
|
||||
}
|
||||
// I don't think we need a prefix here do we? Made it blank. If we do, we should call the query param "prefix" instead of bucket.
|
||||
bucketPrefix := ""
|
||||
if url.Query()["bucket"] != nil {
|
||||
bucketPrefix = url.Query()["bucket"][0]
|
||||
}
|
||||
callLogsBucketName := []byte(bucketPrefix + "call_logs")
|
||||
err = db.Update(func(tx *bolt.Tx) error {
|
||||
for _, name := range [][]byte{callLogsBucketName} {
|
||||
_, err := tx.CreateBucketIfNotExists(name)
|
||||
if err != nil {
|
||||
log.WithError(err).WithFields(logrus.Fields{"name": name}).Error("create bucket")
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
log.WithError(err).Errorln("Error creating bolt buckets")
|
||||
return nil, err
|
||||
}
|
||||
|
||||
fnl := &BoltLogDatastore{
|
||||
callLogsBucket: callLogsBucketName,
|
||||
db: db,
|
||||
log: log,
|
||||
}
|
||||
log.WithFields(logrus.Fields{"prefix": bucketPrefix, "file": url.Path}).Debug("BoltDB initialized")
|
||||
|
||||
return NewValidator(fnl), nil
|
||||
}
|
||||
|
||||
func (fnl *BoltLogDatastore) InsertLog(ctx context.Context, callID string, callLog string) error {
|
||||
log := &models.FnCallLog{
|
||||
CallID: callID,
|
||||
Log: callLog,
|
||||
}
|
||||
id := []byte(callID)
|
||||
err := fnl.db.Update(
|
||||
func(tx *bolt.Tx) error {
|
||||
bIm := tx.Bucket(fnl.callLogsBucket)
|
||||
buf, err := json.Marshal(log)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = bIm.Put(id, buf)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
func (fnl *BoltLogDatastore) GetLog(ctx context.Context, callID string) (*models.FnCallLog, error) {
|
||||
var res *models.FnCallLog
|
||||
err := fnl.db.View(func(tx *bolt.Tx) error {
|
||||
b := tx.Bucket(fnl.callLogsBucket)
|
||||
v := b.Get([]byte(callID))
|
||||
if v != nil {
|
||||
fnCall := &models.FnCallLog{}
|
||||
err := json.Unmarshal(v, fnCall)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
res = fnCall
|
||||
} else {
|
||||
return models.ErrCallLogNotFound
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return res, err
|
||||
}
|
||||
|
||||
func (fnl *BoltLogDatastore) DeleteLog(ctx context.Context, callID string) error {
|
||||
_, err := fnl.GetLog(ctx, callID)
|
||||
//means object does not exist
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
id := []byte(callID)
|
||||
err = fnl.db.Update(func(tx *bolt.Tx) error {
|
||||
bIm := tx.Bucket(fnl.callLogsBucket)
|
||||
err := bIm.Delete(id)
|
||||
return err
|
||||
})
|
||||
return err
|
||||
}
|
||||
@@ -1,33 +0,0 @@
|
||||
package logs
|
||||
|
||||
import (
|
||||
"net/url"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"gitlab-odx.oracle.com/odx/functions/api/datastore/bolt"
|
||||
logTesting "gitlab-odx.oracle.com/odx/functions/api/logs/testing"
|
||||
)
|
||||
|
||||
const tmpLogDb = "/tmp/func_test_log.db"
|
||||
const tmpDatastore = "/tmp/func_test_datastore.db"
|
||||
|
||||
func TestDatastore(t *testing.T) {
|
||||
os.Remove(tmpLogDb)
|
||||
os.Remove(tmpDatastore)
|
||||
uLog, err := url.Parse("bolt://" + tmpLogDb)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to parse url: %v", err)
|
||||
}
|
||||
uDatastore, err := url.Parse("bolt://" + tmpDatastore)
|
||||
|
||||
fnl, err := NewBolt(uLog)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create bolt log datastore: %v", err)
|
||||
}
|
||||
ds, err := bolt.New(uDatastore)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create bolt datastore: %v", err)
|
||||
}
|
||||
logTesting.Test(t, fnl, ds)
|
||||
}
|
||||
@@ -2,9 +2,11 @@ package logs
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/Sirupsen/logrus"
|
||||
"gitlab-odx.oracle.com/odx/functions/api/models"
|
||||
"net/url"
|
||||
|
||||
"github.com/Sirupsen/logrus"
|
||||
"gitlab-odx.oracle.com/odx/functions/api/datastore/sql"
|
||||
"gitlab-odx.oracle.com/odx/functions/api/models"
|
||||
)
|
||||
|
||||
func New(dbURL string) (models.FnLog, error) {
|
||||
@@ -12,10 +14,10 @@ func New(dbURL string) (models.FnLog, error) {
|
||||
if err != nil {
|
||||
logrus.WithError(err).WithFields(logrus.Fields{"url": dbURL}).Fatal("bad DB URL")
|
||||
}
|
||||
logrus.WithFields(logrus.Fields{"db": u.Scheme}).Debug("creating new datastore")
|
||||
logrus.WithFields(logrus.Fields{"db": u.Scheme}).Debug("creating log store")
|
||||
switch u.Scheme {
|
||||
case "bolt":
|
||||
return NewBolt(u)
|
||||
case "sqlite3", "postgres", "mysql":
|
||||
return sql.New(u)
|
||||
default:
|
||||
return nil, fmt.Errorf("db type not supported %v", u.Scheme)
|
||||
}
|
||||
|
||||
26
api/logs/log_test.go
Normal file
26
api/logs/log_test.go
Normal file
@@ -0,0 +1,26 @@
|
||||
package logs
|
||||
|
||||
import (
|
||||
"net/url"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"gitlab-odx.oracle.com/odx/functions/api/datastore/sql"
|
||||
logTesting "gitlab-odx.oracle.com/odx/functions/api/logs/testing"
|
||||
)
|
||||
|
||||
const tmpLogDb = "/tmp/func_test_log.db"
|
||||
|
||||
func TestDatastore(t *testing.T) {
|
||||
os.Remove(tmpLogDb)
|
||||
uLog, err := url.Parse("sqlite3://" + tmpLogDb)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to parse url: %v", err)
|
||||
}
|
||||
|
||||
ds, err := sql.New(uLog)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create sqlite3 datastore: %v", err)
|
||||
}
|
||||
logTesting.Test(t, ds, ds)
|
||||
}
|
||||
Reference in New Issue
Block a user