alihan/fn-serverless
1
0
Fork 0
mirror of https://github.com/fnproject/fn.git synced 2022-10-28 21:29:17 +03:00
Code Issues Packages Projects Releases Wiki Activity

HTTP Triggers hookup (#1086)

Browse Source 
* Initial suypport for invoking tiggers

* dupe method

* tighten server constraints

* runner tests not working yet

* basic route tests passing

* post rebase fixes

* add hybrid support for trigger invoke and tests

* consoloidate all hybrid evil into one place

* cleanup and make triggers unique by source

* fix oops with Agent

* linting

* review fixes
This commit is contained in:
Owen Cliffe
2018-07-05 18:56:07 +01:00
committed by Reed Allman
parent b07a000a18
commit b8b544ed25
38 changed files with 2208 additions and 865 deletions
Download Patch File Download Diff File Expand all files Collapse all files

141
api/agent/data_access.go
View File

@@ -11,26 +11,34 @@ import (
"github.com/patrickmn/go-cache"
)
// DataAccess abstracts the datastore and message queue operations done by the
// agent, so that API nodes and runner nodes can work with the same interface
// but actually operate on the data in different ways (by direct access or by
// mediation through an API node).
type DataAccess interface {
//ReadDataAccess represents read operations required to operate a load balancer node
type ReadDataAccess interface {
GetAppID(ctx context.Context, appName string) (string, error)
// GetAppByID abstracts querying the datastore for an app.
GetAppByID(ctx context.Context, appID string) (*models.App, error)
GetTriggerBySource(ctx context.Context, appId string, triggerType, source string) (*models.Trigger, error)
GetFnByID(ctx context.Context, fnId string) (*models.Fn, error)
// GetRoute abstracts querying the datastore for a route within an app.
GetRoute(ctx context.Context, appID string, routePath string) (*models.Route, error)
}
// Enqueue will add a Call to the queue (ultimately forwards to mq.Push).
Enqueue(ctx context.Context, mCall *models.Call) error
//DequeueDataAccess abstracts an underlying dequeue for async runners
type DequeueDataAccess interface {
// Dequeue will query the queue for the next available Call that can be run
// by this Agent, and reserve it (ultimately forwards to mq.Reserve).
Dequeue(ctx context.Context) (*models.Call, error)
}
//EnqueueDataAccess abstracts an underying enqueue for async queueing
type EnqueueDataAccess interface {
// Enqueue will add a Call to the queue (ultimately forwards to mq.Push).
Enqueue(ctx context.Context, mCall *models.Call) error
}
// CallHandler consumes the start and finish events for a call
// This is effectively a callback that is allowed to read the logs -
// TODO Deprecate this - this could be a CallListener except it also consumes logs
type CallHandler interface {
// Start will attempt to start the provided Call within an appropriate
// context.
Start(ctx context.Context, mCall *models.Call) error
@@ -38,25 +46,27 @@ type DataAccess interface {
// Finish will notify the system that the Call has been processed, and
// fulfill the reservation in the queue if the call came from a queue.
Finish(ctx context.Context, mCall *models.Call, stderr io.Reader, async bool) error
}
// Close will wait for any pending operations to complete and
// shuts down connections to the underlying datastore/queue resources.
// Close is not safe to be called from multiple threads.
io.Closer
// DataAccess is currently
type DataAccess interface {
ReadDataAccess
DequeueDataAccess
CallHandler
}
// CachedDataAccess wraps a DataAccess and caches the results of GetApp and GetRoute.
type CachedDataAccess struct {
DataAccess
type cachedDataAccess struct {
ReadDataAccess
cache *cache.Cache
singleflight singleflight.SingleFlight
}
func NewCachedDataAccess(da DataAccess) DataAccess {
cda := &CachedDataAccess{
DataAccess: da,
cache: cache.New(5*time.Second, 1*time.Minute),
func NewCachedDataAccess(da ReadDataAccess) ReadDataAccess {
cda := &cachedDataAccess{
ReadDataAccess: da,
cache: cache.New(5*time.Second, 1*time.Minute),
}
return cda
}
@@ -69,11 +79,11 @@ func appIDCacheKey(appID string) string {
return "a:" + appID
}
func (da *CachedDataAccess) GetAppID(ctx context.Context, appName string) (string, error) {
return da.DataAccess.GetAppID(ctx, appName)
func (da *cachedDataAccess) GetAppID(ctx context.Context, appName string) (string, error) {
return da.ReadDataAccess.GetAppID(ctx, appName)
}
func (da *CachedDataAccess) GetAppByID(ctx context.Context, appID string) (*models.App, error) {
func (da *cachedDataAccess) GetAppByID(ctx context.Context, appID string) (*models.App, error) {
key := appIDCacheKey(appID)
app, ok := da.cache.Get(key)
if ok {
@@ -82,7 +92,7 @@ func (da *CachedDataAccess) GetAppByID(ctx context.Context, appID string) (*mode
resp, err := da.singleflight.Do(key,
func() (interface{}, error) {
return da.DataAccess.GetAppByID(ctx, appID)
return da.ReadDataAccess.GetAppByID(ctx, appID)
})
if err != nil {
@@ -93,7 +103,7 @@ func (da *CachedDataAccess) GetAppByID(ctx context.Context, appID string) (*mode
return app.(*models.App), nil
}
func (da *CachedDataAccess) GetRoute(ctx context.Context, appID string, routePath string) (*models.Route, error) {
func (da *cachedDataAccess) GetRoute(ctx context.Context, appID string, routePath string) (*models.Route, error) {
key := routeCacheKey(appID, routePath)
r, ok := da.cache.Get(key)
if ok {
@@ -102,7 +112,7 @@ func (da *CachedDataAccess) GetRoute(ctx context.Context, appID string, routePat
resp, err := da.singleflight.Do(key,
func() (interface{}, error) {
return da.DataAccess.GetRoute(ctx, appID, routePath)
return da.ReadDataAccess.GetRoute(ctx, appID, routePath)
})
if err != nil {
@@ -113,48 +123,55 @@ func (da *CachedDataAccess) GetRoute(ctx context.Context, appID string, routePat
return r.(*models.Route), nil
}
// Close invokes close on the underlying DataAccess
func (da *CachedDataAccess) Close() error {
return da.DataAccess.Close()
}
type directDataAccess struct {
mq models.MessageQueue
ds models.Datastore
ls models.LogStore
}
func NewDirectDataAccess(ds models.Datastore, ls models.LogStore, mq models.MessageQueue) DataAccess {
type directDequeue struct {
mq models.MessageQueue
}
func (ddq *directDequeue) Dequeue(ctx context.Context) (*models.Call, error) {
return ddq.mq.Reserve(ctx)
}
func NewDirectDequeueAccess(mq models.MessageQueue) DequeueDataAccess {
return &directDequeue{
mq: mq,
}
}
type directEnequeue struct {
mq models.MessageQueue
}
func NewDirectEnqueueAccess(mq models.MessageQueue) EnqueueDataAccess {
return &directEnequeue{
mq: mq,
}
}
func (da *directEnequeue) Enqueue(ctx context.Context, mCall *models.Call) error {
_, err := da.mq.Push(ctx, mCall)
return err
// TODO: Insert a call in the datastore with the 'queued' state
}
func NewDirectCallDataAccess(ls models.LogStore, mq models.MessageQueue) CallHandler {
da := &directDataAccess{
mq: mq,
ds: ds,
ls: ls,
}
return da
}
func (da *directDataAccess) GetAppID(ctx context.Context, appName string) (string, error) {
return da.ds.GetAppID(ctx, appName)
}
func (da *directDataAccess) GetAppByID(ctx context.Context, appID string) (*models.App, error) {
return da.ds.GetAppByID(ctx, appID)
}
func (da *directDataAccess) GetRoute(ctx context.Context, appID string, routePath string) (*models.Route, error) {
return da.ds.GetRoute(ctx, appID, routePath)
}
func (da *directDataAccess) Enqueue(ctx context.Context, mCall *models.Call) error {
_, err := da.mq.Push(ctx, mCall)
return err
// TODO: Insert a call in the datastore with the 'queued' state
}
func (da *directDataAccess) Dequeue(ctx context.Context) (*models.Call, error) {
return da.mq.Reserve(ctx)
}
func (da *directDataAccess) Start(ctx context.Context, mCall *models.Call) error {
// TODO Access datastore and try a Compare-And-Swap to set the call to
// 'running'. If it fails, delete the message from the MQ and return an
@@ -183,22 +200,18 @@ func (da *directDataAccess) Finish(ctx context.Context, mCall *models.Call, stde
if async {
// XXX (reed): delete MQ message, eventually
// YYY (hhexo): yes, once we have the queued/running/finished mechanics
// return da.mq.Delete(ctx, mCall)
// return cda.mq.Delete(ctx, mCall)
}
return nil
}
// Close calls close on the underlying Datastore and MessageQueue. If the Logstore
// and Datastore are different, it will call Close on the Logstore as well.
func (da *directDataAccess) Close() error {
err := da.ds.Close()
if ls, ok := da.ds.(models.LogStore); ok && ls != da.ls {
if daErr := da.ls.Close(); daErr != nil {
err = daErr
}
}
if mqErr := da.mq.Close(); mqErr != nil {
err = mqErr
}
return err
type noAsyncEnqueueAccess struct{}
func (noAsyncEnqueueAccess) Enqueue(ctx context.Context, mCall *models.Call) error {
return models.ErrAsyncUnsupported
}
//NewUnsupportedEnqueueAccess is a backstop that errors when you try to enqueue an async operation on a server that doesn't support async
func NewUnsupportedAsyncEnqueueAccess() EnqueueDataAccess {
return &noAsyncEnqueueAccess{}
}