alihan/fn-serverless
1
0
Fork 0
mirror of https://github.com/fnproject/fn.git synced 2022-10-28 21:29:17 +03:00
Code Issues Packages Projects Releases Wiki Activity

Updated README and simplified/cleaned up some code.

Browse Source
This commit is contained in:
Travis Reeder
2016-10-12 01:35:44 -07:00
parent e85c7560c3
commit 25f582b180
11 changed files with 182 additions and 84 deletions
Download Patch File Download Diff File Expand all files Collapse all files

58
README.md
View File

@@ -47,9 +47,9 @@ The app `myapp` that we created above along with the `/hello` route we added wou
curl http://localhost:8080/r/myapp/hello curl http://localhost:8080/r/myapp/hello
``` ```
### To pass in data to your function ### Passing data into a function
Your function will get the body of the request as is, and the headers of the request will be passed in as env vars. Try this: Your function will get the body of the HTTP request via STDIN, and the headers of the request will be passed in as env vars. Try this:
```sh ```sh
curl -H "Content-Type: application/json" -X POST -d '{ curl -H "Content-Type: application/json" -X POST -d '{
@@ -57,42 +57,44 @@ curl -H "Content-Type: application/json" -X POST -d '{
}' http://localhost:8080/r/myapp/hello }' http://localhost:8080/r/myapp/hello
``` ```
### Add an asynchronous route You should see it say `Hello Johnny!` now instead of `Hello World!`.
### Adding a route with URL params ### Add an asynchronous function
You can create a route with dynamic URL parameters that will be available inside your function by prefixing path segments with a `:`, for example: IronFunctions supports synchronous function calls like we just tried above, and asynchronous for background processing.
Asynchronous function calls are great for tasks that are CPU heavy or take more than a few seconds to complete.
For instance, image processing, video processing, data processing, ETL, etc.
Architecturally, the main difference between synchronous and asynchronous is that requests
to asynchronous functions are put in a queue and executed on upon resource availability so that they do not interfere with the fast synchronous responses required for an API.
Also, since it uses a message queue, you can queue up millions of function calls without worrying about capacity as requests will
just be queued up and run at some point in the future.
To add an asynchronous function, create another route with the `"type":"async"`, for example:
```sh ```sh
$ curl -H "Content-Type: application/json" -X POST -d '{ curl -H "Content-Type: application/json" -X POST -d '{
"route": { "route": {
"path":"/comments/:author_id/:num_page", "type": "async",
"image":"IMAGE_NAME" "path":"/hello-async",
} "image":"iron/hello"
}
}' http://localhost:8080/v1/apps/myapp/routes }' http://localhost:8080/v1/apps/myapp/routes
``` ```
`:author_id` and `:num_page` in the path will be passed into your function as `PARAM_AUTHOR_ID` and `PARAM_NUM_PAGE`. Now if you request this route, you will just get a `call_id` response:
```json
{"call_id":"572415fd-e26e-542b-846f-f1f5870034f2"}
```
See the [Blog Example](https://github.com/iron-io/functions/blob/master/examples/blog/README.md#creating-our-blog-application-in-your-ironfunctions). If you watch the logs, you will see the function actually runs in the background.
## Writing Functions
## Adding Asynchronous Data Processing Support TODO:
Data processing is for functions that run in the background. This type of functionality is good for functions See examples for now.
that are CPU heavy or take more than a few seconds to complete.
Architecturally, the main difference between synchronous you tried above and asynchronous is that requests
to asynchronous functions are put in a queue and executed on upon resource availablitiy on the same process
or a remote functions process so that they do not interfere with the fast synchronous responses required by an API.
Also, since it uses a queue, you can queue up millions of jobs without worrying about capacity as requests will
just be queued up and run at some point in the future.
TODO: Add link to differences here in README.io docs here.
#### Running remote functions process
Coming soon...
## Using IronFunctions Hosted by Iron.io ## Using IronFunctions Hosted by Iron.io
@@ -112,10 +114,6 @@ myapp.USER_ID.ironfunctions.com/hello
https://swaggerhub.com/api/iron/functions https://swaggerhub.com/api/iron/functions
## Full Documentation
http://docs-new.iron.io/docs
## Join Our Community ## Join Our Community
[![Slack Status](https://open-iron.herokuapp.com/badge.svg)](https://open-iron.herokuapp.com) [![Slack Status](https://open-iron.herokuapp.com/badge.svg)](https://open-iron.herokuapp.com)

6
api/models/task.go
View File

@@ -48,11 +48,13 @@ type Task struct {
*/ */
Error string `json:"error,omitempty"` Error string `json:"error,omitempty"`
/* Route this task belongs to. /* App this task belongs to.
Read Only: true Read Only: true
*/ */
RouteName string `json:"route_name,omitempty"` AppName string `json:"route_name,omitempty"`
Path string `json:"path"`
/* Machine usable reason for task being in this state. /* Machine usable reason for task being in this state.
Valid values for error status are `timeout | killed | bad_exit`. Valid values for error status are `timeout | killed | bad_exit`.

61
api/runner/async_runner.go
View File

@@ -12,12 +12,16 @@ import (
"sync" "sync"
"time" "time"
log "github.com/Sirupsen/logrus" "github.com/Sirupsen/logrus"
"github.com/iron-io/functions/api/models" "github.com/iron-io/functions/api/models"
"github.com/iron-io/runner/common"
"github.com/iron-io/runner/drivers" "github.com/iron-io/runner/drivers"
) )
func getTask(url string) (*models.Task, error) { func getTask(ctx context.Context, url string) (*models.Task, error) {
// log := common.Logger(ctx)
// log.Infoln("Getting task from URL:", url)
resp, err := http.Get(url) resp, err := http.Get(url)
if err != nil { if err != nil {
return nil, err return nil, err
@@ -41,8 +45,8 @@ func getTask(url string) (*models.Task, error) {
} }
func getCfg(task *models.Task) *Config { func getCfg(task *models.Task) *Config {
var stdout bytes.Buffer // TODO: should limit the size of this, error if gets too big. akin to: https://golang.org/pkg/io/#LimitReader // TODO: should limit the size of this, error if gets too big. akin to: https://golang.org/pkg/io/#LimitReader
stderr := NewFuncLogger(task.RouteName, "", *task.Image, task.ID) // TODO: missing path here, how do i get that? stderr := NewFuncLogger(task.AppName, task.Path, *task.Image, task.ID) // TODO: missing path here, how do i get that?
if task.Timeout == nil { if task.Timeout == nil {
timeout := int32(30) timeout := int32(30)
task.Timeout = &timeout task.Timeout = &timeout
@@ -51,8 +55,8 @@ func getCfg(task *models.Task) *Config {
Image: *task.Image, Image: *task.Image,
Timeout: time.Duration(*task.Timeout) * time.Second, Timeout: time.Duration(*task.Timeout) * time.Second,
ID: task.ID, ID: task.ID,
AppName: task.RouteName, AppName: task.AppName,
Stdout: &stdout, Stdout: stderr,
Stderr: stderr, Stderr: stderr,
Env: task.EnvVars, Env: task.EnvVars,
} }
@@ -84,10 +88,9 @@ func deleteTask(url string, task *models.Task) error {
return nil return nil
} }
func runTask(task *models.Task) (drivers.RunResult, error) { func runTask(ctx context.Context, task *models.Task) (drivers.RunResult, error) {
// Set up runner and process task // Set up runner and process task
cfg := getCfg(task) cfg := getCfg(task)
ctx := context.Background()
rnr, err := New(NewMetricLogger()) rnr, err := New(NewMetricLogger())
if err != nil { if err != nil {
return nil, err return nil, err
@@ -96,8 +99,8 @@ func runTask(task *models.Task) (drivers.RunResult, error) {
} }
// RunAsyncRunner pulls tasks off a queue and processes them // RunAsyncRunner pulls tasks off a queue and processes them
func RunAsyncRunner(ctx context.Context, tasksrv, port string, n int) { func RunAsyncRunner(ctx context.Context, tasksrv string, n int) {
u, h := tasksrvURL(tasksrv, port) u, h := tasksrvURL(tasksrv)
if isHostOpen(h) { if isHostOpen(h) {
return return
} }
@@ -105,7 +108,7 @@ func RunAsyncRunner(ctx context.Context, tasksrv, port string, n int) {
var wg sync.WaitGroup var wg sync.WaitGroup
for i := 0; i < n; i++ { for i := 0; i < n; i++ {
wg.Add(1) wg.Add(1)
go startAsyncRunners(ctx, &wg, i, u, runTask) go startAsyncRunners(ctx, &wg, i, u)
} }
wg.Wait() wg.Wait()
@@ -121,7 +124,8 @@ func isHostOpen(host string) bool {
return available return available
} }
func startAsyncRunners(ctx context.Context, wg *sync.WaitGroup, i int, url string, runTask func(task *models.Task) (drivers.RunResult, error)) { func startAsyncRunners(ctx context.Context, wg *sync.WaitGroup, i int, url string) {
ctx, log := common.LoggerWithFields(ctx, logrus.Fields{"async_runner": i})
defer wg.Done() defer wg.Done()
for { for {
select { select {
@@ -129,8 +133,12 @@ func startAsyncRunners(ctx context.Context, wg *sync.WaitGroup, i int, url strin
return return
default: default:
task, err := getTask(url) task, err := getTask(ctx, url)
if err != nil { if err != nil {
if err, ok := err.(net.Error); ok && err.Timeout() {
log.Infoln("Could not fetch task, timeout. Probably no tasks to run.")
continue
}
log.WithError(err).Error("Could not fetch task") log.WithError(err).Error("Could not fetch task")
time.Sleep(1 * time.Second) time.Sleep(1 * time.Second)
continue continue
@@ -139,33 +147,36 @@ func startAsyncRunners(ctx context.Context, wg *sync.WaitGroup, i int, url strin
time.Sleep(1 * time.Second) time.Sleep(1 * time.Second)
continue continue
} }
log.Debug("Picked up task:", task.ID)
ctx, log := common.LoggerWithFields(ctx, logrus.Fields{"call_id": task.ID})
log.Debug("Running task:", task.ID) log.Debug("Running task:", task.ID)
// Process Task // Process Task
if _, err := runTask(task); err != nil { if _, err := runTask(ctx, task); err != nil {
log.WithError(err).WithFields(log.Fields{"async runner": i, "task_id": task.ID}).Error("Cannot run task") log.WithError(err).Error("Cannot run task")
continue continue
} }
log.Debug("Processed task:", task.ID) log.Debug("Processed task")
// Delete task from queue // Delete task from queue
if err := deleteTask(url, task); err != nil { if err := deleteTask(url, task); err != nil {
log.WithError(err).WithFields(log.Fields{"async runner": i, "task_id": task.ID}).Error("Cannot delete task") log.WithError(err).Error("Cannot delete task")
continue continue
} }
log.Debug("Deleted task:", task.ID)
log.Info("Task complete:", task.ID) log.Info("Task complete:")
} }
} }
} }
func tasksrvURL(tasksrv, port string) (parsedURL, host string) { func tasksrvURL(tasksrv string) (parsedURL, host string) {
parsed, err := url.Parse(tasksrv) parsed, err := url.Parse(tasksrv)
if err != nil { if err != nil {
log.Fatalf("cannot parse TASKSRV endpoint: %v", err) logrus.WithError(err).Fatalln("cannot parse TASKSRV endpoint")
} }
// host, port, err := net.SplitHostPort(parsed.Host)
// if err != nil {
// log.WithError(err).Fatalln("net.SplitHostPort")
// }
if parsed.Scheme == "" { if parsed.Scheme == "" {
parsed.Scheme = "http" parsed.Scheme = "http"
@@ -175,9 +186,9 @@ func tasksrvURL(tasksrv, port string) (parsedURL, host string) {
parsed.Path = "/tasks" parsed.Path = "/tasks"
} }
if _, _, err := net.SplitHostPort(parsed.Host); err != nil { // if _, _, err := net.SplitHostPort(parsed.Host); err != nil {
parsed.Host = net.JoinHostPort(parsed.Host, port) // parsed.Host = net.JoinHostPort(parsed.Host, parsed)
} // }
return parsed.String(), parsed.Host return parsed.String(), parsed.Host
} }

2
api/runner/async_runner_test.go
View File

@@ -37,7 +37,7 @@ func getMockTask() models.Task {
task := &models.Task{} task := &models.Task{}
task.Image = &image task.Image = &image
task.ID = fmt.Sprintf("ID-%d", rand.Int31()%1000) task.ID = fmt.Sprintf("ID-%d", rand.Int31()%1000)
task.RouteName = fmt.Sprintf("RouteName-%d", rand.Int31()%1000) task.AppName = fmt.Sprintf("RouteName-%d", rand.Int31()%1000)
task.Priority = &priority task.Priority = &priority
return *task return *task
} }

2
api/runner/logger.go
View File

@@ -19,7 +19,7 @@ func NewFuncLogger(appName, path, function, requestID string) io.Writer {
r: r, r: r,
w: w, w: w,
} }
log := logrus.WithFields(logrus.Fields{"user_log": true, "app_name": appName, "path": path, "function": function, "request_id": requestID}) log := logrus.WithFields(logrus.Fields{"user_log": true, "app_name": appName, "path": path, "function": function, "call_id": requestID})
go func(reader io.Reader) { go func(reader io.Reader) {
scanner := bufio.NewScanner(reader) scanner := bufio.NewScanner(reader)
for scanner.Scan() { for scanner.Scan() {

19
api/server/apps_test.go
View File

@@ -27,7 +27,8 @@ func setLogBuffer() *bytes.Buffer {
func TestAppCreate(t *testing.T) { func TestAppCreate(t *testing.T) {
buf := setLogBuffer() buf := setLogBuffer()
New(&datastore.Mock{}, &mqs.Mock{}, testRunner(t)) New(&datastore.Mock{}, &mqs.Mock{}, testRunner(t))
router := testRouter() s := New(&datastore.Mock{}, &mqs.Mock{}, testRunner(t))
router := testRouter(s)
for i, test := range []struct { for i, test := range []struct {
path string path string
@@ -70,8 +71,8 @@ func TestAppCreate(t *testing.T) {
func TestAppDelete(t *testing.T) { func TestAppDelete(t *testing.T) {
buf := setLogBuffer() buf := setLogBuffer()
New(&datastore.Mock{}, &mqs.Mock{}, testRunner(t)) s := New(&datastore.Mock{}, &mqs.Mock{}, testRunner(t))
router := testRouter() router := testRouter(s)
for i, test := range []struct { for i, test := range []struct {
path string path string
@@ -104,8 +105,8 @@ func TestAppDelete(t *testing.T) {
func TestAppList(t *testing.T) { func TestAppList(t *testing.T) {
buf := setLogBuffer() buf := setLogBuffer()
New(&datastore.Mock{}, &mqs.Mock{}, testRunner(t)) s := New(&datastore.Mock{}, &mqs.Mock{}, testRunner(t))
router := testRouter() router := testRouter(s)
for i, test := range []struct { for i, test := range []struct {
path string path string
@@ -137,8 +138,8 @@ func TestAppList(t *testing.T) {
func TestAppGet(t *testing.T) { func TestAppGet(t *testing.T) {
buf := setLogBuffer() buf := setLogBuffer()
New(&datastore.Mock{}, &mqs.Mock{}, testRunner(t)) s := New(&datastore.Mock{}, &mqs.Mock{}, testRunner(t))
router := testRouter() router := testRouter(s)
for i, test := range []struct { for i, test := range []struct {
path string path string
@@ -170,8 +171,8 @@ func TestAppGet(t *testing.T) {
func TestAppUpdate(t *testing.T) { func TestAppUpdate(t *testing.T) {
buf := setLogBuffer() buf := setLogBuffer()
New(&datastore.Mock{}, &mqs.Mock{}, testRunner(t)) s := New(&datastore.Mock{}, &mqs.Mock{}, testRunner(t))
router := testRouter() router := testRouter(s)
for i, test := range []struct { for i, test := range []struct {
path string path string

90
api/server/helpers.go Normal file
View File

@@ -0,0 +1,90 @@
package server
// TODO: this whole file shouldn't be in a non test file
import (
"context"
"encoding/json"
"io"
"io/ioutil"
"net/http"
"net/http/httptest"
"testing"
"github.com/gin-gonic/gin"
"github.com/iron-io/functions/api/models"
"github.com/iron-io/functions/api/runner"
"github.com/iron-io/runner/common"
)
type appResponse struct {
Message string `json:"message"`
App *models.App `json:"app"`
}
type appsResponse struct {
Message string `json:"message"`
Apps models.Apps `json:"apps"`
}
type routeResponse struct {
Message string `json:"message"`
Route *models.Route `json:"route"`
}
type routesResponse struct {
Message string `json:"message"`
Routes models.Routes `json:"routes"`
}
type tasksResponse struct {
Message string `json:"message"`
Task models.Task `json:"tasksResponse"`
}
func testRouter(s *Server) *gin.Engine {
r := gin.Default()
ctx := context.Background()
r.Use(func(c *gin.Context) {
ctx, _ := common.LoggerWithFields(ctx, extractFields(c))
c.Set("ctx", ctx)
c.Next()
})
s.bindHandlers()
return r
}
func testRunner(t *testing.T) *runner.Runner {
r, err := runner.New(runner.NewMetricLogger())
if err != nil {
t.Fatal("Test: failed to create new runner")
}
return r
}
func routerRequest(t *testing.T, router *gin.Engine, method, path string, body io.Reader) (*http.Request, *httptest.ResponseRecorder) {
req, err := http.NewRequest(method, "http://localhost:8080"+path, body)
if err != nil {
t.Fatalf("Test: Could not create %s request to %s: %v", method, path, err)
}
rec := httptest.NewRecorder()
router.ServeHTTP(rec, req)
return req, rec
}
func getErrorResponse(t *testing.T, rec *httptest.ResponseRecorder) models.Error {
respBody, err := ioutil.ReadAll(rec.Body)
if err != nil {
t.Error("Test: Expected not empty response body")
}
var errResp models.Error
err = json.Unmarshal(respBody, &errResp)
if err != nil {
t.Error("Test: Expected response body to be a valid models.Error object")
}
return errResp
}

6
api/server/runner.go
View File

@@ -42,12 +42,11 @@ func handleRequest(c *gin.Context, enqueue models.Enqueue) {
} }
ctx := c.MustGet("ctx").(context.Context) ctx := c.MustGet("ctx").(context.Context)
log := common.Logger(ctx)
reqID := uuid.NewV5(uuid.Nil, fmt.Sprintf("%s%s%d", c.Request.RemoteAddr, c.Request.URL.Path, time.Now().Unix())).String() reqID := uuid.NewV5(uuid.Nil, fmt.Sprintf("%s%s%d", c.Request.RemoteAddr, c.Request.URL.Path, time.Now().Unix())).String()
c.Set("reqID", reqID) // todo: put this in the ctx instead of gin's c.Set("reqID", reqID) // todo: put this in the ctx instead of gin's
ctx, log = common.LoggerWithFields(ctx, logrus.Fields{"call_id": reqID}) ctx, log := common.LoggerWithFields(ctx, logrus.Fields{"call_id": reqID})
var err error var err error
var payload io.Reader var payload io.Reader
@@ -164,7 +163,8 @@ func handleRequest(c *gin.Context, enqueue models.Enqueue) {
task := &models.Task{} task := &models.Task{}
task.Image = &cfg.Image task.Image = &cfg.Image
task.ID = cfg.ID task.ID = cfg.ID
task.RouteName = cfg.AppName task.Path = el.Path
task.AppName = cfg.AppName
task.Priority = &priority task.Priority = &priority
task.EnvVars = cfg.Env task.EnvVars = cfg.Env
task.Payload = string(pl) task.Payload = string(pl)

14
api/server/server.go
View File

@@ -131,7 +131,6 @@ func extractFields(c *gin.Context) logrus.Fields {
for _, param := range c.Params { for _, param := range c.Params {
fields[param.Key] = param.Value fields[param.Key] = param.Value
} }
return fields return fields
} }
@@ -142,7 +141,7 @@ func (s *Server) Run(ctx context.Context) {
c.Next() c.Next()
}) })
bindHandlers(s.Router, s.handleRunnerRequest, s.handleTaskRequest) s.bindHandlers()
// By default it serves on :8080 unless a // By default it serves on :8080 unless a
// PORT environment variable was defined. // PORT environment variable was defined.
@@ -150,8 +149,9 @@ func (s *Server) Run(ctx context.Context) {
<-ctx.Done() <-ctx.Done()
} }
func bindHandlers(engine *gin.Engine, reqHandler func(ginC *gin.Context), taskHandler func(ginC *gin.Context)) { func (s *Server) bindHandlers() {
engine.Use(gin.Logger())
engine := s.Router
engine.GET("/", handlePing) engine.GET("/", handlePing)
engine.GET("/version", handleVersion) engine.GET("/version", handleVersion)
@@ -177,9 +177,9 @@ func bindHandlers(engine *gin.Engine, reqHandler func(ginC *gin.Context), taskHa
} }
} }
engine.DELETE("/tasks", taskHandler) engine.DELETE("/tasks", s.handleTaskRequest)
engine.GET("/tasks", taskHandler) engine.GET("/tasks", s.handleTaskRequest)
engine.Any("/r/:app/*route", reqHandler) engine.Any("/r/:app/*route", s.handleRunnerRequest)
// This final route is used for extensions, see Server.Add // This final route is used for extensions, see Server.Add
engine.NoRoute(handleSpecial) engine.NoRoute(handleSpecial)

4
api/swagger.yml
View File

@@ -263,10 +263,6 @@ definitions:
allOf: allOf:
- type: object - type: object
properties: properties:
name:
type: string
description: "Route name"
readOnly: true
app_name: app_name:
type: string type: string
description: "App this route belongs to." description: "App this route belongs to."

4
main.go
View File

@@ -90,11 +90,11 @@ func main() {
srv.Run(ctx) srv.Run(ctx)
}) })
apiURL, port, numAsync := viper.GetString(envAPIURL), viper.GetString(envPort), viper.GetInt(envNumAsync) apiURL, numAsync := viper.GetString(envAPIURL), viper.GetInt(envNumAsync)
log.Debug("async workers:", numAsync) log.Debug("async workers:", numAsync)
if numAsync > 0 { if numAsync > 0 {
svr.AddFunc(func(ctx context.Context) { svr.AddFunc(func(ctx context.Context) {
runner.RunAsyncRunner(ctx, apiURL, port, numAsync) runner.RunAsyncRunner(ctx, apiURL, numAsync)
}) })
} }