From 69f0201818b46adc372809b436c6bc0580a03d4c Mon Sep 17 00:00:00 2001 From: Travis Reeder Date: Fri, 26 May 2017 18:54:26 +0000 Subject: [PATCH] Some small cleanup to docs. --- api/runner/async_runner.go | 23 +++++++++--------- api/runner/runner.go | 6 ++++- api/server/runner.go | 43 ++++++++++++++++++++-------------- docs/README.md | 7 ++---- docs/function-file.md | 5 ++++ docs/lambda/README.md | 2 ++ docs/lambda/about.md | 48 -------------------------------------- docs/lambda/aws.md | 2 ++ docs/lambda/create.md | 2 ++ docs/lambda/import.md | 2 ++ docs/lambda/test.md | 2 ++ docs/packaging.md | 32 +++++++++---------------- docs/writing.md | 25 ++++++++++++++++---- 13 files changed, 92 insertions(+), 107 deletions(-) delete mode 100644 docs/lambda/about.md diff --git a/api/runner/async_runner.go b/api/runner/async_runner.go index 5abeb13d4..b19ec4d6a 100644 --- a/api/runner/async_runner.go +++ b/api/runner/async_runner.go @@ -14,8 +14,8 @@ import ( "github.com/Sirupsen/logrus" "gitlab.oracledx.com/odx/functions/api/models" - "gitlab.oracledx.com/odx/functions/api/runner/task" "gitlab.oracledx.com/odx/functions/api/runner/common" + "gitlab.oracledx.com/odx/functions/api/runner/task" ) func getTask(ctx context.Context, url string) (*models.Task, error) { @@ -42,22 +42,23 @@ func getTask(ctx context.Context, url string) (*models.Task, error) { } func getCfg(t *models.Task) *task.Config { - timeout := int32(30) - if t.Timeout == nil { - t.Timeout = &timeout - } - if t.IdleTimeout == nil { - t.IdleTimeout = &timeout - } - cfg := &task.Config{ Image: *t.Image, - Timeout: time.Duration(*t.Timeout) * time.Second, - IdleTimeout: time.Duration(*t.IdleTimeout) * time.Second, ID: t.ID, AppName: t.AppName, Env: t.EnvVars, } + if t.Timeout == nil || *t.Timeout <= 0 { + cfg.Timeout = DefaultTimeout + } else { + cfg.Timeout = time.Duration(*t.Timeout) * time.Second + } + if t.IdleTimeout == nil || *t.IdleTimeout <= 0 { + cfg.IdleTimeout = DefaultIdleTimeout + } else { + cfg.IdleTimeout = time.Duration(*t.IdleTimeout) * time.Second + } + return cfg } diff --git a/api/runner/runner.go b/api/runner/runner.go index b2b1b8bcb..0ddde9f28 100644 --- a/api/runner/runner.go +++ b/api/runner/runner.go @@ -37,8 +37,12 @@ type Runner struct { var ( ErrTimeOutNoMemory = errors.New("Task timed out. No available memory.") ErrFullQueue = errors.New("The runner queue is full") + WaitMemoryTimeout = 10 * time.Second +) - WaitMemoryTimeout = 10 * time.Second +const ( + DefaultTimeout = 30 * time.Second + DefaultIdleTimeout = 30 * time.Second ) func New(ctx context.Context, flog FuncLogger, mlog MetricLogger) (*Runner, error) { diff --git a/api/server/runner.go b/api/server/runner.go index 95ecd08f9..9d08a7387 100644 --- a/api/server/runner.go +++ b/api/server/runner.go @@ -14,12 +14,12 @@ import ( "github.com/Sirupsen/logrus" "github.com/gin-gonic/gin" + uuid "github.com/satori/go.uuid" "gitlab.oracledx.com/odx/functions/api" "gitlab.oracledx.com/odx/functions/api/models" "gitlab.oracledx.com/odx/functions/api/runner" - "gitlab.oracledx.com/odx/functions/api/runner/task" "gitlab.oracledx.com/odx/functions/api/runner/common" - uuid "github.com/satori/go.uuid" + "gitlab.oracledx.com/odx/functions/api/runner/task" ) type runnerResponse struct { @@ -162,13 +162,14 @@ func (s *Server) serve(ctx context.Context, c *gin.Context, appName string, foun var stdout bytes.Buffer // TODO: should limit the size of this, error if gets too big. akin to: https://golang.org/pkg/io/#LimitReader envVars := map[string]string{ - "METHOD": c.Request.Method, - "ROUTE": found.Path, + "METHOD": c.Request.Method, + "ROUTE": found.Path, "REQUEST_URL": fmt.Sprintf("%v//%v%v", func() string { if c.Request.TLS == nil { return "http" } - return "https"}(), c.Request.Host, c.Request.URL.String()), + return "https" + }(), c.Request.Host, c.Request.URL.String()), } // app config @@ -190,18 +191,26 @@ func (s *Server) serve(ctx context.Context, c *gin.Context, appName string, foun } cfg := &task.Config{ - AppName: appName, - Path: found.Path, - Env: envVars, - Format: found.Format, - ID: reqID, - Image: found.Image, - MaxConcurrency: found.MaxConcurrency, - Memory: found.Memory, - Stdin: payload, - Stdout: &stdout, - Timeout: time.Duration(found.Timeout) * time.Second, - IdleTimeout: time.Duration(found.IdleTimeout) * time.Second, + AppName: appName, + Path: found.Path, + Env: envVars, + Format: found.Format, + ID: reqID, + Image: found.Image, + MaxConcurrency: found.MaxConcurrency, + Memory: found.Memory, + Stdin: payload, + Stdout: &stdout, + Timeout: time.Duration(found.Timeout) * time.Second, + IdleTimeout: time.Duration(found.IdleTimeout) * time.Second, + } + + // ensure valid values + if cfg.Timeout <= 0 { + cfg.Timeout = DefaultTimeout + } + if cfg.IdleTimeout <= 0 { + cfg.IdleTimeout = DefaultIdleTimeout } s.Runner.Enqueue() diff --git a/docs/README.md b/docs/README.md index 496ffa6ac..41529dc22 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,22 +1,19 @@ # Oracle Functions Documentation -* [FAQ](faq.md) - ## For Developers If you are a developer using Oracle Functions through the API, this section is for you. * [Quickstart](https://github.com/treeder/functions#quickstart) * [Usage](usage.md) -* [Definitions](definitions.md) -* [fn (CLI Tool)](/fn/README.md) * [Writing functions](writing.md) -* [Writing Lambda functions](lambda/README.md) +* [fn (CLI Tool)](/fn/README.md) * [Function file (func.yaml)](function-file.md) * [Packaging functions](packaging.md) * [Open Function Format](function-format.md) * [API Reference](http://petstore.swagger.io/?url=https://raw.githubusercontent.com/treeder/functions/master/docs/swagger.yml) * [Hot functions](hot-functions.md) +* [FAQ](faq.md) ## For Operators diff --git a/docs/function-file.md b/docs/function-file.md index 9064cd35d..93f0aca7d 100644 --- a/docs/function-file.md +++ b/docs/function-file.md @@ -45,6 +45,8 @@ Default: `sync`. function. If this function exceeds this limit during execution, it is stopped and error message is logged. Default: `128`. +`timeout` (optional) is the maximum time a function will be allowed to run. Default is 30 seconds. + `headers` (optional) is a set of HTTP headers to be returned in the response of this function calls. @@ -61,6 +63,9 @@ hot functions support also adds two extra options to this configuration file. `format` (optional) is one of the streaming formats covered at [function-format.md](function-format.md). +`idle_timeout` (optional) is the time in seconds a container will remain alive without receiving any new requests; +hot functions will stay alive as long as they receive a request in this interval. Default: `30`. + `max_concurrency` (optional) is the maximum of hot functions per node to be started for a certain function. It defaults to one per function. If you understand you need more processing power, make sure to raise this number. diff --git a/docs/lambda/README.md b/docs/lambda/README.md index d4022310b..814de3937 100644 --- a/docs/lambda/README.md +++ b/docs/lambda/README.md @@ -13,3 +13,5 @@ fn init --runtime lambda-node /lambda-node ``` Be sure the filename for your main handler is `func.js`. + +TODO: Make Java and Python use the new workflow too. diff --git a/docs/lambda/about.md b/docs/lambda/about.md deleted file mode 100644 index 4bd463fd7..000000000 --- a/docs/lambda/about.md +++ /dev/null @@ -1,48 +0,0 @@ - - -AWS Lambda introduced serverless computing to the masses. Wouldn't it be nice -if you could run the same Lambda functions on any platform, in any cloud? -Oracle is proud to release a set of tools that allow just this. Package your -Lambda function in a Docker container and run it anywhere with an environment -similar to AWS Lambda. - -Using a job scheduler such as Oracle Functions, you can connect these functions to -webhooks and run them on-demand, at scale. You can also use a container -management system paired with a task queue to run these functions in -a self-contained, platform-independent manner. - -## Use cases - -Lambda functions are great for writing "worker" processes that perform some -simple, parallelizable task like image processing, ETL transformations, -asynchronous operations driven by Web APIs, or large batch processing. - -All the benefits that containerization brings apply here. Our tools make it -easy to write containerized applications that will run anywhere without having -to fiddle with Docker and get the various runtimes set up. Instead you can just -write a simple function and have an "executable" ready to go. - -## How does it work? - -We provide base Docker images for the various runtimes that AWS Lambda -supports. The `fn` tool helps package up your Lambda function into -a Docker image layered on the base image. We provide a bootstrap script and -utilities that provide a AWS Lambda environment to your code. You can then run -the Docker image on any platform that supports Docker. This allows you to -easily move Lambda functions to any cloud provider, or host it yourself. - -## Next steps - -Write, package and run your Lambda functions with our [Getting started -guide](./getting-started.md). [Here is the environment](./environment.md) that -Lambda provides. `fn lambda` lists the commands to work with Lambda -functions locally. - -You can [import](./import.md) existing Lambda functions hosted on Amazon! -The Docker environment required to run Lambda functions is described -[here](./docker.md). - -Non-AWS Lambda functions can continue to interact with AWS services. [Working -with AWS](./aws.md) describes how to access AWS credentials, interact with -services like S3 and how to launch a Lambda function due a notification from -SNS. diff --git a/docs/lambda/aws.md b/docs/lambda/aws.md index b19db9bb9..eee200e07 100644 --- a/docs/lambda/aws.md +++ b/docs/lambda/aws.md @@ -1,3 +1,5 @@ +# OUT OF DATE - NEEDS TO BE UPDATED + Interacting with AWS Services ============================= diff --git a/docs/lambda/create.md b/docs/lambda/create.md index d44330157..1d2a8bdc9 100644 --- a/docs/lambda/create.md +++ b/docs/lambda/create.md @@ -1,3 +1,5 @@ +# OUT OF DATE - NEEDS TO BE UPDATED + # Creating Docker images out of Lambda functions Docker images created by running the `create-function` subcommand on a Lambda function are ready to execute. diff --git a/docs/lambda/import.md b/docs/lambda/import.md index 71f7f08b2..3b73f60a8 100644 --- a/docs/lambda/import.md +++ b/docs/lambda/import.md @@ -1,3 +1,5 @@ +# OUT OF DATE - NEEDS TO BE UPDATED + Import existing AWS Lambda functions ==================================== diff --git a/docs/lambda/test.md b/docs/lambda/test.md index 94d470cea..8927e1811 100644 --- a/docs/lambda/test.md +++ b/docs/lambda/test.md @@ -1,3 +1,5 @@ +# OUT OF DATE - NEEDS TO BE UPDATED + # Testing the Lambda Docker images The `test-function` subcommand can pass the correct parameters to `docker run` diff --git a/docs/packaging.md b/docs/packaging.md index c58038a20..17fa845aa 100644 --- a/docs/packaging.md +++ b/docs/packaging.md @@ -1,15 +1,18 @@ # Packaging your Function +## Option 1 (recommended): Use the `fn` cli tool + +We recommend using the [fn cli tool](../fn/README.md) which will handle all of this for you. But if you'd like to dig in +and customize your images, look at Option 2. + +## Option 2: Build your own images + Packaging a function has two parts: * Create a Docker image for your function with an ENTRYPOINT * Push your Docker image to a registry (Docker Hub by default) -Once it's pushed to a registry, you can use it by referencing it when adding a route. - -## Using fn - -This is the easiest way to build, package and deploy your functions. +Once it's pushed to a registry, you can use the image location when adding a route. ### Creating an image @@ -17,7 +20,7 @@ The basic Dockerfile for most languages is along these lines: ``` # Choose base image -FROM golang:alpine +FROM funcy/node:dev # Set the working directory WORKDIR /function # Add your binary or code to the working directory @@ -26,29 +29,16 @@ ADD funcbin /function/ ENTRYPOINT ["./funcbin"] ``` -Then you simply build your function: +Then build your function image: ```sh -docker run --rm -v ${pwd}:/go/src/$FUNCPKG -w /go/src/$FUNCPKG funcy/go:dev go build -o funcbin docker build -t $USERNAME/myfunction . ``` -Or using [fn](../fn/README.md): - -```sh -fn build -``` - ### Push your image -This part is simple: - ```sh docker push $USERNAME/myfunction ``` -Or using [fn](../fn/README.md): - -```sh -fn push -``` +Now you can use that image when creating or updating routes. diff --git a/docs/writing.md b/docs/writing.md index 703942a8d..ccb723608 100644 --- a/docs/writing.md +++ b/docs/writing.md @@ -1,5 +1,5 @@ # Writing Functions - + This will give you the basic overview of writing base level functions. You can also use higher level abstractions that make it easier such as [lambda](lambda/README.md). @@ -18,9 +18,9 @@ body = JSON.parse(STDIN) # Do something return_struct = doSomething(body) -# Respond if sync: +# If sync, respond: STDOUT.write(JSON.generate(return_struct)) -# or update something if async +# If async, update something: db.update(return_struct) ``` @@ -66,7 +66,24 @@ In Ruby: STDERR.puts("hi") ``` +## Using Lambda Functions + +### Lambda everywhere + +Lambda support for Oracle Functios enables you to take your AWS Lambda functions and run them +anywhere. You should be able to take your code and run them without any changes. + +Creating Lambda functions is not much different than using regular functions, just use +the `lambda-node` runtime. + +```sh +fn init --runtime lambda-node /lambda-node +``` + +Be sure the filename for your main handler is `func.js`. + +TODO: Make Java and Python use the new workflow too. + ## Next Steps * [Packaging your function](packaging.md) -