From ca18ae88fa345a023d9154ce5605a46fc899d282 Mon Sep 17 00:00:00 2001 From: Travis Reeder Date: Wed, 22 Mar 2017 13:41:27 -0700 Subject: [PATCH] Added Lambda Node support as part of the regular functions workflow. (#601) * Added Lambda Node support as part of the regular functions workflow. * Fixes for PR comments. --- docs/README.md | 2 +- docs/lambda/README.md | 53 +---- docs/lambda/about.md | 48 +++++ examples/hello/python/.gitignore | 1 + examples/lambda/node/.gitignore | 2 + examples/lambda/node/README.md | 13 ++ examples/lambda/node/func.js | 12 ++ examples/lambda/node/payload.json | 5 + fn/common.go | 67 +++--- fn/deploy.go | 14 +- fn/funcfile.go | 40 ++-- fn/init.go | 40 ++-- fn/lambda.go | 3 +- fn/lambda/node/Dockerfile | 13 ++ fn/lambda/node/bootstrap.js | 330 ++++++++++++++++++++++++++++++ fn/lambda/node/build.sh | 3 + fn/lambda/node/release.sh | 3 + fn/langs/base.go | 29 ++- fn/langs/dotnet.go | 4 +- fn/langs/go.go | 1 + fn/langs/lambda_node.go | 26 +++ fn/langs/node.go | 1 + fn/langs/python.go | 1 + fn/langs/ruby.go | 1 + fn/langs/rust.go | 4 +- fn/routes.go | 8 +- fn/testfn.go | 4 +- 27 files changed, 591 insertions(+), 137 deletions(-) create mode 100644 docs/lambda/about.md create mode 100644 examples/lambda/node/.gitignore create mode 100644 examples/lambda/node/README.md create mode 100644 examples/lambda/node/func.js create mode 100644 examples/lambda/node/payload.json create mode 100644 fn/lambda/node/Dockerfile create mode 100644 fn/lambda/node/bootstrap.js create mode 100755 fn/lambda/node/build.sh create mode 100755 fn/lambda/node/release.sh create mode 100644 fn/langs/lambda_node.go diff --git a/docs/README.md b/docs/README.md index cf88ebda8..74703ea89 100644 --- a/docs/README.md +++ b/docs/README.md @@ -10,7 +10,7 @@ If you are a developer using IronFunctions through the API, this section is for * [Definitions](definitions.md) * [fn (CLI Tool)](/fn/README.md) * [Writing functions](writing.md) -* [Writing Lambda functions](lambda/create.md) +* [Writing Lambda functions](lambda/README.md) * [Function file (func.yaml)](function-file.md) * [Packaging functions](packaging.md) * [Open Function Format](function-format.md) diff --git a/docs/lambda/README.md b/docs/lambda/README.md index ef13e1651..4d7e609a1 100644 --- a/docs/lambda/README.md +++ b/docs/lambda/README.md @@ -1,48 +1,15 @@ -# Lambda everywhere. +# Lambda everywhere -AWS Lambda introduced serverless computing to the masses. Wouldn't it be nice -if you could run the same Lambda functions on any platform, in any cloud? -Iron.io is proud to release a set of tools that allow just this. Package your -Lambda function in a Docker container and run it anywhere with an environment -similar to AWS Lambda. +Lambda support for IronFunctions enables you to take your AWS Lambda functions and run them +anywhere. You should be able to take your code and run them without any changes. -Using a job scheduler such as IronFunctions, you can connect these functions to -webhooks and run them on-demand, at scale. You can also use a container -management system paired with a task queue to run these functions in -a self-contained, platform-independent manner. +## Creating Lambda Functions -## Use cases +Creating Lambda functions is not much different than using regular functions, just use +the `lambda-node` runtime. -Lambda functions are great for writing "worker" processes that perform some -simple, parallelizable task like image processing, ETL transformations, -asynchronous operations driven by Web APIs, or large batch processing. +```sh +fn init --runtime lambda-node /lambda-node +``` -All the benefits that containerization brings apply here. Our tools make it -easy to write containerized applications that will run anywhere without having -to fiddle with Docker and get the various runtimes set up. Instead you can just -write a simple function and have an "executable" ready to go. - -## How does it work? - -We provide base Docker images for the various runtimes that AWS Lambda -supports. The `fn` tool helps package up your Lambda function into -a Docker image layered on the base image. We provide a bootstrap script and -utilities that provide a AWS Lambda environment to your code. You can then run -the Docker image on any platform that supports Docker. This allows you to -easily move Lambda functions to any cloud provider, or host it yourself. - -## Next steps - -Write, package and run your Lambda functions with our [Getting started -guide](./getting-started.md). [Here is the environment](./environment.md) that -Lambda provides. `fn lambda` lists the commands to work with Lambda -functions locally. - -You can [import](./import.md) existing Lambda functions hosted on Amazon! -The Docker environment required to run Lambda functions is described -[here](./docker.md). - -Non-AWS Lambda functions can continue to interact with AWS services. [Working -with AWS](./aws.md) describes how to access AWS credentials, interact with -services like S3 and how to launch a Lambda function due a notification from -SNS. +Be sure the filename for your main handler is `func.js`. diff --git a/docs/lambda/about.md b/docs/lambda/about.md new file mode 100644 index 000000000..c8fa20ab2 --- /dev/null +++ b/docs/lambda/about.md @@ -0,0 +1,48 @@ + + +AWS Lambda introduced serverless computing to the masses. Wouldn't it be nice +if you could run the same Lambda functions on any platform, in any cloud? +Iron.io is proud to release a set of tools that allow just this. Package your +Lambda function in a Docker container and run it anywhere with an environment +similar to AWS Lambda. + +Using a job scheduler such as IronFunctions, you can connect these functions to +webhooks and run them on-demand, at scale. You can also use a container +management system paired with a task queue to run these functions in +a self-contained, platform-independent manner. + +## Use cases + +Lambda functions are great for writing "worker" processes that perform some +simple, parallelizable task like image processing, ETL transformations, +asynchronous operations driven by Web APIs, or large batch processing. + +All the benefits that containerization brings apply here. Our tools make it +easy to write containerized applications that will run anywhere without having +to fiddle with Docker and get the various runtimes set up. Instead you can just +write a simple function and have an "executable" ready to go. + +## How does it work? + +We provide base Docker images for the various runtimes that AWS Lambda +supports. The `fn` tool helps package up your Lambda function into +a Docker image layered on the base image. We provide a bootstrap script and +utilities that provide a AWS Lambda environment to your code. You can then run +the Docker image on any platform that supports Docker. This allows you to +easily move Lambda functions to any cloud provider, or host it yourself. + +## Next steps + +Write, package and run your Lambda functions with our [Getting started +guide](./getting-started.md). [Here is the environment](./environment.md) that +Lambda provides. `fn lambda` lists the commands to work with Lambda +functions locally. + +You can [import](./import.md) existing Lambda functions hosted on Amazon! +The Docker environment required to run Lambda functions is described +[here](./docker.md). + +Non-AWS Lambda functions can continue to interact with AWS services. [Working +with AWS](./aws.md) describes how to access AWS credentials, interact with +services like S3 and how to launch a Lambda function due a notification from +SNS. diff --git a/examples/hello/python/.gitignore b/examples/hello/python/.gitignore index 23053de09..44536cce5 100644 --- a/examples/hello/python/.gitignore +++ b/examples/hello/python/.gitignore @@ -1 +1,2 @@ packages/ +func.yaml diff --git a/examples/lambda/node/.gitignore b/examples/lambda/node/.gitignore new file mode 100644 index 000000000..e92740660 --- /dev/null +++ b/examples/lambda/node/.gitignore @@ -0,0 +1,2 @@ +func.yaml +/node_modules diff --git a/examples/lambda/node/README.md b/examples/lambda/node/README.md new file mode 100644 index 000000000..640a0a91c --- /dev/null +++ b/examples/lambda/node/README.md @@ -0,0 +1,13 @@ +# Lambda Node Example + +This is the exact same function that is in the AWS Lambda tutorial. + +Other than a different runtime, this is no different than any other node example. + +To use the lambda-node runtime, use this `fn init` command: + +```sh +fn init --runtime lambda-node /lambda-node +fn build +cat payload.json | fn run +``` diff --git a/examples/lambda/node/func.js b/examples/lambda/node/func.js new file mode 100644 index 000000000..b92b0e8f4 --- /dev/null +++ b/examples/lambda/node/func.js @@ -0,0 +1,12 @@ +'use strict'; + +console.log('Loading function'); + +exports.handler = (event, context, callback) => { + //console.log('Received event:', JSON.stringify(event, null, 2)); + console.log('value1 =', event.key1); + console.log('value2 =', event.key2); + console.log('value3 =', event.key3); + callback(null, event.key1); // Echo back the first key value + //callback('Something went wrong'); +}; diff --git a/examples/lambda/node/payload.json b/examples/lambda/node/payload.json new file mode 100644 index 000000000..9162de753 --- /dev/null +++ b/examples/lambda/node/payload.json @@ -0,0 +1,5 @@ +{ + "key3": "value3", + "key2": "value2", + "key1": "value1" +} \ No newline at end of file diff --git a/fn/common.go b/fn/common.go index db7ef286e..8e909d674 100644 --- a/fn/common.go +++ b/fn/common.go @@ -79,9 +79,9 @@ func dockerbuild(verbwriter io.Writer, path string, ff *funcfile) error { if err != nil { return err } - helper, err = langs.GetLangHelper(*ff.Runtime) - if err != nil { - return err + helper = langs.GetLangHelper(*ff.Runtime) + if helper == nil { + return fmt.Errorf("Cannot build, no language helper found for %v", *ff.Runtime) } if helper.HasPreBuild() { err := helper.PreBuild() @@ -118,32 +118,34 @@ func exists(name string) bool { } var acceptableFnRuntimes = map[string]string{ - "elixir": "iron/elixir", - "erlang": "iron/erlang", - "gcc": "iron/gcc", - "go": "iron/go", - "java": "iron/java", - "leiningen": "iron/leiningen", - "mono": "iron/mono", - "node": "iron/node", - "perl": "iron/perl", - "php": "iron/php", - "python": "iron/python:2", - "ruby": "iron/ruby", - "scala": "iron/scala", - "rust": "corey/rust-alpine", - "dotnet": "microsoft/dotnet:runtime", + "elixir": "iron/elixir", + "erlang": "iron/erlang", + "gcc": "iron/gcc", + "go": "iron/go", + "java": "iron/java", + "leiningen": "iron/leiningen", + "mono": "iron/mono", + "node": "iron/node", + "perl": "iron/perl", + "php": "iron/php", + "python": "iron/python:2", + "ruby": "iron/ruby", + "scala": "iron/scala", + "rust": "corey/rust-alpine", + "dotnet": "microsoft/dotnet:runtime", + "lambda-node": "iron/functions-lambda:node", } const tplDockerfile = `FROM {{ .BaseImage }} WORKDIR /function ADD . /function/ -ENTRYPOINT [{{ .Entrypoint }}] +{{ if ne .Entrypoint "" }} ENTRYPOINT [{{ .Entrypoint }}] {{ end }} +{{ if ne .Cmd "" }} CMD [{{ .Cmd }}] {{ end }} ` func writeTmpDockerfile(dir string, ff *funcfile) error { - if ff.Entrypoint == nil || *ff.Entrypoint == "" { - return errors.New("entrypoint is missing") + if ff.Entrypoint == "" && ff.Cmd == "" { + return errors.New("entrypoint and cmd are missing, you must provide one or the other") } runtime, tag := ff.RuntimeTag() @@ -160,9 +162,22 @@ func writeTmpDockerfile(dir string, ff *funcfile) error { if err != nil { return err } + defer fd.Close() // convert entrypoint string to slice - epvals := strings.Fields(*ff.Entrypoint) + bufferEp := stringToSlice(ff.Entrypoint) + bufferCmd := stringToSlice(ff.Cmd) + + t := template.Must(template.New("Dockerfile").Parse(tplDockerfile)) + err = t.Execute(fd, struct { + BaseImage, Entrypoint, Cmd string + }{rt, bufferEp.String(), bufferCmd.String()}) + + return err +} + +func stringToSlice(in string) bytes.Buffer { + epvals := strings.Fields(in) var buffer bytes.Buffer for i, s := range epvals { if i > 0 { @@ -172,13 +187,7 @@ func writeTmpDockerfile(dir string, ff *funcfile) error { buffer.WriteString(s) buffer.WriteString("\"") } - - t := template.Must(template.New("Dockerfile").Parse(tplDockerfile)) - err = t.Execute(fd, struct { - BaseImage, Entrypoint string - }{rt, buffer.String()}) - fd.Close() - return err + return buffer } func extractEnvConfig(configs []string) map[string]string { diff --git a/fn/deploy.go b/fn/deploy.go index 90aba0208..509ad25f9 100644 --- a/fn/deploy.go +++ b/fn/deploy.go @@ -135,9 +135,9 @@ func (p *deploycmd) route(path string, ff *funcfile) error { return fmt.Errorf("error setting endpoint: %v", err) } - if ff.path == nil { + if ff.Path == nil { _, path := appNamePath(ff.FullName()) - ff.path = &path + ff.Path = &path } if ff.Memory == nil { @@ -149,8 +149,8 @@ func (p *deploycmd) route(path string, ff *funcfile) error { if ff.Format == nil { ff.Format = new(string) } - if ff.maxConcurrency == nil { - ff.maxConcurrency = new(int) + if ff.MaxConcurrency == nil { + ff.MaxConcurrency = new(int) } if ff.Timeout == nil { dur := time.Duration(0) @@ -163,19 +163,19 @@ func (p *deploycmd) route(path string, ff *funcfile) error { } body := functions.RouteWrapper{ Route: functions.Route{ - Path: *ff.path, + Path: *ff.Path, Image: ff.FullName(), Memory: *ff.Memory, Type_: *ff.Type, Config: expandEnvConfig(ff.Config), Headers: headers, Format: *ff.Format, - MaxConcurrency: int32(*ff.maxConcurrency), + MaxConcurrency: int32(*ff.MaxConcurrency), Timeout: int32(ff.Timeout.Seconds()), }, } - fmt.Fprintf(p.verbwriter, "updating API with app: %s route: %s name: %s \n", p.appName, *ff.path, ff.Name) + fmt.Fprintf(p.verbwriter, "updating API with app: %s route: %s name: %s \n", p.appName, *ff.Path, ff.Name) wrapper, resp, err := p.AppsAppRoutesPost(p.appName, body) if err != nil { diff --git a/fn/funcfile.go b/fn/funcfile.go index 6939be956..1545eb1f2 100644 --- a/fn/funcfile.go +++ b/fn/funcfile.go @@ -24,29 +24,29 @@ var ( ) type fftest struct { - Name string `yaml:"name,omitempty",json:"name,omitempty"` - In *string `yaml:"in,omitempty",json:"in,omitempty"` - Out *string `yaml:"out,omitempty",json:"out,omitempty"` - Err *string `yaml:"err,omitempty",json:"err,omitempty"` - Env map[string]string `yaml:"env,omitempty",json:"env,omitempty"` + Name string `yaml:"name,omitempty" json:"name,omitempty"` + In *string `yaml:"in,omitempty" json:"in,omitempty"` + Out *string `yaml:"out,omitempty" json:"out,omitempty"` + Err *string `yaml:"err,omitempty" json:"err,omitempty"` + Env map[string]string `yaml:"env,omitempty" json:"env,omitempty"` } type funcfile struct { - Name string `yaml:"name,omitempty",json:"name,omitempty"` - Version string `yaml:"version,omitempty",json:"version,omitempty"` - Runtime *string `yaml:"runtime,omitempty",json:"runtime,omitempty"` - Entrypoint *string `yaml:"entrypoint,omitempty",json:"entrypoint,omitempty"` - Type *string `yaml:"type,omitempty",json:"type,omitempty"` - Memory *int64 `yaml:"memory,omitempty",json:"memory,omitempty"` - Format *string `yaml:"format,omitempty",json:"format,omitempty"` - Timeout *time.Duration `yaml:"timeout,omitempty",json:"timeout,omitempty"` - Headers map[string]string `yaml:"headers,omitempty",json:"headers,omitempty"` - Config map[string]string `yaml:"config,omitempty",json:"config,omitempty"` - Build []string `yaml:"build,omitempty",json:"build,omitempty"` - Tests []fftest `yaml:"tests,omitempty",json:"tests,omitempty"` - - path *string `yaml:"path,omitempty",json:"path,omitempty"` - maxConcurrency *int `yaml:"max_concurrency,omitempty",json:"max_concurrency,omitempty"` + Name string `yaml:"name,omitempty" json:"name,omitempty"` + Version string `yaml:"version,omitempty" json:"version,omitempty"` + Runtime *string `yaml:"runtime,omitempty" json:"runtime,omitempty"` + Entrypoint string `yaml:"entrypoint,omitempty" json:"entrypoint,omitempty"` + Cmd string `yaml:"cmd,omitempty" json:"cmd,omitempty"` + Type *string `yaml:"type,omitempty" json:"type,omitempty"` + Memory *int64 `yaml:"memory,omitempty" json:"memory,omitempty"` + Format *string `yaml:"format,omitempty" json:"format,omitempty"` + Timeout *time.Duration `yaml:"timeout,omitempty" json:"timeout,omitempty"` + Headers map[string]string `yaml:"headers,omitempty" json:"headers,omitempty"` + Config map[string]string `yaml:"config,omitempty" json:"config,omitempty"` + Build []string `yaml:"build,omitempty" json:"build,omitempty"` + Tests []fftest `yaml:"tests,omitempty" json:"tests,omitempty"` + Path *string `yaml:"path,omitempty" json:"path,omitempty"` + MaxConcurrency *int `yaml:"max_concurrency,omitempty" json:"max_concurrency,omitempty"` } func (ff *funcfile) FullName() string { diff --git a/fn/init.go b/fn/init.go index 0c79446a3..b89ff44b5 100644 --- a/fn/init.go +++ b/fn/init.go @@ -46,6 +46,7 @@ type initFnCmd struct { force bool runtime string entrypoint string + cmd string format string maxConcurrency int } @@ -116,13 +117,14 @@ func (a *initFnCmd) init(c *cli.Context) error { Name: a.name, Runtime: &a.runtime, Version: initialVersion, - Entrypoint: &a.entrypoint, + Entrypoint: a.entrypoint, + Cmd: a.cmd, Format: ffmt, - maxConcurrency: &a.maxConcurrency, + MaxConcurrency: &a.maxConcurrency, } _, path := appNamePath(ff.FullName()) - ff.path = &path + ff.Path = &path if err := encodeFuncfileYAML("func.yaml", ff); err != nil { return err @@ -135,7 +137,7 @@ func (a *initFnCmd) init(c *cli.Context) error { func (a *initFnCmd) buildFuncFile(c *cli.Context) error { pwd, err := os.Getwd() if err != nil { - return fmt.Errorf("error detecting current working directory: %s\n", err) + return fmt.Errorf("error detecting current working directory: %s", err) } a.name = c.Args().First() @@ -157,16 +159,28 @@ func (a *initFnCmd) buildFuncFile(c *cli.Context) error { a.runtime = rt fmt.Printf("assuming %v runtime\n", rt) } + fmt.Println("runtime:", a.runtime) if _, ok := acceptableFnRuntimes[a.runtime]; !ok { return fmt.Errorf("init does not support the %s runtime, you'll have to create your own Dockerfile for this function", a.runtime) } + helper := langs.GetLangHelper(a.runtime) + if helper == nil { + fmt.Printf("No helper found for %s runtime, you'll have to pass in the appropriate flags or use a Dockerfile.", a.runtime) + } + if a.entrypoint == "" { - ep, err := detectEntrypoint(a.runtime) - if err != nil { - return fmt.Errorf("could not detect entrypoint for %v, use --entrypoint to add it explicitly. %v", a.runtime, err) + if helper != nil { + a.entrypoint = helper.Entrypoint() } - a.entrypoint = ep + } + if a.cmd == "" { + if helper != nil { + a.cmd = helper.Cmd() + } + } + if a.entrypoint == "" && a.cmd == "" { + return fmt.Errorf("could not detect entrypoint or cmd for %v, use --entrypoint and/or --cmd to set them explicitly", a.runtime) } return nil @@ -179,15 +193,5 @@ func detectRuntime(path string) (runtime string, err error) { return runtime, nil } } - return "", fmt.Errorf("no supported files found to guess runtime, please set runtime explicitly with --runtime flag") } - -func detectEntrypoint(runtime string) (string, error) { - helper, err := langs.GetLangHelper(runtime) - if err != nil { - return "", err - } - - return helper.Entrypoint(), nil -} diff --git a/fn/lambda.go b/fn/lambda.go index e497c4807..6376f6548 100644 --- a/fn/lambda.go +++ b/fn/lambda.go @@ -100,6 +100,7 @@ func transcribeEnvConfig(configs []string) map[string]string { return c } +// create creates the Docker image for the Lambda function func create(c *cli.Context) error { args := c.Args() functionName := args[0] @@ -272,7 +273,7 @@ func createFunctionYaml(opts createImageOptions) error { path := fmt.Sprintf("/%s", strs[1]) funcDesc := &funcfile{ Name: opts.Name, - path: &path, + Path: &path, Config: opts.Config, } diff --git a/fn/lambda/node/Dockerfile b/fn/lambda/node/Dockerfile new file mode 100644 index 000000000..b8a3d1acf --- /dev/null +++ b/fn/lambda/node/Dockerfile @@ -0,0 +1,13 @@ +FROM node:4-alpine + +WORKDIR /function + +# Install ImageMagick and AWS SDK as provided by Lambda. +RUN apk update && apk --no-cache add imagemagick +RUN npm install aws-sdk@2.2.32 imagemagick && npm cache clear + +# ironcli should forbid this name +ADD bootstrap.js /function/lambda-bootstrap.js + +# Run the handler, with a payload in the future. +ENTRYPOINT ["node", "./lambda-bootstrap"] diff --git a/fn/lambda/node/bootstrap.js b/fn/lambda/node/bootstrap.js new file mode 100644 index 000000000..e51c74643 --- /dev/null +++ b/fn/lambda/node/bootstrap.js @@ -0,0 +1,330 @@ +'use strict'; + +var fs = require('fs'); + +var oldlog = console.log +console.log = console.error + +// Some notes on the semantics of the succeed(), fail() and done() methods. +// Tests are the source of truth! +// First call wins in terms of deciding the result of the function. BUT, +// subsequent calls also log. Further, code execution does not stop, even where +// for done(), the docs say that the "function terminates". It seems though +// that further cycles of the event loop do not run. For example: +// index.handler = function(event, context) { +// context.fail("FAIL") +// process.nextTick(function() { +// console.log("This does not get logged") +// }) +// console.log("This does get logged") +// } +// on the other hand: +// index.handler = function(event, context) { +// process.nextTick(function() { +// console.log("This also gets logged") +// context.fail("FAIL") +// }) +// console.log("This does get logged") +// } +// +// The same is true for context.succeed() and done() captures the semantics of +// both. It seems this is implemented simply by having process.nextTick() cause +// process.exit() or similar, because the following: +// exports.handler = function(event, context) { +// process.nextTick(function() {console.log("This gets logged")}) +// process.nextTick(function() {console.log("This also gets logged")}) +// context.succeed("END") +// process.nextTick(function() {console.log("This does not get logged")}) +// }; +// +// So the context object needs to have some sort of hidden boolean that is only +// flipped once, by the first call, and dictates the behavior on the next tick. +// +// In addition, the response behaviour depends on the invocation type. If we +// are to only support the async type, succeed() must return a 202 response +// code, not sure how to do this. +// +// Only the first 256kb, followed by a truncation message, should be logged. +// +// Also, the error log is always in a json literal +// { "errorMessage": "" } +var Context = function() { + var concluded = false; + + var contextSelf = this; + + // The succeed, fail and done functions are public, but access a private + // member (concluded). Hence this ugly nested definition. + this.succeed = function(result) { + if (concluded) { + return + } + + // We have to process the result before we can conclude, because otherwise + // we have to fail. This means NO EARLY RETURNS from this function without + // review! + if (result === undefined) { + result = null + } + + var failed = false; + try { + // Output result to log + oldlog(JSON.stringify(result)); + } catch(e) { + // Set X-Amz-Function-Error: Unhandled header + console.log("Unable to stringify body as json: " + e); + failed = true; + } + + // FIXME(nikhil): Return 202 or 200 based on invocation type and set response + // to result. Should probably be handled externally by the runner/swapi. + + // OK, everything good. + concluded = true; + process.nextTick(function() { process.exit(failed ? 1 : 0) }) + } + + this.fail = function(error) { + if (concluded) { + return + } + + concluded = true + process.nextTick(function() { process.exit(1) }) + + if (error === undefined) { + error = null + } + + // FIXME(nikhil): Truncated log of error, plus non-truncated response body + var errstr = "fail() called with argument but a problem was encountered while converting it to a to string"; + + // The semantics of fail() are weird. If the error is something that can be + // converted to a string, the log output wraps the string in a JSON literal + // with key "errorMessage". If toString() fails, then the output is only + // the error string. + try { + if (error === null) { + errstr = null + } else { + errstr = error.toString() + } + oldlog(JSON.stringify({"errorMessage": errstr })) + } catch(e) { + // Set X-Amz-Function-Error: Unhandled header + oldlog(errstr) + } + } + + this.done = function() { + var error = arguments[0]; + var result = arguments[1]; + if (error) { + contextSelf.fail(error) + } else { + contextSelf.succeed(result) + } + } + + var plannedEnd = Date.now() + (getTimeoutInSeconds() * 1000); + this.getRemainingTimeInMillis = function() { + return Math.max(plannedEnd - Date.now(), 0); + } +} + +function getTimeoutInSeconds() { + var t = parseInt(getEnv("TASK_TIMEOUT")); + if (Number.isNaN(t)) { + return 3600; + } + + return t; +} + +var getEnv = function(name) { + return process.env[name] || ""; +} + +var makeCtx = function() { + var fnname = getEnv("AWS_LAMBDA_FUNCTION_NAME"); + // FIXME(nikhil): Generate UUID. + var taskID = getEnv("TASK_ID"); + + var mem = getEnv("TASK_MAXRAM").toLowerCase(); + var bytes = 300 * 1024 * 1024; + + var scale = { 'b': 1, 'k': 1024, 'm': 1024*1024, 'g': 1024*1024*1024 }; + // We don't bother validating too much, if the last character is not a number + // and not in the scale table we just return a default value. + // We use slice instead of indexing so that we always get an empty string, + // instead of undefined. + if (mem.slice(-1).match(/[0-9]/)) { + var a = parseInt(mem); + if (!Number.isNaN(a)) { + bytes = a; + } + } else { + var rem = parseInt(mem.slice(0, -1)); + if (!Number.isNaN(rem)) { + var multiplier = scale[mem.slice(-1)]; + if (multiplier) { + bytes = rem * multiplier + } + } + } + + var memoryMB = bytes / (1024 * 1024); + + var ctx = new Context(); + Object.defineProperties(ctx, { + "functionName": { + value: fnname, + enumerable: true, + }, + "functionVersion": { + value: "$LATEST", + enumerable: true, + }, + "invokedFunctionArn": { + // FIXME(nikhil): Should be filled in. + value: "", + enumerable: true, + }, + "memoryLimitInMB": { + // Sigh, yes it is a string. + value: ""+memoryMB, + enumerable: true, + }, + "awsRequestId": { + value: taskID, + enumerable: true, + }, + "logGroupName": { + // FIXME(nikhil): Should be filled in. + value: "", + enumerable: true, + }, + "logStreamName": { + // FIXME(nikhil): Should be filled in. + value: "", + enumerable: true, + }, + "identity": { + // FIXME(nikhil): Should be filled in. + value: null, + enumerable: true, + }, + "clientContext": { + // FIXME(nikhil): Should be filled in. + value: null, + enumerable: true, + }, + }); + + return ctx; +} + +var setEnvFromHeader = function () { + var headerPrefix = "CONFIG_"; + var newEnvVars = {}; + for (var key in process.env) { + if (key.indexOf(headerPrefix) == 0) { + newEnvVars[key.slice(headerPrefix.length)] = process.env[key]; + } + } + + for (var key in newEnvVars) { + process.env[key] = newEnvVars[key]; + } +} + + +function run() { + setEnvFromHeader(); + // FIXME(nikhil): Check for file existence and allow non-payload. + var path = process.env["PAYLOAD_FILE"]; + var stream = process.stdin; + if (path) { + try { + stream = fs.createReadStream(path); + } catch(e) { + console.error("bootstrap: Error opening payload file", e) + process.exit(1); + } + } + + var input = ""; + stream.setEncoding('utf8'); + stream.on('data', function(chunk) { + input += chunk; + }); + + stream.on('error', function(err) { + console.error("bootstrap: Error reading payload stream", err); + process.exit(1); + }); + + stream.on('end', function() { + var payload = {} + try { + if (input.length > 0) { + payload = JSON.parse(input); + } + } catch(e) { + console.error("bootstrap: Error parsing JSON", e); + process.exit(1); + } + + if (process.argv.length > 2) { + var handler = process.argv[2]; + var parts = handler.split('.'); + // FIXME(nikhil): Error checking. + var script = parts[0]; + var entry = parts[1]; + var started = false; + try { + var mod = require('./'+script); + var func = mod[entry]; + if (func === undefined) { + oldlog("Handler '" + entry + "' missing on module '" + script + "'"); + return; + } + + if (typeof func !== 'function') { + throw "TypeError: " + (typeof func) + " is not a function"; + } + started = true; + var cback + // RUN THE FUNCTION: + mod[entry](payload, makeCtx(), functionCallback) + } catch(e) { + if (typeof e === 'string') { + oldlog(e) + } else { + oldlog(e.message) + } + if (!started) { + oldlog("Process exited before completing request\n") + } + } + } else { + console.error("bootstrap: No script specified") + process.exit(1); + } + }) +} + +function functionCallback(err, result) { + if (err != null) { + // then user returned error and we should respond with error + // http://docs.aws.amazon.com/lambda/latest/dg/nodejs-prog-mode-exceptions.html + oldlog(JSON.stringify({"errorMessage": errstr })) + return + } + if (result != null) { + oldlog(JSON.stringify(result)) + } +} + +run() \ No newline at end of file diff --git a/fn/lambda/node/build.sh b/fn/lambda/node/build.sh new file mode 100755 index 000000000..43c3bf0af --- /dev/null +++ b/fn/lambda/node/build.sh @@ -0,0 +1,3 @@ +set -ex + +docker build -t iron/functions-lambda:node . diff --git a/fn/lambda/node/release.sh b/fn/lambda/node/release.sh new file mode 100755 index 000000000..4244c43df --- /dev/null +++ b/fn/lambda/node/release.sh @@ -0,0 +1,3 @@ +set -ex + +docker push iron/functions-lambda:node diff --git a/fn/langs/base.go b/fn/langs/base.go index 7bf1a36c8..16585b74d 100644 --- a/fn/langs/base.go +++ b/fn/langs/base.go @@ -1,29 +1,38 @@ package langs -import "fmt" - // GetLangHelper returns a LangHelper for the passed in language -func GetLangHelper(lang string) (LangHelper, error) { +func GetLangHelper(lang string) LangHelper { switch lang { case "go": - return &GoLangHelper{}, nil + return &GoLangHelper{} case "node": - return &NodeLangHelper{}, nil + return &NodeLangHelper{} case "ruby": - return &RubyLangHelper{}, nil + return &RubyLangHelper{} case "python": - return &PythonHelper{}, nil + return &PythonHelper{} case "rust": - return &RustLangHelper{}, nil + return &RustLangHelper{} case "dotnet": - return &DotNetLangHelper{}, nil + return &DotNetLangHelper{} + case "lambda-node": + return &LambdaNodeHelper{} } - return nil, fmt.Errorf("No language helper found for %v", lang) + return nil } type LangHelper interface { + // Entrypoint sets the Docker Entrypoint. One of Entrypoint or Cmd is required. Entrypoint() string + // Cmd sets the Docker command. One of Entrypoint or Cmd is required. + Cmd() string HasPreBuild() bool PreBuild() error AfterBuild() error } + +// BaseHelper is empty implementation of LangHelper for embedding in implementations. +type BaseHelper struct { +} + +func (h *BaseHelper) Cmd() string { return "" } diff --git a/fn/langs/dotnet.go b/fn/langs/dotnet.go index 23422892b..1ecf21482 100644 --- a/fn/langs/dotnet.go +++ b/fn/langs/dotnet.go @@ -6,7 +6,9 @@ import ( "os/exec" ) -type DotNetLangHelper struct{} +type DotNetLangHelper struct { + BaseHelper +} func (lh *DotNetLangHelper) Entrypoint() string { return "dotnet dotnet.dll" diff --git a/fn/langs/go.go b/fn/langs/go.go index 53d3e4657..bc3d1dea0 100644 --- a/fn/langs/go.go +++ b/fn/langs/go.go @@ -8,6 +8,7 @@ import ( ) type GoLangHelper struct { + BaseHelper } func (lh *GoLangHelper) Entrypoint() string { diff --git a/fn/langs/lambda_node.go b/fn/langs/lambda_node.go new file mode 100644 index 000000000..73c43705c --- /dev/null +++ b/fn/langs/lambda_node.go @@ -0,0 +1,26 @@ +package langs + +type LambdaNodeHelper struct { + BaseHelper +} + +func (lh *LambdaNodeHelper) Entrypoint() string { + return "" +} + +func (lh *LambdaNodeHelper) Cmd() string { + return "func.handler" +} + +func (lh *LambdaNodeHelper) HasPreBuild() bool { + return false +} + +// PreBuild for Go builds the binary so the final image can be as small as possible +func (lh *LambdaNodeHelper) PreBuild() error { + return nil +} + +func (lh *LambdaNodeHelper) AfterBuild() error { + return nil +} diff --git a/fn/langs/node.go b/fn/langs/node.go index d542849fb..a9e593e21 100644 --- a/fn/langs/node.go +++ b/fn/langs/node.go @@ -1,6 +1,7 @@ package langs type NodeLangHelper struct { + BaseHelper } func (lh *NodeLangHelper) Entrypoint() string { diff --git a/fn/langs/python.go b/fn/langs/python.go index 9dce5aeed..beafed0cd 100644 --- a/fn/langs/python.go +++ b/fn/langs/python.go @@ -8,6 +8,7 @@ import ( ) type PythonHelper struct { + BaseHelper } func (lh *PythonHelper) Entrypoint() string { diff --git a/fn/langs/ruby.go b/fn/langs/ruby.go index 75d9f143f..158b5fb83 100644 --- a/fn/langs/ruby.go +++ b/fn/langs/ruby.go @@ -9,6 +9,7 @@ import ( ) type RubyLangHelper struct { + BaseHelper } func (lh *RubyLangHelper) Entrypoint() string { diff --git a/fn/langs/rust.go b/fn/langs/rust.go index eb580742d..95c33cacd 100644 --- a/fn/langs/rust.go +++ b/fn/langs/rust.go @@ -6,7 +6,9 @@ import ( "os/exec" ) -type RustLangHelper struct{} +type RustLangHelper struct { + BaseHelper +} func (lh *RustLangHelper) Entrypoint() string { return "/function/target/release/func" diff --git a/fn/routes.go b/fn/routes.go index d8351fc39..08f814306 100644 --- a/fn/routes.go +++ b/fn/routes.go @@ -296,15 +296,15 @@ func routeWithFuncFile(c *cli.Context, rt *models.Route) { if ff.Format != nil { rt.Format = *ff.Format } - if ff.maxConcurrency != nil { - rt.MaxConcurrency = int32(*ff.maxConcurrency) + if ff.MaxConcurrency != nil { + rt.MaxConcurrency = int32(*ff.MaxConcurrency) } if ff.Timeout != nil { to := int64(ff.Timeout.Seconds()) rt.Timeout = &to } - if rt.Path == "" && ff.path != nil { - rt.Path = *ff.path + if rt.Path == "" && ff.Path != nil { + rt.Path = *ff.Path } } } diff --git a/fn/testfn.go b/fn/testfn.go index acb1a4129..0758eed66 100644 --- a/fn/testfn.go +++ b/fn/testfn.go @@ -68,7 +68,7 @@ func (t *testcmd) test(c *cli.Context) error { target := ff.FullName() runtest := runlocaltest if t.remote != "" { - if ff.path == nil || *ff.path == "" { + if ff.Path == nil || *ff.Path == "" { return errors.New("execution of tests on remote server demand that this function to have a `path`.") } if err := resetBasePath(t.Configuration); err != nil { @@ -80,7 +80,7 @@ func (t *testcmd) test(c *cli.Context) error { } u, err := url.Parse("../") - u.Path = path.Join(u.Path, "r", t.remote, *ff.path) + u.Path = path.Join(u.Path, "r", t.remote, *ff.Path) target = baseURL.ResolveReference(u).String() runtest = runremotetest }