From c6a315ae7db24822afa7b1389ac34118ec666a94 Mon Sep 17 00:00:00 2001 From: Travis Reeder Date: Mon, 19 Jun 2017 11:28:14 -0700 Subject: [PATCH] Fixed up lambda-node to work with multi-stage changes. --- docs/lambda/README.md | 4 +- examples/lambda/node/func.js | 2 +- fn/lambda/{node => node-4}/Dockerfile | 0 fn/lambda/{node => node-4}/bootstrap.js | 0 fn/lambda/node-4/build.sh | 3 + fn/lambda/node-4/release.sh | 5 + fn/lambda/node-6/Dockerfile | 13 + fn/lambda/node-6/bootstrap.js | 330 ++++++++++++++++++++++++ fn/lambda/node-6/build.sh | 3 + fn/lambda/node-6/release.sh | 5 + fn/lambda/node/build.sh | 3 - fn/lambda/node/release.sh | 5 - fn/lambda/release.sh | 6 +- fn/langs/base.go | 9 +- fn/langs/lambda_node.go | 28 +- 15 files changed, 386 insertions(+), 30 deletions(-) rename fn/lambda/{node => node-4}/Dockerfile (100%) rename fn/lambda/{node => node-4}/bootstrap.js (100%) create mode 100755 fn/lambda/node-4/build.sh create mode 100755 fn/lambda/node-4/release.sh create mode 100644 fn/lambda/node-6/Dockerfile create mode 100644 fn/lambda/node-6/bootstrap.js create mode 100755 fn/lambda/node-6/build.sh create mode 100755 fn/lambda/node-6/release.sh delete mode 100755 fn/lambda/node/build.sh delete mode 100755 fn/lambda/node/release.sh diff --git a/docs/lambda/README.md b/docs/lambda/README.md index 814de3937..4831db9aa 100644 --- a/docs/lambda/README.md +++ b/docs/lambda/README.md @@ -6,10 +6,10 @@ anywhere. You should be able to take your code and run them without any changes. ## Creating Lambda Functions Creating Lambda functions is not much different than using regular functions, just use -the `lambda-node` runtime. +the `lambda-node-4` runtime. ```sh -fn init --runtime lambda-node /lambda-node +fn init --runtime lambda-node-4 /lambda-node ``` Be sure the filename for your main handler is `func.js`. diff --git a/examples/lambda/node/func.js b/examples/lambda/node/func.js index b92b0e8f4..288f82ad8 100644 --- a/examples/lambda/node/func.js +++ b/examples/lambda/node/func.js @@ -7,6 +7,6 @@ exports.handler = (event, context, callback) => { console.log('value1 =', event.key1); console.log('value2 =', event.key2); console.log('value3 =', event.key3); - callback(null, event.key1); // Echo back the first key value + callback(null, "hello " + event.key1); // Echo back the first key value //callback('Something went wrong'); }; diff --git a/fn/lambda/node/Dockerfile b/fn/lambda/node-4/Dockerfile similarity index 100% rename from fn/lambda/node/Dockerfile rename to fn/lambda/node-4/Dockerfile diff --git a/fn/lambda/node/bootstrap.js b/fn/lambda/node-4/bootstrap.js similarity index 100% rename from fn/lambda/node/bootstrap.js rename to fn/lambda/node-4/bootstrap.js diff --git a/fn/lambda/node-4/build.sh b/fn/lambda/node-4/build.sh new file mode 100755 index 000000000..4c5d6286f --- /dev/null +++ b/fn/lambda/node-4/build.sh @@ -0,0 +1,3 @@ +set -ex + +docker build --build-arg HTTP_PROXY -t funcy/lambda:node-4 . diff --git a/fn/lambda/node-4/release.sh b/fn/lambda/node-4/release.sh new file mode 100755 index 000000000..ad1b93434 --- /dev/null +++ b/fn/lambda/node-4/release.sh @@ -0,0 +1,5 @@ +set -ex + +./build.sh + +docker push funcy/lambda:node-4 diff --git a/fn/lambda/node-6/Dockerfile b/fn/lambda/node-6/Dockerfile new file mode 100644 index 000000000..bb9f5f495 --- /dev/null +++ b/fn/lambda/node-6/Dockerfile @@ -0,0 +1,13 @@ +FROM node:6-alpine + +WORKDIR /function + +# Install ImageMagick and AWS SDK as provided by Lambda. +RUN apk update && apk --no-cache add imagemagick +RUN npm install aws-sdk@2.2.32 imagemagick && npm cache clear + +# cli should forbid this name +ADD bootstrap.js /function/lambda-bootstrap.js + +# Run the handler, with a payload in the future. +ENTRYPOINT ["node", "./lambda-bootstrap"] diff --git a/fn/lambda/node-6/bootstrap.js b/fn/lambda/node-6/bootstrap.js new file mode 100644 index 000000000..e51c74643 --- /dev/null +++ b/fn/lambda/node-6/bootstrap.js @@ -0,0 +1,330 @@ +'use strict'; + +var fs = require('fs'); + +var oldlog = console.log +console.log = console.error + +// Some notes on the semantics of the succeed(), fail() and done() methods. +// Tests are the source of truth! +// First call wins in terms of deciding the result of the function. BUT, +// subsequent calls also log. Further, code execution does not stop, even where +// for done(), the docs say that the "function terminates". It seems though +// that further cycles of the event loop do not run. For example: +// index.handler = function(event, context) { +// context.fail("FAIL") +// process.nextTick(function() { +// console.log("This does not get logged") +// }) +// console.log("This does get logged") +// } +// on the other hand: +// index.handler = function(event, context) { +// process.nextTick(function() { +// console.log("This also gets logged") +// context.fail("FAIL") +// }) +// console.log("This does get logged") +// } +// +// The same is true for context.succeed() and done() captures the semantics of +// both. It seems this is implemented simply by having process.nextTick() cause +// process.exit() or similar, because the following: +// exports.handler = function(event, context) { +// process.nextTick(function() {console.log("This gets logged")}) +// process.nextTick(function() {console.log("This also gets logged")}) +// context.succeed("END") +// process.nextTick(function() {console.log("This does not get logged")}) +// }; +// +// So the context object needs to have some sort of hidden boolean that is only +// flipped once, by the first call, and dictates the behavior on the next tick. +// +// In addition, the response behaviour depends on the invocation type. If we +// are to only support the async type, succeed() must return a 202 response +// code, not sure how to do this. +// +// Only the first 256kb, followed by a truncation message, should be logged. +// +// Also, the error log is always in a json literal +// { "errorMessage": "" } +var Context = function() { + var concluded = false; + + var contextSelf = this; + + // The succeed, fail and done functions are public, but access a private + // member (concluded). Hence this ugly nested definition. + this.succeed = function(result) { + if (concluded) { + return + } + + // We have to process the result before we can conclude, because otherwise + // we have to fail. This means NO EARLY RETURNS from this function without + // review! + if (result === undefined) { + result = null + } + + var failed = false; + try { + // Output result to log + oldlog(JSON.stringify(result)); + } catch(e) { + // Set X-Amz-Function-Error: Unhandled header + console.log("Unable to stringify body as json: " + e); + failed = true; + } + + // FIXME(nikhil): Return 202 or 200 based on invocation type and set response + // to result. Should probably be handled externally by the runner/swapi. + + // OK, everything good. + concluded = true; + process.nextTick(function() { process.exit(failed ? 1 : 0) }) + } + + this.fail = function(error) { + if (concluded) { + return + } + + concluded = true + process.nextTick(function() { process.exit(1) }) + + if (error === undefined) { + error = null + } + + // FIXME(nikhil): Truncated log of error, plus non-truncated response body + var errstr = "fail() called with argument but a problem was encountered while converting it to a to string"; + + // The semantics of fail() are weird. If the error is something that can be + // converted to a string, the log output wraps the string in a JSON literal + // with key "errorMessage". If toString() fails, then the output is only + // the error string. + try { + if (error === null) { + errstr = null + } else { + errstr = error.toString() + } + oldlog(JSON.stringify({"errorMessage": errstr })) + } catch(e) { + // Set X-Amz-Function-Error: Unhandled header + oldlog(errstr) + } + } + + this.done = function() { + var error = arguments[0]; + var result = arguments[1]; + if (error) { + contextSelf.fail(error) + } else { + contextSelf.succeed(result) + } + } + + var plannedEnd = Date.now() + (getTimeoutInSeconds() * 1000); + this.getRemainingTimeInMillis = function() { + return Math.max(plannedEnd - Date.now(), 0); + } +} + +function getTimeoutInSeconds() { + var t = parseInt(getEnv("TASK_TIMEOUT")); + if (Number.isNaN(t)) { + return 3600; + } + + return t; +} + +var getEnv = function(name) { + return process.env[name] || ""; +} + +var makeCtx = function() { + var fnname = getEnv("AWS_LAMBDA_FUNCTION_NAME"); + // FIXME(nikhil): Generate UUID. + var taskID = getEnv("TASK_ID"); + + var mem = getEnv("TASK_MAXRAM").toLowerCase(); + var bytes = 300 * 1024 * 1024; + + var scale = { 'b': 1, 'k': 1024, 'm': 1024*1024, 'g': 1024*1024*1024 }; + // We don't bother validating too much, if the last character is not a number + // and not in the scale table we just return a default value. + // We use slice instead of indexing so that we always get an empty string, + // instead of undefined. + if (mem.slice(-1).match(/[0-9]/)) { + var a = parseInt(mem); + if (!Number.isNaN(a)) { + bytes = a; + } + } else { + var rem = parseInt(mem.slice(0, -1)); + if (!Number.isNaN(rem)) { + var multiplier = scale[mem.slice(-1)]; + if (multiplier) { + bytes = rem * multiplier + } + } + } + + var memoryMB = bytes / (1024 * 1024); + + var ctx = new Context(); + Object.defineProperties(ctx, { + "functionName": { + value: fnname, + enumerable: true, + }, + "functionVersion": { + value: "$LATEST", + enumerable: true, + }, + "invokedFunctionArn": { + // FIXME(nikhil): Should be filled in. + value: "", + enumerable: true, + }, + "memoryLimitInMB": { + // Sigh, yes it is a string. + value: ""+memoryMB, + enumerable: true, + }, + "awsRequestId": { + value: taskID, + enumerable: true, + }, + "logGroupName": { + // FIXME(nikhil): Should be filled in. + value: "", + enumerable: true, + }, + "logStreamName": { + // FIXME(nikhil): Should be filled in. + value: "", + enumerable: true, + }, + "identity": { + // FIXME(nikhil): Should be filled in. + value: null, + enumerable: true, + }, + "clientContext": { + // FIXME(nikhil): Should be filled in. + value: null, + enumerable: true, + }, + }); + + return ctx; +} + +var setEnvFromHeader = function () { + var headerPrefix = "CONFIG_"; + var newEnvVars = {}; + for (var key in process.env) { + if (key.indexOf(headerPrefix) == 0) { + newEnvVars[key.slice(headerPrefix.length)] = process.env[key]; + } + } + + for (var key in newEnvVars) { + process.env[key] = newEnvVars[key]; + } +} + + +function run() { + setEnvFromHeader(); + // FIXME(nikhil): Check for file existence and allow non-payload. + var path = process.env["PAYLOAD_FILE"]; + var stream = process.stdin; + if (path) { + try { + stream = fs.createReadStream(path); + } catch(e) { + console.error("bootstrap: Error opening payload file", e) + process.exit(1); + } + } + + var input = ""; + stream.setEncoding('utf8'); + stream.on('data', function(chunk) { + input += chunk; + }); + + stream.on('error', function(err) { + console.error("bootstrap: Error reading payload stream", err); + process.exit(1); + }); + + stream.on('end', function() { + var payload = {} + try { + if (input.length > 0) { + payload = JSON.parse(input); + } + } catch(e) { + console.error("bootstrap: Error parsing JSON", e); + process.exit(1); + } + + if (process.argv.length > 2) { + var handler = process.argv[2]; + var parts = handler.split('.'); + // FIXME(nikhil): Error checking. + var script = parts[0]; + var entry = parts[1]; + var started = false; + try { + var mod = require('./'+script); + var func = mod[entry]; + if (func === undefined) { + oldlog("Handler '" + entry + "' missing on module '" + script + "'"); + return; + } + + if (typeof func !== 'function') { + throw "TypeError: " + (typeof func) + " is not a function"; + } + started = true; + var cback + // RUN THE FUNCTION: + mod[entry](payload, makeCtx(), functionCallback) + } catch(e) { + if (typeof e === 'string') { + oldlog(e) + } else { + oldlog(e.message) + } + if (!started) { + oldlog("Process exited before completing request\n") + } + } + } else { + console.error("bootstrap: No script specified") + process.exit(1); + } + }) +} + +function functionCallback(err, result) { + if (err != null) { + // then user returned error and we should respond with error + // http://docs.aws.amazon.com/lambda/latest/dg/nodejs-prog-mode-exceptions.html + oldlog(JSON.stringify({"errorMessage": errstr })) + return + } + if (result != null) { + oldlog(JSON.stringify(result)) + } +} + +run() \ No newline at end of file diff --git a/fn/lambda/node-6/build.sh b/fn/lambda/node-6/build.sh new file mode 100755 index 000000000..b00dc260c --- /dev/null +++ b/fn/lambda/node-6/build.sh @@ -0,0 +1,3 @@ +set -ex + +docker build --build-arg HTTP_PROXY -t funcy/lambda:node-6 . diff --git a/fn/lambda/node-6/release.sh b/fn/lambda/node-6/release.sh new file mode 100755 index 000000000..db193d3a7 --- /dev/null +++ b/fn/lambda/node-6/release.sh @@ -0,0 +1,5 @@ +set -ex + +./build.sh + +docker push funcy/lambda:node-6 diff --git a/fn/lambda/node/build.sh b/fn/lambda/node/build.sh deleted file mode 100755 index e6c408e97..000000000 --- a/fn/lambda/node/build.sh +++ /dev/null @@ -1,3 +0,0 @@ -set -ex - -docker build --build-arg HTTP_PROXY -t treeder/functions-lambda:nodejs4.3 . diff --git a/fn/lambda/node/release.sh b/fn/lambda/node/release.sh deleted file mode 100755 index 02df72b1c..000000000 --- a/fn/lambda/node/release.sh +++ /dev/null @@ -1,5 +0,0 @@ -set -ex - -./build.sh -# TODO: where to push? -# docker push treeder/functions-lambda:nodejs4.3 diff --git a/fn/lambda/release.sh b/fn/lambda/release.sh index 93c63ea32..6a53fabce 100755 --- a/fn/lambda/release.sh +++ b/fn/lambda/release.sh @@ -1,5 +1,9 @@ set -ex -cd node +cd node-4 +./release.sh +cd .. + +cd node-6 ./release.sh cd .. diff --git a/fn/langs/base.go b/fn/langs/base.go index 11d0d20e3..1b5d66825 100644 --- a/fn/langs/base.go +++ b/fn/langs/base.go @@ -27,7 +27,7 @@ func GetLangHelper(lang string) LangHelper { return &RustLangHelper{} case "dotnet": return &DotNetLangHelper{} - case "lambda-nodejs4.3": + case "lambda-nodejs4.3", "lambda-node-4": return &LambdaNodeHelper{} case "java": return &JavaLangHelper{} @@ -69,10 +69,11 @@ func (h *BaseHelper) RunFromImage() string { return h.BuildFromImage() func (h *BaseHelper) IsMultiStage() bool { return true } func (h *BaseHelper) DockerfileBuildCmds() []string { return []string{} } func (h *BaseHelper) DockerfileCopyCmds() []string { return []string{} } +func (h *BaseHelper) Entrypoint() string { return "" } func (h *BaseHelper) Cmd() string { return "" } -func (lh *BaseHelper) HasPreBuild() bool { return false } -func (lh *BaseHelper) PreBuild() error { return nil } -func (lh *BaseHelper) AfterBuild() error { return nil } +func (h *BaseHelper) HasPreBuild() bool { return false } +func (h *BaseHelper) PreBuild() error { return nil } +func (h *BaseHelper) AfterBuild() error { return nil } func (h *BaseHelper) HasBoilerplate() bool { return false } func (h *BaseHelper) GenerateBoilerplate() error { return nil } diff --git a/fn/langs/lambda_node.go b/fn/langs/lambda_node.go index 74278145c..e9f33c145 100644 --- a/fn/langs/lambda_node.go +++ b/fn/langs/lambda_node.go @@ -5,26 +5,26 @@ type LambdaNodeHelper struct { } func (lh *LambdaNodeHelper) BuildFromImage() string { - return "funcy/functions-lambda:nodejs4.3" + return "funcy/lambda:node-4" } -func (lh *LambdaNodeHelper) Entrypoint() string { - return "" +func (lh *LambdaNodeHelper) IsMultiStage() bool { + return false } func (lh *LambdaNodeHelper) Cmd() string { return "func.handler" } -func (lh *LambdaNodeHelper) HasPreBuild() bool { - return false -} - -// PreBuild for Go builds the binary so the final image can be as small as possible -func (lh *LambdaNodeHelper) PreBuild() error { - return nil -} - -func (lh *LambdaNodeHelper) AfterBuild() error { - return nil +func (h *LambdaNodeHelper) DockerfileBuildCmds() []string { + r := []string{} + if exists("package.json") { + r = append(r, + "ADD package.json /function/", + "RUN npm install", + ) + } + // single stage build for this one, so add files + r = append(r, "ADD . /function/") + return r }