Moving dep builds into Dockerfile and multi-stage builds.

This commit is contained in:
Travis Reeder
2017-06-06 15:02:08 -07:00
committed by Denis Makogon
parent 0276a1c156
commit f628e39490
17 changed files with 268 additions and 226 deletions

View File

@@ -2,6 +2,6 @@
"name": "my-awesome-func",
"version": "1.0.0",
"dependencies": {
"request": "^2.78.0"
"is-positive": "^3.1.0"
}
}

View File

@@ -1,5 +1,4 @@
import sys
sys.path.append("packages")
import os
import json
@@ -9,9 +8,13 @@ name = "World"
try:
if not os.isatty(sys.stdin.fileno()):
obj = json.loads(sys.stdin.read())
if obj["name"] != "":
name = obj["name"]
try:
obj = json.loads(sys.stdin.read())
if obj["name"] != "":
name = obj["name"]
except ValueError:
# ignore it
sys.stderr.write("no input, but that's ok\n")
except:
pass

View File

@@ -0,0 +1 @@
hello_pip==1.0.4

View File

@@ -1,4 +1,4 @@
require_relative 'bundle/bundler/setup'
# require_relative 'bundle/bundler/setup'
require 'json'
name = "World"

10
fn/Gopkg.lock generated
View File

@@ -1,4 +1,5 @@
memo = "f23a5e941b4b53ce3100a43c698bbc29b85f8bd3257d17e699b238db2f1ebf1e"
# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
[[projects]]
branch = "master"
@@ -209,3 +210,10 @@ memo = "f23a5e941b4b53ce3100a43c698bbc29b85f8bd3257d17e699b238db2f1ebf1e"
name = "gopkg.in/yaml.v2"
packages = ["."]
revision = "cd8b52f8269e0feb286dfeef29f8fe4d5b397e0b"
[solve-meta]
analyzer-name = "dep"
analyzer-version = 1
inputs-digest = "dd217cf3430519442b9eca700adbe81e4ac9abeed9174f9fd9ae3089f18c332a"
solver-name = "gps-cdcl"
solver-version = 1

View File

@@ -1,6 +1,7 @@
package main
import (
"bufio"
"bytes"
"errors"
"fmt"
@@ -9,13 +10,15 @@ import (
"os/exec"
"path/filepath"
"strings"
"text/template"
"github.com/coreos/go-semver/semver"
"gitlab-odx.oracle.com/odx/functions/fn/langs"
)
const (
functionsDockerImage = "treeder/functions"
functionsDockerImage = "treeder/functions"
minRequiredDockerVersion = "17.5.0"
)
func verbwriter(verbose bool) io.Writer {
@@ -73,20 +76,25 @@ func localbuild(verbwriter io.Writer, path string, steps []string) error {
}
func dockerbuild(verbwriter io.Writer, path string, ff *funcfile) error {
err := dockerVersionCheck()
if err != nil {
return err
}
dir := filepath.Dir(path)
var helper langs.LangHelper
dockerfile := filepath.Join(dir, "Dockerfile")
if !exists(dockerfile) {
err := writeTmpDockerfile(dir, ff)
defer os.Remove(filepath.Join(dir, "Dockerfile"))
if err != nil {
return err
}
helper = langs.GetLangHelper(*ff.Runtime)
if helper == nil {
return fmt.Errorf("Cannot build, no language helper found for %v", *ff.Runtime)
}
err := writeTmpDockerfile(helper, dir, ff)
defer os.Remove(filepath.Join(dir, "Dockerfile"))
if err != nil {
return err
}
if helper.HasPreBuild() {
err := helper.PreBuild()
if err != nil {
@@ -98,7 +106,9 @@ func dockerbuild(verbwriter io.Writer, path string, ff *funcfile) error {
fmt.Printf("Building image %v\n", ff.FullName())
cmd := exec.Command("docker", "build",
"-t", ff.FullName(),
// "--no-cache",
"--build-arg", "HTTP_PROXY",
"--build-arg", "HTTPS_PROXY",
".")
cmd.Dir = dir
cmd.Stderr = os.Stderr
@@ -115,6 +125,25 @@ func dockerbuild(verbwriter io.Writer, path string, ff *funcfile) error {
return nil
}
func dockerVersionCheck() error {
out, err := exec.Command("docker", "version", "--format", "{{.Server.Version}}").Output()
if err != nil {
return fmt.Errorf("could not check Docker version: %v", err)
}
v, err := semver.NewVersion(string(out))
if err != nil {
return fmt.Errorf("could not check Docker version: %v", err)
}
vMin, err := semver.NewVersion(minRequiredDockerVersion)
if err != nil {
return fmt.Errorf("our bad, sorry... please make an issue.", err)
}
if v.LessThan(*vMin) {
return fmt.Errorf("please upgrade your version of Docker to %s or greater", minRequiredDockerVersion)
}
return nil
}
func exists(name string) bool {
if _, err := os.Stat(name); err != nil {
if os.IsNotExist(err) {
@@ -124,66 +153,59 @@ func exists(name string) bool {
return true
}
var acceptableFnRuntimes = map[string]string{
"elixir": "funcy/elixir",
"erlang": "funcy/erlang",
"gcc": "funcy/gcc",
"go": "funcy/go",
"java": "funcy/java",
"leiningen": "funcy/leiningen",
"mono": "funcy/mono",
"node": "funcy/node",
"perl": "funcy/perl",
"php": "funcy/php",
"python": "funcy/python:2",
"ruby": "funcy/ruby",
"scala": "funcy/scala",
"rust": "corey/rust-alpine",
"dotnet": "microsoft/dotnet:runtime",
"lambda-nodejs4.3": "funcy/functions-lambda:nodejs4.3",
}
const tplDockerfile = `FROM {{ .BaseImage }}
WORKDIR /function
ADD . /function/
{{ if ne .Entrypoint "" }} ENTRYPOINT [{{ .Entrypoint }}] {{ end }}
{{ if ne .Cmd "" }} CMD [{{ .Cmd }}] {{ end }}
`
func writeTmpDockerfile(dir string, ff *funcfile) error {
func writeTmpDockerfile(helper langs.LangHelper, dir string, ff *funcfile) error {
if ff.Entrypoint == "" && ff.Cmd == "" {
return errors.New("entrypoint and cmd are missing, you must provide one or the other")
}
runtime, tag := ff.RuntimeTag()
rt, ok := acceptableFnRuntimes[runtime]
if !ok {
return fmt.Errorf("cannot use runtime %s", runtime)
}
if tag != "" {
rt = fmt.Sprintf("%s:%s", rt, tag)
}
fd, err := os.Create(filepath.Join(dir, "Dockerfile"))
if err != nil {
return err
}
defer fd.Close()
// convert entrypoint string to slice
bufferEp := stringToSlice(ff.Entrypoint)
bufferCmd := stringToSlice(ff.Cmd)
t := template.Must(template.New("Dockerfile").Parse(tplDockerfile))
err = t.Execute(fd, struct {
BaseImage, Entrypoint, Cmd string
}{rt, bufferEp.String(), bufferCmd.String()})
// multi-stage build: https://medium.com/travis-on-docker/multi-stage-docker-builds-for-creating-tiny-go-images-e0e1867efe5a
dfLines := []string{}
if helper.IsMultiStage() {
// build stage
dfLines = append(dfLines, fmt.Sprintf("FROM %s as build-stage", helper.BuildFromImage()))
} else {
dfLines = append(dfLines, fmt.Sprintf("FROM %s", helper.BuildFromImage()))
}
dfLines = append(dfLines, "WORKDIR /function")
dfLines = append(dfLines, helper.DockerfileBuildCmds()...)
if helper.IsMultiStage() {
// final stage
dfLines = append(dfLines, fmt.Sprintf("FROM %s", helper.RunFromImage()))
dfLines = append(dfLines, "WORKDIR /function")
dfLines = append(dfLines, helper.DockerfileCopyCmds()...)
}
if ff.Entrypoint != "" {
dfLines = append(dfLines, fmt.Sprintf("ENTRYPOINT [%s]", stringToSlice(ff.Entrypoint)))
}
if ff.Cmd != "" {
dfLines = append(dfLines, fmt.Sprintf("CMD [%s]", stringToSlice(ff.Cmd)))
}
err = writeLines(fd, dfLines)
if err != nil {
return err
}
return err
}
func stringToSlice(in string) bytes.Buffer {
func writeLines(w io.Writer, lines []string) error {
writer := bufio.NewWriter(w)
for _, l := range lines {
_, err := writer.WriteString(l + "\n")
if err != nil {
return err
}
}
writer.Flush()
return nil
}
func stringToSlice(in string) string {
epvals := strings.Fields(in)
var buffer bytes.Buffer
for i, s := range epvals {
@@ -194,7 +216,7 @@ func stringToSlice(in string) bytes.Buffer {
buffer.WriteString(s)
buffer.WriteString("\"")
}
return buffer
return buffer.String()
}
func extractEnvConfig(configs []string) map[string]string {

View File

@@ -186,13 +186,9 @@ func (a *initFnCmd) buildFuncFile(c *cli.Context) error {
} else {
fmt.Println("Runtime:", a.runtime)
}
if _, ok := acceptableFnRuntimes[a.runtime]; !ok {
return fmt.Errorf("init does not support the %s runtime, you'll have to create your own Dockerfile for this function", a.runtime)
}
helper := langs.GetLangHelper(a.runtime)
if helper == nil {
fmt.Printf("No helper found for %s runtime, you'll have to pass in the appropriate flags or use a Dockerfile.", a.runtime)
fmt.Printf("init does not support the %s runtime, you'll have to create your own Dockerfile for this function", a.runtime)
}
if a.entrypoint == "" {

View File

@@ -36,6 +36,16 @@ func GetLangHelper(lang string) LangHelper {
}
type LangHelper interface {
// BuildFromImage is the base image to build off, typically funcy/LANG:dev
BuildFromImage() string
// RunFromImage is the base image to use for deployment (usually smaller than the build images)
RunFromImage() string
// If set to false, it will use a single Docker build step, rather than multi-stage
IsMultiStage() bool
// Dockerfile build lines for building dependencies or anything else language specific
DockerfileBuildCmds() []string
// DockerfileCopyCmds will run in second/final stage of multi-stage build to copy artifacts form the build stage
DockerfileCopyCmds() []string
// Entrypoint sets the Docker Entrypoint. One of Entrypoint or Cmd is required.
Entrypoint() string
// Cmd sets the Docker command. One of Entrypoint or Cmd is required.
@@ -54,10 +64,19 @@ type LangHelper interface {
type BaseHelper struct {
}
func (h *BaseHelper) Cmd() string { return "" }
func (h *BaseHelper) HasBoilerplate() bool { return false }
func (h *BaseHelper) GenerateBoilerplate() error { return nil }
func (h *BaseHelper) BuildFromImage() string { return "" }
func (h *BaseHelper) RunFromImage() string { return h.BuildFromImage() }
func (h *BaseHelper) IsMultiStage() bool { return true }
func (h *BaseHelper) DockerfileBuildCmds() []string { return []string{} }
func (h *BaseHelper) DockerfileCopyCmds() []string { return []string{} }
func (h *BaseHelper) Cmd() string { return "" }
func (lh *BaseHelper) HasPreBuild() bool { return false }
func (lh *BaseHelper) PreBuild() error { return nil }
func (lh *BaseHelper) AfterBuild() error { return nil }
func (h *BaseHelper) HasBoilerplate() bool { return false }
func (h *BaseHelper) GenerateBoilerplate() error { return nil }
// exists checks if a file exists
func exists(name string) bool {
if _, err := os.Stat(name); err != nil {
if os.IsNotExist(err) {

View File

@@ -9,6 +9,13 @@ type DotNetLangHelper struct {
BaseHelper
}
func (lh *DotNetLangHelper) BuildFromImage() string {
return "microsoft/dotnet:1.0.1-sdk-projectjson"
}
func (lh *DotNetLangHelper) RunFromImage() string {
return "microsoft/dotnet:runtime"
}
func (lh *DotNetLangHelper) Entrypoint() string {
return "dotnet dotnet.dll"
}

View File

@@ -1,47 +1,40 @@
package langs
import (
"fmt"
"os"
"os/exec"
"strings"
)
type GoLangHelper struct {
BaseHelper
}
func (lh *GoLangHelper) BuildFromImage() string {
return "funcy/go:dev"
}
func (lh *GoLangHelper) RunFromImage() string {
return "funcy/go"
}
func (h *GoLangHelper) DockerfileBuildCmds() []string {
r := []string{}
// more info on Go multi-stage builds: https://medium.com/travis-on-docker/multi-stage-docker-builds-for-creating-tiny-go-images-e0e1867efe5a
// For now we assume that dependencies are vendored already, but we could vendor them
// inside the container. Maybe we should check for /vendor dir and if it doesn't exist,
// either run `dep init` if no Gopkg.toml/lock found or `dep ensure` if it's there.
r = append(r, "ADD . /src")
// if exists("Gopkg.toml") {
// r = append(r,
// "RUN go get -u github.com/golang/dep/cmd/dep",
// "RUN cd /src && dep ensure",
// )
// }
r = append(r, "RUN cd /src && go build -o func")
return r
}
func (h *GoLangHelper) DockerfileCopyCmds() []string {
return []string{
"COPY --from=build-stage /src/func /function/",
}
}
func (lh *GoLangHelper) Entrypoint() string {
return "./func"
}
func (lh *GoLangHelper) HasPreBuild() bool {
return true
}
// PreBuild for Go builds the binary so the final image can be as small as possible
func (lh *GoLangHelper) PreBuild() error {
wd, err := os.Getwd()
if err != nil {
return err
}
// todo: this won't work if the function is more complex since the import paths won't match up, need to fix
pbcmd := fmt.Sprintf("docker run --rm -v %s:/go/src/github.com/x/y -w /go/src/github.com/x/y funcy/go:dev go build -o func", wd)
fmt.Println("Running prebuild command:", pbcmd)
parts := strings.Fields(pbcmd)
head := parts[0]
parts = parts[1:len(parts)]
cmd := exec.Command(head, parts...)
// cmd.Dir = dir
cmd.Stderr = os.Stderr
cmd.Stdout = os.Stdout
if err := cmd.Run(); err != nil {
return dockerBuildError(err)
}
return nil
}
func (lh *GoLangHelper) AfterBuild() error {
return os.Remove("func")
}

View File

@@ -4,7 +4,6 @@ import (
"fmt"
"io/ioutil"
"os"
"os/exec"
"path/filepath"
)
@@ -18,6 +17,31 @@ const (
mainClassFile = mainClass + ".java"
)
// BuildFromImage returns the Docker image used to compile the Java function
func (lh *JavaLangHelper) BuildFromImage() string {
return "funcy/java:dev"
}
// RunFromImage returns the Docker image used to run the Java function
func (lh *JavaLangHelper) RunFromImage() string {
return "funcy/java"
}
// DockerfileBuildCmds returns the build stage steps to compile the Java function
func (lh *JavaLangHelper) DockerfileBuildCmds() []string {
return []string{
fmt.Sprintf("ADD %s . /src/", mainClassFile),
fmt.Sprintf("RUN cd /src && javac %s", mainClassFile),
}
}
// DockerfileCopyCmds returns the Docker COPY command to copy the compiled Java function classes
func (h *JavaLangHelper) DockerfileCopyCmds() []string {
return []string{
"COPY --from=build-stage /src/ /function/",
}
}
// Entrypoint returns the Java runtime Docker entrypoint that will be executed when the function is run
func (lh *JavaLangHelper) Entrypoint() string {
return fmt.Sprintf("java %s", mainClass)
@@ -28,9 +52,8 @@ func (lh *JavaLangHelper) HasPreBuild() bool {
return true
}
// PreBuild executes the pre-build step for the Java runtime which involves compiling the relevant classes. It expects
// the entrypoint to the function, in other words or the class with the main method (not to be confused with the Docker
// entrypoint from Entrypoint()) to be Function.java
// PreBuild ensures that the expected Java source file is there before the build is executed. Returns an error if
// `mainClassFile` is not in the working directory
func (lh *JavaLangHelper) PreBuild() error {
wd, err := os.Getwd()
if err != nil {
@@ -42,40 +65,6 @@ func (lh *JavaLangHelper) PreBuild() error {
"called %s", mainClassFile)
}
cmd := exec.Command(
"docker", "run",
"--rm", "-v", wd+":/java", "-w", "/java",
"funcy/java:dev",
"/bin/sh", "-c", "javac "+mainClassFile,
)
cmd.Stderr = os.Stderr
cmd.Stdout = os.Stdout
if err := cmd.Run(); err != nil {
return dockerBuildError(err)
}
return nil
}
// AfterBuild removes all compiled class files from the host machine
func (lh *JavaLangHelper) AfterBuild() error {
wd, err := os.Getwd()
if err != nil {
return err
}
files, err := filepath.Glob(filepath.Join(wd, "*.class"))
if err != nil {
return err
}
for _, file := range files {
err = os.Remove(file)
if err != nil {
return err
}
}
return nil
}

View File

@@ -4,6 +4,10 @@ type LambdaNodeHelper struct {
BaseHelper
}
func (lh *LambdaNodeHelper) BuildFromImage() string {
return "funcy/functions-lambda:nodejs4.3"
}
func (lh *LambdaNodeHelper) Entrypoint() string {
return ""
}

View File

@@ -4,19 +4,31 @@ type NodeLangHelper struct {
BaseHelper
}
func (lh *NodeLangHelper) BuildFromImage() string {
return "funcy/node:dev"
}
func (lh *NodeLangHelper) RunFromImage() string {
return "funcy/node"
}
func (lh *NodeLangHelper) Entrypoint() string {
return "node func.js"
}
func (lh *NodeLangHelper) HasPreBuild() bool {
return false
func (h *NodeLangHelper) DockerfileBuildCmds() []string {
r := []string{}
if exists("package.json") {
r = append(r,
"ADD package.json /function/",
"RUN npm install",
)
}
return r
}
// PreBuild for Go builds the binary so the final image can be as small as possible
func (lh *NodeLangHelper) PreBuild() error {
return nil
}
func (lh *NodeLangHelper) AfterBuild() error {
return nil
func (h *NodeLangHelper) DockerfileCopyCmds() []string {
return []string{
"ADD . /function/",
"COPY --from=build-stage /function/node_modules/ /function/node_modules/",
}
}

View File

@@ -12,6 +12,9 @@ type PhpLangHelper struct {
BaseHelper
}
func (lh *PhpLangHelper) BuildFromImage() string {
return "funcy/php:dev"
}
func (lh *PhpLangHelper) Entrypoint() string {
return "php func.php"
}

View File

@@ -1,45 +1,41 @@
package langs
import (
"fmt"
"os"
"os/exec"
"strings"
)
type PythonLangHelper struct {
BaseHelper
}
func (lh *PythonLangHelper) BuildFromImage() string {
return "funcy/python:2-dev"
}
func (lh *PythonLangHelper) RunFromImage() string {
return "funcy/python:2-dev"
}
func (lh *PythonLangHelper) Entrypoint() string {
return "python2 func.py"
}
func (lh *PythonLangHelper) HasPreBuild() bool {
return true
}
// PreBuild for Go builds the binary so the final image can be as small as possible
func (lh *PythonLangHelper) PreBuild() error {
wd, err := os.Getwd()
if err != nil {
return err
func (h *PythonLangHelper) DockerfileBuildCmds() []string {
r := []string{}
if exists("requirements.txt") {
r = append(r,
"ADD requirements.txt /function/",
"RUN pip install -r requirements.txt",
"ADD . /function/",
)
}
pbcmd := fmt.Sprintf("docker run --rm -v %s:/worker -w /worker funcy/python:2-dev pip install -t packages -r requirements.txt", wd)
fmt.Println("Running prebuild command:", pbcmd)
parts := strings.Fields(pbcmd)
head := parts[0]
parts = parts[1:len(parts)]
cmd := exec.Command(head, parts...)
cmd.Stderr = os.Stderr
cmd.Stdout = os.Stdout
if err := cmd.Run(); err != nil {
return dockerBuildError(err)
}
return nil
return r
}
func (lh *PythonLangHelper) AfterBuild() error {
return nil
func (h *PythonLangHelper) IsMultiStage() bool {
return false
}
// The multi-stage build didn't work, pip seems to be required for it to load the modules
// func (h *PythonLangHelper) DockerfileCopyCmds() []string {
// return []string{
// "ADD . /function/",
// "COPY --from=build-stage /root/.cache/pip/ /root/.cache/pip/",
// }
// }

View File

@@ -1,49 +1,35 @@
package langs
import (
"fmt"
"os"
"os/exec"
"path/filepath"
"strings"
)
type RubyLangHelper struct {
BaseHelper
}
func (lh *RubyLangHelper) BuildFromImage() string {
return "funcy/ruby:dev"
}
func (lh *RubyLangHelper) RunFromImage() string {
return "funcy/ruby"
}
func (h *RubyLangHelper) DockerfileBuildCmds() []string {
r := []string{}
if exists("Gemfile") {
r = append(r,
"ADD Gemfile* /function/",
"RUN bundle install",
)
}
return r
}
func (h *RubyLangHelper) DockerfileCopyCmds() []string {
return []string{
"COPY --from=build-stage /usr/lib/ruby/gems/ /usr/lib/ruby/gems/", // skip this if no Gemfile? Does it matter?
"ADD . /function/",
}
}
func (lh *RubyLangHelper) Entrypoint() string {
return "ruby func.rb"
}
func (lh *RubyLangHelper) HasPreBuild() bool {
return true
}
func (lh *RubyLangHelper) PreBuild() error {
wd, err := os.Getwd()
if err != nil {
return err
}
if !exists(filepath.Join(wd, "Gemfile")) {
return nil
}
pbcmd := fmt.Sprintf("docker run --rm -v %s:/worker -w /worker funcy/ruby:dev bundle install --standalone --clean", wd)
fmt.Println("Running prebuild command:", pbcmd)
parts := strings.Fields(pbcmd)
head := parts[0]
parts = parts[1:len(parts)]
cmd := exec.Command(head, parts...)
cmd.Stderr = os.Stderr
cmd.Stdout = os.Stdout
if err := cmd.Run(); err != nil {
return dockerBuildError(err)
}
return nil
}
func (lh *RubyLangHelper) AfterBuild() error {
return nil
}

View File

@@ -9,6 +9,9 @@ type RustLangHelper struct {
BaseHelper
}
func (lh *RustLangHelper) BuildFromImage() string {
return "funcy/rust:dev"
}
func (lh *RustLangHelper) Entrypoint() string {
return "/function/target/release/func"
}