Moving dep builds into Dockerfile and multi-stage builds.

fn/langs/base.go

@@ -36,6 +36,16 @@ func GetLangHelper(lang string) LangHelper {
 }
 
 type LangHelper interface {
+	// BuildFromImage is the base image to build off, typically funcy/LANG:dev
+	BuildFromImage() string
+	// RunFromImage is the base image to use for deployment (usually smaller than the build images)
+	RunFromImage() string
+	// If set to false, it will use a single Docker build step, rather than multi-stage
+	IsMultiStage() bool
+	// Dockerfile build lines for building dependencies or anything else language specific
+	DockerfileBuildCmds() []string
+	// DockerfileCopyCmds will run in second/final stage of multi-stage build to copy artifacts form the build stage
+	DockerfileCopyCmds() []string
 	// Entrypoint sets the Docker Entrypoint. One of Entrypoint or Cmd is required.
 	Entrypoint() string
 	// Cmd sets the Docker command. One of Entrypoint or Cmd is required.
@@ -54,10 +64,19 @@ type LangHelper interface {
 type BaseHelper struct {
 }
 
-func (h *BaseHelper) Cmd() string                { return "" }
-func (h *BaseHelper) HasBoilerplate() bool       { return false }
-func (h *BaseHelper) GenerateBoilerplate() error { return nil }
+func (h *BaseHelper) BuildFromImage() string        { return "" }
+func (h *BaseHelper) RunFromImage() string          { return h.BuildFromImage() }
+func (h *BaseHelper) IsMultiStage() bool            { return true }
+func (h *BaseHelper) DockerfileBuildCmds() []string { return []string{} }
+func (h *BaseHelper) DockerfileCopyCmds() []string  { return []string{} }
+func (h *BaseHelper) Cmd() string                   { return "" }
+func (lh *BaseHelper) HasPreBuild() bool            { return false }
+func (lh *BaseHelper) PreBuild() error              { return nil }
+func (lh *BaseHelper) AfterBuild() error            { return nil }
+func (h *BaseHelper) HasBoilerplate() bool          { return false }
+func (h *BaseHelper) GenerateBoilerplate() error    { return nil }
 
+// exists checks if a file exists
 func exists(name string) bool {
 	if _, err := os.Stat(name); err != nil {
 		if os.IsNotExist(err) {

fn/langs/dotnet.go

@@ -9,6 +9,13 @@ type DotNetLangHelper struct {
 	BaseHelper
 }
 
+func (lh *DotNetLangHelper) BuildFromImage() string {
+	return "microsoft/dotnet:1.0.1-sdk-projectjson"
+}
+func (lh *DotNetLangHelper) RunFromImage() string {
+	return "microsoft/dotnet:runtime"
+}
+
 func (lh *DotNetLangHelper) Entrypoint() string {
 	return "dotnet dotnet.dll"
 }

fn/langs/go.go

@@ -1,47 +1,40 @@
 package langs
 
-import (
-	"fmt"
-	"os"
-	"os/exec"
-	"strings"
-)
-
 type GoLangHelper struct {
 	BaseHelper
 }
 
+func (lh *GoLangHelper) BuildFromImage() string {
+	return "funcy/go:dev"
+}
+
+func (lh *GoLangHelper) RunFromImage() string {
+	return "funcy/go"
+}
+
+func (h *GoLangHelper) DockerfileBuildCmds() []string {
+	r := []string{}
+	// more info on Go multi-stage builds: https://medium.com/travis-on-docker/multi-stage-docker-builds-for-creating-tiny-go-images-e0e1867efe5a
+	// For now we assume that dependencies are vendored already, but we could vendor them
+	// inside the container. Maybe we should check for /vendor dir and if it doesn't exist,
+	// either run `dep init` if no Gopkg.toml/lock found or `dep ensure` if it's there.
+	r = append(r, "ADD . /src")
+	// if exists("Gopkg.toml") {
+	// r = append(r,
+	// 	"RUN go get -u github.com/golang/dep/cmd/dep",
+	// 	"RUN cd /src && dep ensure",
+	// )
+	// }
+	r = append(r, "RUN cd /src && go build -o func")
+	return r
+}
+
+func (h *GoLangHelper) DockerfileCopyCmds() []string {
+	return []string{
+		"COPY --from=build-stage /src/func /function/",
+	}
+}
+
 func (lh *GoLangHelper) Entrypoint() string {
 	return "./func"
 }
-
-func (lh *GoLangHelper) HasPreBuild() bool {
-	return true
-}
-
-// PreBuild for Go builds the binary so the final image can be as small as possible
-func (lh *GoLangHelper) PreBuild() error {
-	wd, err := os.Getwd()
-	if err != nil {
-		return err
-	}
-	// todo: this won't work if the function is more complex since the import paths won't match up, need to fix
-	pbcmd := fmt.Sprintf("docker run --rm -v %s:/go/src/github.com/x/y -w /go/src/github.com/x/y funcy/go:dev go build -o func", wd)
-	fmt.Println("Running prebuild command:", pbcmd)
-	parts := strings.Fields(pbcmd)
-	head := parts[0]
-	parts = parts[1:len(parts)]
-	cmd := exec.Command(head, parts...)
-	// cmd.Dir = dir
-	cmd.Stderr = os.Stderr
-	cmd.Stdout = os.Stdout
-	if err := cmd.Run(); err != nil {
-		return dockerBuildError(err)
-	}
-	return nil
-}
-
-func (lh *GoLangHelper) AfterBuild() error {
-	return os.Remove("func")
-
-}

fn/langs/java.go

@@ -4,7 +4,6 @@ import (
 	"fmt"
 	"io/ioutil"
 	"os"
-	"os/exec"
 	"path/filepath"
 )
 
@@ -18,6 +17,31 @@ const (
 	mainClassFile = mainClass + ".java"
 )
 
+// BuildFromImage returns the Docker image used to compile the Java function
+func (lh *JavaLangHelper) BuildFromImage() string {
+	return "funcy/java:dev"
+}
+
+// RunFromImage returns the Docker image used to run the Java function
+func (lh *JavaLangHelper) RunFromImage() string {
+	return "funcy/java"
+}
+
+// DockerfileBuildCmds returns the build stage steps to compile the Java function
+func (lh *JavaLangHelper) DockerfileBuildCmds() []string {
+	return []string{
+		fmt.Sprintf("ADD %s . /src/", mainClassFile),
+		fmt.Sprintf("RUN cd /src && javac %s", mainClassFile),
+	}
+}
+
+// DockerfileCopyCmds returns the Docker COPY command to copy the compiled Java function classes
+func (h *JavaLangHelper) DockerfileCopyCmds() []string {
+	return []string{
+		"COPY --from=build-stage /src/ /function/",
+	}
+}
+
 // Entrypoint returns the Java runtime Docker entrypoint that will be executed when the function is run
 func (lh *JavaLangHelper) Entrypoint() string {
 	return fmt.Sprintf("java %s", mainClass)
@@ -28,9 +52,8 @@ func (lh *JavaLangHelper) HasPreBuild() bool {
 	return true
 }
 
-// PreBuild executes the pre-build step for the Java runtime which involves compiling the relevant classes. It expects
-// the entrypoint to the function, in other words or the class with the main method (not to be confused with the Docker
-// entrypoint from Entrypoint()) to be Function.java
+// PreBuild ensures that the expected Java source file is there before the build is executed. Returns an error if
+// `mainClassFile` is not in the working directory
 func (lh *JavaLangHelper) PreBuild() error {
 	wd, err := os.Getwd()
 	if err != nil {
@@ -42,40 +65,6 @@ func (lh *JavaLangHelper) PreBuild() error {
 			"called %s", mainClassFile)
 	}
 
-	cmd := exec.Command(
-		"docker", "run",
-		"--rm", "-v", wd+":/java", "-w", "/java",
-		"funcy/java:dev",
-		"/bin/sh", "-c", "javac "+mainClassFile,
-	)
-
-	cmd.Stderr = os.Stderr
-	cmd.Stdout = os.Stdout
-	if err := cmd.Run(); err != nil {
-		return dockerBuildError(err)
-	}
-	return nil
-}
-
-// AfterBuild removes all compiled class files from the host machine
-func (lh *JavaLangHelper) AfterBuild() error {
-	wd, err := os.Getwd()
-	if err != nil {
-		return err
-	}
-
-	files, err := filepath.Glob(filepath.Join(wd, "*.class"))
-	if err != nil {
-		return err
-	}
-
-	for _, file := range files {
-		err = os.Remove(file)
-		if err != nil {
-			return err
-		}
-	}
-
 	return nil
 }
 

fn/langs/lambda_node.go

@@ -4,6 +4,10 @@ type LambdaNodeHelper struct {
 	BaseHelper
 }
 
+func (lh *LambdaNodeHelper) BuildFromImage() string {
+	return "funcy/functions-lambda:nodejs4.3"
+}
+
 func (lh *LambdaNodeHelper) Entrypoint() string {
 	return ""
 }

fn/langs/node.go

@@ -4,19 +4,31 @@ type NodeLangHelper struct {
 	BaseHelper
 }
 
+func (lh *NodeLangHelper) BuildFromImage() string {
+	return "funcy/node:dev"
+}
+func (lh *NodeLangHelper) RunFromImage() string {
+	return "funcy/node"
+}
+
 func (lh *NodeLangHelper) Entrypoint() string {
 	return "node func.js"
 }
 
-func (lh *NodeLangHelper) HasPreBuild() bool {
-	return false
+func (h *NodeLangHelper) DockerfileBuildCmds() []string {
+	r := []string{}
+	if exists("package.json") {
+		r = append(r,
+			"ADD package.json /function/",
+			"RUN npm install",
+		)
+	}
+	return r
 }
 
-// PreBuild for Go builds the binary so the final image can be as small as possible
-func (lh *NodeLangHelper) PreBuild() error {
-	return nil
-}
-
-func (lh *NodeLangHelper) AfterBuild() error {
-	return nil
+func (h *NodeLangHelper) DockerfileCopyCmds() []string {
+	return []string{
+		"ADD . /function/",
+		"COPY --from=build-stage /function/node_modules/ /function/node_modules/",
+	}
 }

fn/langs/php.go

@@ -12,6 +12,9 @@ type PhpLangHelper struct {
 	BaseHelper
 }
 
+func (lh *PhpLangHelper) BuildFromImage() string {
+	return "funcy/php:dev"
+}
 func (lh *PhpLangHelper) Entrypoint() string {
 	return "php func.php"
 }

fn/langs/python.go

@@ -1,45 +1,41 @@
 package langs
 
-import (
-	"fmt"
-	"os"
-	"os/exec"
-	"strings"
-)
-
 type PythonLangHelper struct {
 	BaseHelper
 }
 
+func (lh *PythonLangHelper) BuildFromImage() string {
+	return "funcy/python:2-dev"
+}
+
+func (lh *PythonLangHelper) RunFromImage() string {
+	return "funcy/python:2-dev"
+}
+
 func (lh *PythonLangHelper) Entrypoint() string {
 	return "python2 func.py"
 }
 
-func (lh *PythonLangHelper) HasPreBuild() bool {
-	return true
-}
-
-// PreBuild for Go builds the binary so the final image can be as small as possible
-func (lh *PythonLangHelper) PreBuild() error {
-	wd, err := os.Getwd()
-	if err != nil {
-		return err
+func (h *PythonLangHelper) DockerfileBuildCmds() []string {
+	r := []string{}
+	if exists("requirements.txt") {
+		r = append(r,
+			"ADD requirements.txt /function/",
+			"RUN pip install -r requirements.txt",
+			"ADD . /function/",
+		)
 	}
-
-	pbcmd := fmt.Sprintf("docker run --rm -v %s:/worker -w /worker funcy/python:2-dev pip install -t packages -r requirements.txt", wd)
-	fmt.Println("Running prebuild command:", pbcmd)
-	parts := strings.Fields(pbcmd)
-	head := parts[0]
-	parts = parts[1:len(parts)]
-	cmd := exec.Command(head, parts...)
-	cmd.Stderr = os.Stderr
-	cmd.Stdout = os.Stdout
-	if err := cmd.Run(); err != nil {
-		return dockerBuildError(err)
-	}
-	return nil
+	return r
 }
 
-func (lh *PythonLangHelper) AfterBuild() error {
-	return nil
+func (h *PythonLangHelper) IsMultiStage() bool {
+	return false
 }
+
+// The multi-stage build didn't work, pip seems to be required for it to load the modules
+// func (h *PythonLangHelper) DockerfileCopyCmds() []string {
+// return []string{
+// "ADD . /function/",
+// "COPY --from=build-stage /root/.cache/pip/ /root/.cache/pip/",
+// }
+// }

fn/langs/ruby.go

@@ -1,49 +1,35 @@
 package langs
 
-import (
-	"fmt"
-	"os"
-	"os/exec"
-	"path/filepath"
-	"strings"
-)
-
 type RubyLangHelper struct {
 	BaseHelper
 }
 
+func (lh *RubyLangHelper) BuildFromImage() string {
+	return "funcy/ruby:dev"
+}
+
+func (lh *RubyLangHelper) RunFromImage() string {
+	return "funcy/ruby"
+}
+
+func (h *RubyLangHelper) DockerfileBuildCmds() []string {
+	r := []string{}
+	if exists("Gemfile") {
+		r = append(r,
+			"ADD Gemfile* /function/",
+			"RUN bundle install",
+		)
+	}
+	return r
+}
+
+func (h *RubyLangHelper) DockerfileCopyCmds() []string {
+	return []string{
+		"COPY --from=build-stage /usr/lib/ruby/gems/ /usr/lib/ruby/gems/", // skip this if no Gemfile?  Does it matter?
+		"ADD . /function/",
+	}
+}
+
 func (lh *RubyLangHelper) Entrypoint() string {
 	return "ruby func.rb"
 }
-
-func (lh *RubyLangHelper) HasPreBuild() bool {
-	return true
-}
-
-func (lh *RubyLangHelper) PreBuild() error {
-	wd, err := os.Getwd()
-	if err != nil {
-		return err
-	}
-
-	if !exists(filepath.Join(wd, "Gemfile")) {
-		return nil
-	}
-
-	pbcmd := fmt.Sprintf("docker run --rm -v %s:/worker -w /worker funcy/ruby:dev bundle install --standalone --clean", wd)
-	fmt.Println("Running prebuild command:", pbcmd)
-	parts := strings.Fields(pbcmd)
-	head := parts[0]
-	parts = parts[1:len(parts)]
-	cmd := exec.Command(head, parts...)
-	cmd.Stderr = os.Stderr
-	cmd.Stdout = os.Stdout
-	if err := cmd.Run(); err != nil {
-		return dockerBuildError(err)
-	}
-	return nil
-}
-
-func (lh *RubyLangHelper) AfterBuild() error {
-	return nil
-}

fn/langs/rust.go

@@ -9,6 +9,9 @@ type RustLangHelper struct {
 	BaseHelper
 }
 
+func (lh *RustLangHelper) BuildFromImage() string {
+	return "funcy/rust:dev"
+}
 func (lh *RustLangHelper) Entrypoint() string {
 	return "/function/target/release/func"
 }