Consolidate exec package with go-execute

This patch consolidates the exec package for docker build so that
it uses the the go-execute package used in other OpenFaaS projects.

The aim is to allow for conditional printing of stdio whilst also
being able to capture the output.

In a future PR a CLI animation can replace the Docker build, which
will be default, but optional. If an error is found then the
result of the build will be buffered and available to print to the
user.

This change stops Docker from printing progress bars when
downloading layers. Instead a line is printed when pulling and
when a layer is complete.

* Tested for faas-cli build with multiple functions using the
sample stack.yml and --parallel=1/4.

* Adds StreamStdio option and updates Docker build version to use
Go 1.12.

* Add complete build time to output

* Add duration of each build to output

* Add --quiet flag for faas-cli build

* The --quiet flag hides output from Docker during the execution
of the docker build.

Signed-off-by: Alex Ellis (OpenFaaS Ltd) <alexellis2@gmail.com>

.travis.yml

@@ -1,14 +1,11 @@
 sudo: required
-
 language: generic
- 
-services:
-  - docker
 
-addons:
-  apt:
-    packages:
-      - docker-ce
+services:
+- docker
+
+before_script:
+- curl -sSLf https://get.docker.com | sed s/sleep\ 20/sleep\ 0/g | sudo -E sh
 
 script:
   - make build
@@ -55,3 +52,5 @@ deploy:
   on:
     tags: true
 
+env:
+  - GO111MODULE=off

Dockerfile

@@ -1,5 +1,8 @@
 # Build stage
-FROM golang:1.11 as builder
+FROM golang:1.12 as builder
+
+ENV GO111MODULE=off
+ENV CGO_ENABLED=0
 
 WORKDIR /usr/bin/
 RUN curl -sLSf https://raw.githubusercontent.com/teamserverless/license-check/master/get.sh | sh
@@ -13,8 +16,10 @@ RUN test -z "$(gofmt -l $(find . -type f -name '*.go' -not -path "./vendor/*"))"
 # ldflags "-s -w" strips binary
 # ldflags -X injects commit version into binary
 RUN /usr/bin/license-check -path ./ --verbose=false "Alex Ellis" "OpenFaaS Author(s)"
-RUN go test $(go list ./... | grep -v /vendor/ | grep -v /template/|grep -v /build/) -cover \
-    && VERSION=$(git describe --all --exact-match `git rev-parse HEAD` | grep tags | sed 's/tags\///') \
+
+RUN go test $(go list ./... | grep -v /vendor/ | grep -v /template/|grep -v /build/|grep -v /sample/) -cover
+
+RUN VERSION=$(git describe --all --exact-match `git rev-parse HEAD` | grep tags | sed 's/tags\///') \
     && GIT_COMMIT=$(git rev-list -1 HEAD) \
     && CGO_ENABLED=0 GOOS=linux go build --ldflags "-s -w \
     -X github.com/openfaas/faas-cli/version.GitCommit=${GIT_COMMIT} \
@@ -23,7 +28,7 @@ RUN go test $(go list ./... | grep -v /vendor/ | grep -v /template/|grep -v /bui
     -a -installsuffix cgo -o faas-cli
 
 # Release stage
-FROM alpine:3.9
+FROM alpine:3.10
 
 RUN apk --no-cache add ca-certificates git
 

Dockerfile.redist

@@ -1,5 +1,8 @@
 # Build stage
-FROM golang:1.11 as builder
+FROM golang:1.12 as builder
+
+ENV GO111MODULE=off
+ENV CGO_ENABLED=0
 
 WORKDIR /usr/bin/
 RUN curl -sLSf https://raw.githubusercontent.com/teamserverless/license-check/master/get.sh | sh
@@ -44,7 +47,7 @@ RUN /usr/bin/license-check -path ./ --verbose=false "Alex Ellis" "OpenFaaS Autho
         -a -installsuffix cgo -o faas-cli-arm64
 
 # Release stage
-FROM alpine:3.9
+FROM alpine:3.10
 
 RUN apk --no-cache add ca-certificates git
 

Gopkg.lock

@@ -20,6 +20,14 @@
   revision = "839c75faf7f98a33d445d181f3018b5c3409a45e"
   version = "v1.4.2"
 
+[[projects]]
+  digest = "1:74860eb071d52337d67e9ffd6893b29affebd026505aa917ec23131576a91a77"
+  name = "github.com/alexellis/go-execute"
+  packages = ["pkg/v1"]
+  pruneopts = "UT"
+  revision = "961405ea754427780f2151adff607fa740d377f7"
+  version = "0.3.0"
+
 [[projects]]
   digest = "1:871b7cfa5fe18bfdbd4bf117c166c3cff8d3b61c8afe4e998b5b8ac0c160ca24"
   name = "github.com/alexellis/hmac"
@@ -166,6 +174,7 @@
   analyzer-name = "dep"
   analyzer-version = 1
   input-imports = [
+    "github.com/alexellis/go-execute/pkg/v1",
     "github.com/alexellis/hmac",
     "github.com/docker/docker-credential-helpers/client",
     "github.com/docker/docker/pkg/term",

Gopkg.toml

@@ -49,3 +49,9 @@
 [[constraint]]
   name = "github.com/pkg/errors"
   version = "v0.8.1"
+
+[[constraint]]
+  name = "github.com/alexellis/go-execute"
+  version = "0.3.0"
+
+

build.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 
 export eTAG="latest-dev"
 echo $1
@@ -8,8 +8,14 @@ fi
 
 echo Building openfaas/faas-cli:$eTAG
 
-docker build --build-arg http_proxy=$http_proxy --build-arg https_proxy=$https_proxy -t openfaas/faas-cli:$eTAG . && \
- docker create --name faas-cli openfaas/faas-cli:$eTAG && \
- docker cp faas-cli:/usr/bin/faas-cli . && \
- docker rm -f faas-cli
+docker build --build-arg http_proxy=$http_proxy --build-arg https_proxy=$https_proxy -t openfaas/faas-cli:$eTAG .
 
+if [ $? == 0 ] ; then
+
+  docker create --name faas-cli openfaas/faas-cli:$eTAG && \
+  docker cp faas-cli:/usr/bin/faas-cli . && \
+  docker rm -f faas-cli
+
+else
+ exit 1
+fi

builder/build.go

@@ -11,7 +11,7 @@ import (
 	"path/filepath"
 	"strings"
 
-	"github.com/openfaas/faas-cli/exec"
+	v1execute "github.com/alexellis/go-execute/pkg/v1"
 	"github.com/openfaas/faas-cli/schema"
 	"github.com/openfaas/faas-cli/stack"
 	vcs "github.com/openfaas/faas-cli/versioncontrol"
@@ -22,7 +22,7 @@ import (
 const AdditionalPackageBuildArg = "ADDITIONAL_PACKAGE"
 
 // BuildImage construct Docker image from function parameters
-func BuildImage(image string, handler string, functionName string, language string, nocache bool, squash bool, shrinkwrap bool, buildArgMap map[string]string, buildOptions []string, tagMode schema.BuildFormat, buildLabelMap map[string]string) error {
+func BuildImage(image string, handler string, functionName string, language string, nocache bool, squash bool, shrinkwrap bool, buildArgMap map[string]string, buildOptions []string, tagMode schema.BuildFormat, buildLabelMap map[string]string, quietBuild bool) error {
 
 	if stack.IsValidTemplate(language) {
 		branch, version, err := GetImageTagValues(tagMode)
@@ -67,9 +67,25 @@ func BuildImage(image string, handler string, functionName string, language stri
 			BuildLabelMap:    buildLabelMap,
 		}
 
-		spaceSafeCmdLine := getDockerBuildCommand(dockerBuildVal)
+		command, args := getDockerBuildCommand(dockerBuildVal)
+
+		task := v1execute.ExecTask{
+			Cwd:         tempPath,
+			Command:     command,
+			Args:        args,
+			StreamStdio: !quietBuild,
+		}
+
+		res, err := task.Execute()
+
+		if err != nil {
+			return err
+		}
+
+		if res.ExitCode != 0 {
+			return fmt.Errorf("[%s] received non-zero exit code from build, error: %s", functionName, res.Stderr)
+		}
 
-		exec.Command(tempPath, spaceSafeCmdLine)
 		fmt.Printf("Image: %s built.\n", imageName)
 
 	} else {
@@ -113,13 +129,15 @@ func GetImageTagValues(tagType schema.BuildFormat) (branch, version string, err
 	return branch, version, nil
 }
 
-func getDockerBuildCommand(build dockerBuild) []string {
+func getDockerBuildCommand(build dockerBuild) (string, []string) {
 	flagSlice := buildFlagSlice(build.NoCache, build.Squash, build.HTTPProxy, build.HTTPSProxy, build.BuildArgMap, build.BuildOptPackages, build.BuildLabelMap)
-	command := []string{"docker", "build"}
-	command = append(command, flagSlice...)
-	command = append(command, "-t", build.Image, ".")
+	args := []string{"build"}
+	args = append(args, flagSlice...)
+	args = append(args, "-t", build.Image, ".")
 
-	return command
+	command := "docker"
+
+	return command, args
 }
 
 type dockerBuild struct {

builder/build_test.go

@@ -41,14 +41,20 @@ func Test_getDockerBuildCommand_NoOpts(t *testing.T) {
 		BuildOptPackages: []string{},
 	}
 
-	values := getDockerBuildCommand(dockerBuildVal)
+	want := "build -t imagename:latest ."
+	wantCommand := "docker"
 
-	joined := strings.Join(values, " ")
-	want := "docker build -t imagename:latest ."
+	command, args := getDockerBuildCommand(dockerBuildVal)
+
+	joined := strings.Join(args, " ")
 
 	if joined != want {
 		t.Errorf("getDockerBuildCommand want: \"%s\", got: \"%s\"", want, joined)
 	}
+
+	if command != wantCommand {
+		t.Errorf("getDockerBuildCommand want command: \"%s\", got: \"%s\"", wantCommand, command)
+	}
 }
 
 func Test_getDockerBuildCommand_WithNoCache(t *testing.T) {
@@ -62,14 +68,21 @@ func Test_getDockerBuildCommand_WithNoCache(t *testing.T) {
 		BuildOptPackages: []string{},
 	}
 
-	values := getDockerBuildCommand(dockerBuildVal)
+	want := "build --no-cache -t imagename:latest ."
 
-	joined := strings.Join(values, " ")
-	want := "docker build --no-cache -t imagename:latest ."
+	wantCommand := "docker"
+
+	command, args := getDockerBuildCommand(dockerBuildVal)
+
+	joined := strings.Join(args, " ")
 
 	if joined != want {
 		t.Errorf("getDockerBuildCommand want: \"%s\", got: \"%s\"", want, joined)
 	}
+
+	if command != wantCommand {
+		t.Errorf("getDockerBuildCommand want command: \"%s\", got: \"%s\"", wantCommand, command)
+	}
 }
 
 func Test_getDockerBuildCommand_WithProxies(t *testing.T) {
@@ -83,14 +96,21 @@ func Test_getDockerBuildCommand_WithProxies(t *testing.T) {
 		BuildOptPackages: []string{},
 	}
 
-	values := getDockerBuildCommand(dockerBuildVal)
+	want := "build --build-arg http_proxy=http://127.0.0.1:3128 --build-arg https_proxy=https://127.0.0.1:3128 -t imagename:latest ."
 
-	joined := strings.Join(values, " ")
-	want := "docker build --build-arg http_proxy=http://127.0.0.1:3128 --build-arg https_proxy=https://127.0.0.1:3128 -t imagename:latest ."
+	wantCommand := "docker"
+
+	command, args := getDockerBuildCommand(dockerBuildVal)
+
+	joined := strings.Join(args, " ")
 
 	if joined != want {
 		t.Errorf("getDockerBuildCommand want: \"%s\", got: \"%s\"", want, joined)
 	}
+
+	if command != wantCommand {
+		t.Errorf("getDockerBuildCommand want command: \"%s\", got: \"%s\"", wantCommand, command)
+	}
 }
 
 func Test_getDockerBuildCommand_WithBuildArg(t *testing.T) {
@@ -105,7 +125,7 @@ func Test_getDockerBuildCommand_WithBuildArg(t *testing.T) {
 		BuildOptPackages: []string{},
 	}
 
-	values := getDockerBuildCommand(dockerBuildVal)
+	_, values := getDockerBuildCommand(dockerBuildVal)
 
 	joined := strings.Join(values, " ")
 	wantArg1 := "--build-arg USERNAME=admin"

commands/build.go

@@ -9,6 +9,7 @@ import (
 	"os"
 	"strings"
 	"sync"
+	"time"
 
 	"github.com/morikuni/aec"
 	"github.com/openfaas/faas-cli/builder"
@@ -31,6 +32,7 @@ var (
 	buildLabels   []string
 	buildLabelMap map[string]string
 	envsubst      bool
+	quietBuild    bool
 )
 
 func init() {
@@ -52,6 +54,8 @@ func init() {
 
 	buildCmd.Flags().BoolVar(&envsubst, "envsubst", true, "Substitute environment variables in stack.yml file")
 
+	buildCmd.Flags().BoolVar(&quietBuild, "quiet", false, "Perform a quiet build, without showing output from Docker")
+
 	// Set bash-completion.
 	_ = buildCmd.Flags().SetAnnotation("handler", cobra.BashCompSubdirsInDir, []string{})
 
@@ -103,6 +107,10 @@ func preRunBuild(cmd *cobra.Command, args []string) error {
 
 	buildLabelMap, err = parseMap(buildLabels, "build-label")
 
+	if parallel < 1 {
+		return fmt.Errorf("the --parallel flag must be great than 0")
+	}
+
 	return err
 }
 
@@ -156,11 +164,7 @@ func runBuild(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("could not pull templates for OpenFaaS: %v", pullErr)
 	}
 
-	if len(services.Functions) > 0 {
-
-		build(&services, parallel, shrinkwrap)
-
-	} else {
+	if len(services.Functions) == 0 {
 		if len(image) == 0 {
 			return fmt.Errorf("please provide a valid --image name for your Docker image")
 		}
@@ -170,16 +174,29 @@ func runBuild(cmd *cobra.Command, args []string) error {
 		if len(functionName) == 0 {
 			return fmt.Errorf("please provide the deployed --name of your function")
 		}
-		err := builder.BuildImage(image, handler, functionName, language, nocache, squash, shrinkwrap, buildArgMap, buildOptions, tagFormat, buildLabelMap)
+		err := builder.BuildImage(image, handler, functionName, language, nocache, squash, shrinkwrap, buildArgMap, buildOptions, tagFormat, buildLabelMap, quietBuild)
 		if err != nil {
 			return err
 		}
+		return nil
 	}
 
+	errors := build(&services, parallel, shrinkwrap, quietBuild)
+	if len(errors) > 0 {
+		errorSummary := "Errors received during build:\n"
+		for _, err := range errors {
+			errorSummary = errorSummary + "- " + err.Error() + "\n"
+		}
+		return fmt.Errorf("%s", aec.Apply(errorSummary, aec.RedF))
+	}
 	return nil
 }
 
-func build(services *stack.Services, queueDepth int, shrinkwrap bool) {
+func build(services *stack.Services, queueDepth int, shrinkwrap, quietBuild bool) []error {
+	startOuter := time.Now()
+
+	errors := []error{}
+
 	wg := sync.WaitGroup{}
 
 	workChannel := make(chan stack.Function)
@@ -188,23 +205,28 @@ func build(services *stack.Services, queueDepth int, shrinkwrap bool) {
 	for i := 0; i < queueDepth; i++ {
 		go func(index int) {
 			for function := range workChannel {
+				start := time.Now()
+
 				fmt.Printf(aec.YellowF.Apply("[%d] > Building %s.\n"), index, function.Name)
 				if len(function.Language) == 0 {
 					fmt.Println("Please provide a valid language for your function.")
 				} else {
 
 					combinedBuildOptions := combineBuildOpts(function.BuildOptions, buildOptions)
-					err := builder.BuildImage(function.Image, function.Handler, function.Name, function.Language, nocache, squash, shrinkwrap, buildArgMap, combinedBuildOptions, tagFormat, buildLabelMap)
+					err := builder.BuildImage(function.Image, function.Handler, function.Name, function.Language, nocache, squash, shrinkwrap, buildArgMap, combinedBuildOptions, tagFormat, buildLabelMap, quietBuild)
 					if err != nil {
-						log.Println(err)
+						errors = append(errors, err)
 					}
 				}
-				fmt.Printf(aec.YellowF.Apply("[%d] < Building %s done.\n"), index, function.Name)
+
+				duration := time.Since(start)
+				fmt.Printf(aec.YellowF.Apply("[%d] < Building %s done in %1.2fs.\n"), index, function.Name, duration.Seconds())
 			}
 
-			fmt.Printf(aec.YellowF.Apply("[%d] worker done.\n"), index)
+			fmt.Printf(aec.YellowF.Apply("[%d] Worker done.\n"), index)
 			wg.Done()
 		}(i)
+
 	}
 
 	for k, function := range services.Functions {
@@ -220,6 +242,9 @@ func build(services *stack.Services, queueDepth int, shrinkwrap bool) {
 
 	wg.Wait()
 
+	duration := time.Since(startOuter)
+	fmt.Printf("\n%s\n", aec.Apply(fmt.Sprintf("Total build time: %1.2f", duration.Seconds()), aec.YellowF))
+	return errors
 }
 
 // PullTemplates pulls templates from specified git remote. templateURL may be a pinned repository.

commands/build_test.go

@@ -26,6 +26,21 @@ func Test_build(t *testing.T) {
 	}
 }
 
+func Test_preRunBuild_ParallelOverZero(t *testing.T) {
+	buildCmd.ParseFlags([]string{"--parallel=0"})
+	got := buildCmd.PreRunE(buildCmd, nil)
+
+	if got == nil {
+		t.Error("Parallel should have errored about being over zero")
+		t.Fail()
+		return
+	}
+	want := "the --parallel flag must be great than 0"
+	if got.Error() != want {
+		t.Errorf("parsing error, want %s, got %s", want, got.Error())
+	}
+}
+
 func Test_parseBuildArgs_ValidParts(t *testing.T) {
 	mapped, err := parseBuildArgs([]string{"k=v"})
 

commands/push.go

@@ -109,7 +109,7 @@ func pushStack(services *stack.Services, queueDepth int, tagMode schema.BuildFor
 				}
 			}
 
-			fmt.Printf(aec.YellowF.Apply("[%d] worker done.\n"), index)
+			fmt.Printf(aec.YellowF.Apply("[%d] Worker done.\n"), index)
 			wg.Done()
 		}(i)
 	}

exec/exec.go

@@ -18,6 +18,7 @@ func Command(tempPath string, builder []string) {
 	targetCmd.Dir = tempPath
 	targetCmd.Stdout = os.Stdout
 	targetCmd.Stderr = os.Stderr
+
 	targetCmd.Start()
 	err := targetCmd.Wait()
 	if err != nil {

sample/imagemagick/Dockerfile

@@ -1,4 +1,4 @@
-FROM alpine:3.9
+FROM alpine:3.10
 
 # Alternatively use ADD https:// (which will not be cached by Docker builder)
 RUN apk --no-cache add curl ca-certificates imagemagick \