Add "get" command

The get command downloads tools from GitHub release pages
and from custom URLs using a go template to define the
URL and filename according to OS and architecture.

Tested with unit tests and with the two initial apps: faas-cli
and kubectl.

Signed-off-by: Alex Ellis (OpenFaaS Ltd) <alexellis2@gmail.com>

.gitignore

@@ -5,4 +5,4 @@ kubeconfig
 .idea/
 mc
 /arkade-*
-/faas-cli-darwin
+/faas-cli*

cmd/get.go

@@ -0,0 +1,111 @@
+// Copyright (c) arkade author(s) 2020. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+package cmd
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"path"
+	"strings"
+
+	"github.com/alexellis/arkade/pkg/env"
+	"github.com/alexellis/arkade/pkg/get"
+	"github.com/spf13/cobra"
+)
+
+func MakeGet() *cobra.Command {
+	tools := get.MakeTools()
+
+	var command = &cobra.Command{
+		Use:   "get",
+		Short: "Get a release of a tool or application and install it on your local computer.",
+		Example: `  arkade get kubectl
+  arkade get openfaas`,
+		SilenceUsage: true,
+	}
+
+	command.RunE = func(cmd *cobra.Command, args []string) error {
+		if len(args) == 0 {
+			fmt.Println(arkadeGet)
+			return nil
+		}
+		var tool *get.Tool
+
+		if len(args) == 1 {
+			for _, t := range tools {
+				if t.Name == args[0] {
+					tool = &t
+					break
+				}
+			}
+		}
+		if tool == nil {
+			return fmt.Errorf("cannot get tool: %s", args[0])
+		}
+
+		fmt.Printf("Downloading %s\n", tool.Name)
+
+		arch, operatingSystem := env.GetClientArch()
+		version := ""
+
+		downloadURL, err := get.GetDownloadURL(tool, strings.ToLower(operatingSystem), strings.ToLower(arch), version)
+		if err != nil {
+			return err
+		}
+
+		fmt.Println(downloadURL)
+
+		res, err := http.DefaultClient.Get(downloadURL)
+		if err != nil {
+			return err
+		}
+
+		if res.Body != nil {
+			defer res.Body.Close()
+		}
+
+		if res.StatusCode != http.StatusOK {
+			return fmt.Errorf("incorrect status for downloading tool: %d", res.StatusCode)
+		}
+
+		_, fileName := path.Split(downloadURL)
+		tmp := os.TempDir()
+
+		outFilePath := path.Join(tmp, fileName)
+
+		out, err := os.Create(outFilePath)
+		if err != nil {
+			return err
+		}
+		defer out.Close()
+
+		if _, err = io.Copy(out, res.Body); err != nil {
+			return err
+		}
+
+		finalName := tool.Name
+		if strings.Contains(strings.ToLower(operatingSystem), "mingw") {
+			finalName = finalName + ".exe"
+		}
+
+		fmt.Printf(`Tool written to: %s
+
+Run the following to copy to install the tool:
+
+chmod +x %s
+sudo install -m 755 %s /usr/local/bin/%s
+`, outFilePath, outFilePath, outFilePath, finalName)
+
+		return err
+	}
+
+	return command
+}
+
+const arkadeGet = `Use "arkade get TOOL" to download a tool or application:
+
+  - kubectl
+  - faas-cli`

go.mod

@@ -7,5 +7,6 @@ require (
 	github.com/morikuni/aec v1.0.0
 	github.com/sethvargo/go-password v0.1.2
 	github.com/spf13/cobra v0.0.5
+	github.com/spf13/pflag v1.0.5 // indirect
 	golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4
 )

go.sum

@@ -29,6 +29,8 @@ github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tL
 github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
 github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
+github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
@@ -43,4 +45,5 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

main.go

@@ -15,7 +15,6 @@ func main() {
 	cmdVersion := cmd.MakeVersion()
 	cmdInstall := cmd.MakeInstall()
 	cmdInfo := cmd.MakeInfo()
-	cmdUpdate := cmd.MakeUpdate()
 
 	printarkadeASCIIArt := cmd.PrintArkadeASCIIArt
 
@@ -30,7 +29,8 @@ func main() {
 	rootCmd.AddCommand(cmdInstall)
 	rootCmd.AddCommand(cmdVersion)
 	rootCmd.AddCommand(cmdInfo)
-	rootCmd.AddCommand(cmdUpdate)
+	rootCmd.AddCommand(cmd.MakeUpdate())
+	rootCmd.AddCommand(cmd.MakeGet())
 
 	if err := rootCmd.Execute(); err != nil {
 		os.Exit(1)

pkg/env/env.go

@@ -13,14 +13,14 @@ import (
 )
 
 // GetClientArch returns a pair of arch and os
-func GetClientArch() (string, string) {
+func GetClientArch() (arch string, os string) {
 	task := execute.ExecTask{Command: "uname", Args: []string{"-m"}, StreamStdio: false}
 	res, err := task.Execute()
 	if err != nil {
 		log.Println(err)
 	}
 
-	arch := strings.TrimSpace(res.Stdout)
+	archResult := strings.TrimSpace(res.Stdout)
 
 	taskOS := execute.ExecTask{Command: "uname", Args: []string{"-s"}, StreamStdio: false}
 	resOS, errOS := taskOS.Execute()
@@ -28,9 +28,9 @@ func GetClientArch() (string, string) {
 		log.Println(errOS)
 	}
 
-	os := strings.TrimSpace(resOS.Stdout)
+	osResult := strings.TrimSpace(resOS.Stdout)
 
-	return arch, os
+	return archResult, osResult
 }
 
 func LocalBinary(name, subdir string) string {

pkg/get/get.go

@@ -0,0 +1,229 @@
+package get
+
+import (
+	"bytes"
+	"crypto/tls"
+	"fmt"
+	"html/template"
+	"net"
+	"net/http"
+	"strings"
+	"time"
+)
+
+type Tool struct {
+	Name           string
+	Repo           string
+	Owner          string
+	Version        string
+	URLTemplate    string
+	BinaryTemplate string
+}
+
+var templateFuncs = map[string]interface{}{
+	"HasPrefix": func(s, prefix string) bool { return strings.HasPrefix(s, prefix) },
+}
+
+// Download fetches the download URL for a release of a tool
+// for a given os,  architecture and version
+func GetDownloadURL(tool *Tool, os, arch, version string) (string, error) {
+	ver := tool.Version
+	if len(version) > 0 {
+		ver = version
+	}
+
+	dlURL, err := tool.GetURL(os, arch, ver)
+	if err != nil {
+		return "", err
+	}
+
+	return dlURL, nil
+}
+
+func (tool Tool) GetURL(os, arch, version string) (string, error) {
+
+	if len(tool.URLTemplate) == 0 {
+		return getURLByGithubTemplate(tool, os, arch, version)
+	}
+
+	return getByDownloadTemplate(tool, os, arch, version)
+
+}
+
+func (t Tool) Latest() bool {
+	return len(t.Version) == 0
+}
+
+func getURLByGithubTemplate(tool Tool, os, arch, version string) (string, error) {
+	if len(version) == 0 {
+		releases := fmt.Sprintf("https://github.com/%s/%s/releases/latest", tool.Owner, tool.Name)
+		var err error
+		version, err = findGitHubRelease(releases)
+		if err != nil {
+			return "", err
+		}
+	}
+
+	var err error
+	t := template.New(tool.Name + "binary")
+
+	t = t.Funcs(templateFuncs)
+	t, err = t.Parse(tool.BinaryTemplate)
+	if err != nil {
+		return "", err
+	}
+
+	var buf bytes.Buffer
+	pref := map[string]string{
+		"OS":   os,
+		"Arch": arch,
+		"Name": tool.Name,
+	}
+
+	err = t.Execute(&buf, pref)
+	if err != nil {
+		return "", err
+	}
+	res := strings.TrimSpace(buf.String())
+
+	return fmt.Sprintf(
+		"https://github.com/%s/%s/releases/download/%s/%s",
+		tool.Owner, tool.Name, version, res), nil
+}
+
+func findGitHubRelease(url string) (string, error) {
+	timeout := time.Second * 5
+	client := makeHTTPClient(&timeout, false)
+	client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
+		return http.ErrUseLastResponse
+	}
+
+	req, err := http.NewRequest(http.MethodHead, url, nil)
+	if err != nil {
+		return "", err
+	}
+
+	res, err := client.Do(req)
+	if err != nil {
+		return "", err
+	}
+
+	if res.Body != nil {
+		defer res.Body.Close()
+	}
+
+	if res.StatusCode != 302 {
+		return "", fmt.Errorf("incorrect status code: %d", res.StatusCode)
+	}
+
+	loc := res.Header.Get("Location")
+	if len(loc) == 0 {
+		return "", fmt.Errorf("unable to determine release of tool")
+	}
+
+	version := loc[strings.LastIndex(loc, "/")+1:]
+	return version, nil
+}
+
+func getByDownloadTemplate(tool Tool, os, arch, version string) (string, error) {
+	var err error
+	t := template.New(tool.Name)
+	t = t.Funcs(templateFuncs)
+	t, err = t.Parse(tool.URLTemplate)
+	if err != nil {
+		return "", err
+	}
+
+	var buf bytes.Buffer
+	err = t.Execute(&buf, map[string]string{
+		"OS":      os,
+		"Arch":    arch,
+		"Version": version,
+	})
+	if err != nil {
+		return "", err
+	}
+	res := strings.TrimSpace(buf.String())
+	return res, nil
+}
+
+func MakeTools() []Tool {
+	tools := []Tool{
+		{
+			Owner: "openfaas",
+			Repo:  "faas-cli",
+			Name:  "faas-cli",
+			BinaryTemplate: `{{ if HasPrefix .OS "ming" -}}
+{{.Name}}.exe
+{{- else if eq .OS "darwin" -}}
+{{.Name}}-darwin
+{{- else if eq .Arch "armv6l" -}}
+{{.Name}}-armhf
+{{- else if eq .Arch "armv7l" -}}
+{{.Name}}-armhf
+{{- else if eq .Arch "aarch64" -}}
+{{.Name}}-arm64
+{{- end -}}`,
+		},
+		//https://storage.googleapis.com/kubernetes-release/release/v1.18.0/bin/darwin/amd64/kubectl
+		{
+			Owner:   "kubernetes",
+			Repo:    "kubernetes",
+			Name:    "kubectl",
+			Version: "v1.18.0",
+			URLTemplate: `{{$arch := "arm"}}
+
+			{{- if eq .Arch "x86_64" -}}
+{{$arch = "amd64"}}
+{{- end -}}
+
+{{$ext := ""}}
+{{$os := .OS}}
+
+{{ if HasPrefix .OS "ming" -}}
+{{$ext = ".exe"}}
+{{$os = "windows"}}
+{{- end -}}
+
+https://storage.googleapis.com/kubernetes-release/release/{{.Version}}/bin/{{$os}}/{{$arch}}/kubectl{{$ext}}`,
+		},
+	}
+	return tools
+}
+
+// makeHTTPClient makes a HTTP client with good defaults for timeouts.
+func makeHTTPClient(timeout *time.Duration, tlsInsecure bool) http.Client {
+	return makeHTTPClientWithDisableKeepAlives(timeout, tlsInsecure, false)
+}
+
+// makeHTTPClientWithDisableKeepAlives makes a HTTP client with good defaults for timeouts.
+func makeHTTPClientWithDisableKeepAlives(timeout *time.Duration, tlsInsecure bool, disableKeepAlives bool) http.Client {
+	client := http.Client{}
+
+	if timeout != nil || tlsInsecure {
+		tr := &http.Transport{
+			Proxy:             http.ProxyFromEnvironment,
+			DisableKeepAlives: disableKeepAlives,
+		}
+
+		if timeout != nil {
+			client.Timeout = *timeout
+			tr.DialContext = (&net.Dialer{
+				Timeout: *timeout,
+			}).DialContext
+
+			tr.IdleConnTimeout = 120 * time.Millisecond
+			tr.ExpectContinueTimeout = 1500 * time.Millisecond
+		}
+
+		if tlsInsecure {
+			tr.TLSClientConfig = &tls.Config{InsecureSkipVerify: tlsInsecure}
+		}
+
+		tr.DisableKeepAlives = disableKeepAlives
+
+		client.Transport = tr
+	}
+
+	return client
+}

pkg/get/get_test.go

@@ -0,0 +1,132 @@
+package get
+
+import "testing"
+
+const faasCLIVersion = "0.12.4"
+const arch64bit = "x86_64"
+
+func Test_DownloadDarwin(t *testing.T) {
+	tools := MakeTools()
+	name := "faas-cli"
+	var tool *Tool
+	for _, target := range tools {
+		if name == target.Name {
+			tool = &target
+			break
+		}
+	}
+
+	got, err := tool.GetURL("darwin", "", "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	want := "https://github.com/openfaas/faas-cli/releases/download/" + faasCLIVersion + "/faas-cli-darwin"
+	if got != want {
+		t.Fatalf("want: %s, got: %s", want, got)
+	}
+}
+
+func Test_DownloadKubectlDarwin(t *testing.T) {
+	tools := MakeTools()
+	name := "kubectl"
+	var tool *Tool
+	for _, target := range tools {
+		if name == target.Name {
+			tool = &target
+			break
+		}
+	}
+
+	got, err := tool.GetURL("darwin", arch64bit, tool.Version)
+	if err != nil {
+		t.Fatal(err)
+	}
+	want := "https://storage.googleapis.com/kubernetes-release/release/v1.18.0/bin/darwin/amd64/kubectl"
+	if got != want {
+		t.Fatalf("want: %s, got: %s", want, got)
+	}
+}
+
+func Test_DownloadKubectlLinux(t *testing.T) {
+	tools := MakeTools()
+	name := "kubectl"
+	var tool *Tool
+	for _, target := range tools {
+		if name == target.Name {
+			tool = &target
+			break
+		}
+	}
+
+	got, err := tool.GetURL("linux", arch64bit, tool.Version)
+	if err != nil {
+		t.Fatal(err)
+	}
+	want := "https://storage.googleapis.com/kubernetes-release/release/v1.18.0/bin/linux/amd64/kubectl"
+	if got != want {
+		t.Fatalf("want: %s, got: %s", want, got)
+	}
+}
+
+func Test_DownloadArmhf(t *testing.T) {
+	tools := MakeTools()
+	name := "faas-cli"
+	var tool *Tool
+	for _, target := range tools {
+		if name == target.Name {
+			tool = &target
+			break
+		}
+	}
+
+	got, err := tool.GetURL("Linux", "armv7l", "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	want := "https://github.com/openfaas/faas-cli/releases/download/" + faasCLIVersion + "/faas-cli-armhf"
+	if got != want {
+		t.Fatalf("want: %s, got: %s", want, got)
+	}
+}
+
+func Test_DownloadArm64(t *testing.T) {
+	tools := MakeTools()
+	name := "faas-cli"
+	var tool *Tool
+	for _, target := range tools {
+		if name == target.Name {
+			tool = &target
+			break
+		}
+	}
+
+	got, err := tool.GetURL("Linux", "aarch64", "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	want := "https://github.com/openfaas/faas-cli/releases/download/" + faasCLIVersion + "/faas-cli-arm64"
+	if got != want {
+		t.Fatalf("want: %s, got: %s", want, got)
+	}
+}
+
+func Test_DownloadWindows(t *testing.T) {
+	tools := MakeTools()
+	name := "faas-cli"
+	var tool *Tool
+	for _, target := range tools {
+		if name == target.Name {
+			tool = &target
+			break
+		}
+	}
+
+	got, err := tool.GetURL("mingw64_nt-10.0-18362", arch64bit, "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	want := "https://github.com/openfaas/faas-cli/releases/download/" + faasCLIVersion + "/faas-cli.exe"
+	if got != want {
+		t.Fatalf("want: %s, got: %s", want, got)
+	}
+}