From 4efd6d7869abd557e28d95e15da26eaf0ca31f35 Mon Sep 17 00:00:00 2001 From: "Alex Ellis (OpenFaaS Ltd)" Date: Tue, 16 Jun 2020 10:21:43 +0100 Subject: [PATCH] Add "get" command The get command downloads tools from GitHub release pages and from custom URLs using a go template to define the URL and filename according to OS and architecture. Tested with unit tests and with the two initial apps: faas-cli and kubectl. Signed-off-by: Alex Ellis (OpenFaaS Ltd) --- .gitignore | 2 +- cmd/get.go | 111 +++++++++++++++++++++ go.mod | 1 + go.sum | 3 + main.go | 4 +- pkg/env/env.go | 8 +- pkg/get/get.go | 229 ++++++++++++++++++++++++++++++++++++++++++++ pkg/get/get_test.go | 132 +++++++++++++++++++++++++ 8 files changed, 483 insertions(+), 7 deletions(-) create mode 100644 cmd/get.go create mode 100644 pkg/get/get.go create mode 100644 pkg/get/get_test.go diff --git a/.gitignore b/.gitignore index 33ccffe..2488b9a 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,4 @@ kubeconfig .idea/ mc /arkade-* -/faas-cli-darwin +/faas-cli* diff --git a/cmd/get.go b/cmd/get.go new file mode 100644 index 0000000..7c18c4e --- /dev/null +++ b/cmd/get.go @@ -0,0 +1,111 @@ +// Copyright (c) arkade author(s) 2020. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +package cmd + +import ( + "fmt" + "io" + "net/http" + "os" + "path" + "strings" + + "github.com/alexellis/arkade/pkg/env" + "github.com/alexellis/arkade/pkg/get" + "github.com/spf13/cobra" +) + +func MakeGet() *cobra.Command { + tools := get.MakeTools() + + var command = &cobra.Command{ + Use: "get", + Short: "Get a release of a tool or application and install it on your local computer.", + Example: ` arkade get kubectl + arkade get openfaas`, + SilenceUsage: true, + } + + command.RunE = func(cmd *cobra.Command, args []string) error { + if len(args) == 0 { + fmt.Println(arkadeGet) + return nil + } + var tool *get.Tool + + if len(args) == 1 { + for _, t := range tools { + if t.Name == args[0] { + tool = &t + break + } + } + } + if tool == nil { + return fmt.Errorf("cannot get tool: %s", args[0]) + } + + fmt.Printf("Downloading %s\n", tool.Name) + + arch, operatingSystem := env.GetClientArch() + version := "" + + downloadURL, err := get.GetDownloadURL(tool, strings.ToLower(operatingSystem), strings.ToLower(arch), version) + if err != nil { + return err + } + + fmt.Println(downloadURL) + + res, err := http.DefaultClient.Get(downloadURL) + if err != nil { + return err + } + + if res.Body != nil { + defer res.Body.Close() + } + + if res.StatusCode != http.StatusOK { + return fmt.Errorf("incorrect status for downloading tool: %d", res.StatusCode) + } + + _, fileName := path.Split(downloadURL) + tmp := os.TempDir() + + outFilePath := path.Join(tmp, fileName) + + out, err := os.Create(outFilePath) + if err != nil { + return err + } + defer out.Close() + + if _, err = io.Copy(out, res.Body); err != nil { + return err + } + + finalName := tool.Name + if strings.Contains(strings.ToLower(operatingSystem), "mingw") { + finalName = finalName + ".exe" + } + + fmt.Printf(`Tool written to: %s + +Run the following to copy to install the tool: + +chmod +x %s +sudo install -m 755 %s /usr/local/bin/%s +`, outFilePath, outFilePath, outFilePath, finalName) + + return err + } + + return command +} + +const arkadeGet = `Use "arkade get TOOL" to download a tool or application: + + - kubectl + - faas-cli` diff --git a/go.mod b/go.mod index 9c0cb5f..e2fc81f 100644 --- a/go.mod +++ b/go.mod @@ -7,5 +7,6 @@ require ( github.com/morikuni/aec v1.0.0 github.com/sethvargo/go-password v0.1.2 github.com/spf13/cobra v0.0.5 + github.com/spf13/pflag v1.0.5 // indirect golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 ) diff --git a/go.sum b/go.sum index f1c5f7c..4c2ee93 100644 --- a/go.sum +++ b/go.sum @@ -29,6 +29,8 @@ github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tL github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= @@ -43,4 +45,5 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/main.go b/main.go index 5cd4630..657d2ee 100644 --- a/main.go +++ b/main.go @@ -15,7 +15,6 @@ func main() { cmdVersion := cmd.MakeVersion() cmdInstall := cmd.MakeInstall() cmdInfo := cmd.MakeInfo() - cmdUpdate := cmd.MakeUpdate() printarkadeASCIIArt := cmd.PrintArkadeASCIIArt @@ -30,7 +29,8 @@ func main() { rootCmd.AddCommand(cmdInstall) rootCmd.AddCommand(cmdVersion) rootCmd.AddCommand(cmdInfo) - rootCmd.AddCommand(cmdUpdate) + rootCmd.AddCommand(cmd.MakeUpdate()) + rootCmd.AddCommand(cmd.MakeGet()) if err := rootCmd.Execute(); err != nil { os.Exit(1) diff --git a/pkg/env/env.go b/pkg/env/env.go index 707d971..7316b5c 100644 --- a/pkg/env/env.go +++ b/pkg/env/env.go @@ -13,14 +13,14 @@ import ( ) // GetClientArch returns a pair of arch and os -func GetClientArch() (string, string) { +func GetClientArch() (arch string, os string) { task := execute.ExecTask{Command: "uname", Args: []string{"-m"}, StreamStdio: false} res, err := task.Execute() if err != nil { log.Println(err) } - arch := strings.TrimSpace(res.Stdout) + archResult := strings.TrimSpace(res.Stdout) taskOS := execute.ExecTask{Command: "uname", Args: []string{"-s"}, StreamStdio: false} resOS, errOS := taskOS.Execute() @@ -28,9 +28,9 @@ func GetClientArch() (string, string) { log.Println(errOS) } - os := strings.TrimSpace(resOS.Stdout) + osResult := strings.TrimSpace(resOS.Stdout) - return arch, os + return archResult, osResult } func LocalBinary(name, subdir string) string { diff --git a/pkg/get/get.go b/pkg/get/get.go new file mode 100644 index 0000000..13f5a17 --- /dev/null +++ b/pkg/get/get.go @@ -0,0 +1,229 @@ +package get + +import ( + "bytes" + "crypto/tls" + "fmt" + "html/template" + "net" + "net/http" + "strings" + "time" +) + +type Tool struct { + Name string + Repo string + Owner string + Version string + URLTemplate string + BinaryTemplate string +} + +var templateFuncs = map[string]interface{}{ + "HasPrefix": func(s, prefix string) bool { return strings.HasPrefix(s, prefix) }, +} + +// Download fetches the download URL for a release of a tool +// for a given os, architecture and version +func GetDownloadURL(tool *Tool, os, arch, version string) (string, error) { + ver := tool.Version + if len(version) > 0 { + ver = version + } + + dlURL, err := tool.GetURL(os, arch, ver) + if err != nil { + return "", err + } + + return dlURL, nil +} + +func (tool Tool) GetURL(os, arch, version string) (string, error) { + + if len(tool.URLTemplate) == 0 { + return getURLByGithubTemplate(tool, os, arch, version) + } + + return getByDownloadTemplate(tool, os, arch, version) + +} + +func (t Tool) Latest() bool { + return len(t.Version) == 0 +} + +func getURLByGithubTemplate(tool Tool, os, arch, version string) (string, error) { + if len(version) == 0 { + releases := fmt.Sprintf("https://github.com/%s/%s/releases/latest", tool.Owner, tool.Name) + var err error + version, err = findGitHubRelease(releases) + if err != nil { + return "", err + } + } + + var err error + t := template.New(tool.Name + "binary") + + t = t.Funcs(templateFuncs) + t, err = t.Parse(tool.BinaryTemplate) + if err != nil { + return "", err + } + + var buf bytes.Buffer + pref := map[string]string{ + "OS": os, + "Arch": arch, + "Name": tool.Name, + } + + err = t.Execute(&buf, pref) + if err != nil { + return "", err + } + res := strings.TrimSpace(buf.String()) + + return fmt.Sprintf( + "https://github.com/%s/%s/releases/download/%s/%s", + tool.Owner, tool.Name, version, res), nil +} + +func findGitHubRelease(url string) (string, error) { + timeout := time.Second * 5 + client := makeHTTPClient(&timeout, false) + client.CheckRedirect = func(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse + } + + req, err := http.NewRequest(http.MethodHead, url, nil) + if err != nil { + return "", err + } + + res, err := client.Do(req) + if err != nil { + return "", err + } + + if res.Body != nil { + defer res.Body.Close() + } + + if res.StatusCode != 302 { + return "", fmt.Errorf("incorrect status code: %d", res.StatusCode) + } + + loc := res.Header.Get("Location") + if len(loc) == 0 { + return "", fmt.Errorf("unable to determine release of tool") + } + + version := loc[strings.LastIndex(loc, "/")+1:] + return version, nil +} + +func getByDownloadTemplate(tool Tool, os, arch, version string) (string, error) { + var err error + t := template.New(tool.Name) + t = t.Funcs(templateFuncs) + t, err = t.Parse(tool.URLTemplate) + if err != nil { + return "", err + } + + var buf bytes.Buffer + err = t.Execute(&buf, map[string]string{ + "OS": os, + "Arch": arch, + "Version": version, + }) + if err != nil { + return "", err + } + res := strings.TrimSpace(buf.String()) + return res, nil +} + +func MakeTools() []Tool { + tools := []Tool{ + { + Owner: "openfaas", + Repo: "faas-cli", + Name: "faas-cli", + BinaryTemplate: `{{ if HasPrefix .OS "ming" -}} +{{.Name}}.exe +{{- else if eq .OS "darwin" -}} +{{.Name}}-darwin +{{- else if eq .Arch "armv6l" -}} +{{.Name}}-armhf +{{- else if eq .Arch "armv7l" -}} +{{.Name}}-armhf +{{- else if eq .Arch "aarch64" -}} +{{.Name}}-arm64 +{{- end -}}`, + }, + //https://storage.googleapis.com/kubernetes-release/release/v1.18.0/bin/darwin/amd64/kubectl + { + Owner: "kubernetes", + Repo: "kubernetes", + Name: "kubectl", + Version: "v1.18.0", + URLTemplate: `{{$arch := "arm"}} + + {{- if eq .Arch "x86_64" -}} +{{$arch = "amd64"}} +{{- end -}} + +{{$ext := ""}} +{{$os := .OS}} + +{{ if HasPrefix .OS "ming" -}} +{{$ext = ".exe"}} +{{$os = "windows"}} +{{- end -}} + +https://storage.googleapis.com/kubernetes-release/release/{{.Version}}/bin/{{$os}}/{{$arch}}/kubectl{{$ext}}`, + }, + } + return tools +} + +// makeHTTPClient makes a HTTP client with good defaults for timeouts. +func makeHTTPClient(timeout *time.Duration, tlsInsecure bool) http.Client { + return makeHTTPClientWithDisableKeepAlives(timeout, tlsInsecure, false) +} + +// makeHTTPClientWithDisableKeepAlives makes a HTTP client with good defaults for timeouts. +func makeHTTPClientWithDisableKeepAlives(timeout *time.Duration, tlsInsecure bool, disableKeepAlives bool) http.Client { + client := http.Client{} + + if timeout != nil || tlsInsecure { + tr := &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DisableKeepAlives: disableKeepAlives, + } + + if timeout != nil { + client.Timeout = *timeout + tr.DialContext = (&net.Dialer{ + Timeout: *timeout, + }).DialContext + + tr.IdleConnTimeout = 120 * time.Millisecond + tr.ExpectContinueTimeout = 1500 * time.Millisecond + } + + if tlsInsecure { + tr.TLSClientConfig = &tls.Config{InsecureSkipVerify: tlsInsecure} + } + + tr.DisableKeepAlives = disableKeepAlives + + client.Transport = tr + } + + return client +} diff --git a/pkg/get/get_test.go b/pkg/get/get_test.go new file mode 100644 index 0000000..c46099f --- /dev/null +++ b/pkg/get/get_test.go @@ -0,0 +1,132 @@ +package get + +import "testing" + +const faasCLIVersion = "0.12.4" +const arch64bit = "x86_64" + +func Test_DownloadDarwin(t *testing.T) { + tools := MakeTools() + name := "faas-cli" + var tool *Tool + for _, target := range tools { + if name == target.Name { + tool = &target + break + } + } + + got, err := tool.GetURL("darwin", "", "") + if err != nil { + t.Fatal(err) + } + want := "https://github.com/openfaas/faas-cli/releases/download/" + faasCLIVersion + "/faas-cli-darwin" + if got != want { + t.Fatalf("want: %s, got: %s", want, got) + } +} + +func Test_DownloadKubectlDarwin(t *testing.T) { + tools := MakeTools() + name := "kubectl" + var tool *Tool + for _, target := range tools { + if name == target.Name { + tool = &target + break + } + } + + got, err := tool.GetURL("darwin", arch64bit, tool.Version) + if err != nil { + t.Fatal(err) + } + want := "https://storage.googleapis.com/kubernetes-release/release/v1.18.0/bin/darwin/amd64/kubectl" + if got != want { + t.Fatalf("want: %s, got: %s", want, got) + } +} + +func Test_DownloadKubectlLinux(t *testing.T) { + tools := MakeTools() + name := "kubectl" + var tool *Tool + for _, target := range tools { + if name == target.Name { + tool = &target + break + } + } + + got, err := tool.GetURL("linux", arch64bit, tool.Version) + if err != nil { + t.Fatal(err) + } + want := "https://storage.googleapis.com/kubernetes-release/release/v1.18.0/bin/linux/amd64/kubectl" + if got != want { + t.Fatalf("want: %s, got: %s", want, got) + } +} + +func Test_DownloadArmhf(t *testing.T) { + tools := MakeTools() + name := "faas-cli" + var tool *Tool + for _, target := range tools { + if name == target.Name { + tool = &target + break + } + } + + got, err := tool.GetURL("Linux", "armv7l", "") + if err != nil { + t.Fatal(err) + } + want := "https://github.com/openfaas/faas-cli/releases/download/" + faasCLIVersion + "/faas-cli-armhf" + if got != want { + t.Fatalf("want: %s, got: %s", want, got) + } +} + +func Test_DownloadArm64(t *testing.T) { + tools := MakeTools() + name := "faas-cli" + var tool *Tool + for _, target := range tools { + if name == target.Name { + tool = &target + break + } + } + + got, err := tool.GetURL("Linux", "aarch64", "") + if err != nil { + t.Fatal(err) + } + want := "https://github.com/openfaas/faas-cli/releases/download/" + faasCLIVersion + "/faas-cli-arm64" + if got != want { + t.Fatalf("want: %s, got: %s", want, got) + } +} + +func Test_DownloadWindows(t *testing.T) { + tools := MakeTools() + name := "faas-cli" + var tool *Tool + for _, target := range tools { + if name == target.Name { + tool = &target + break + } + } + + got, err := tool.GetURL("mingw64_nt-10.0-18362", arch64bit, "") + if err != nil { + t.Fatal(err) + } + want := "https://github.com/openfaas/faas-cli/releases/download/" + faasCLIVersion + "/faas-cli.exe" + if got != want { + t.Fatalf("want: %s, got: %s", want, got) + } +}