Add URL checker

Fixes: #557
Closes: #628 - thanks to @yankeexe for his suggestions

Runs every day at midnight, or upon new commits / PRs.

Signed-off-by: Alex Ellis (OpenFaaS Ltd) <alexellis2@gmail.com>

.github/workflows/e2e-url-checker.yml

@@ -0,0 +1,29 @@
+name: e2e-url-checker
+
+on:
+  schedule:
+    - cron: '0 0 * * *'
+  push:
+    branches:
+    - '*'
+  pull_request:
+    branches:
+    - '*'
+
+jobs:
+  build:
+    strategy:
+      matrix:
+        go-version: [1.17.x]
+        os: [ubuntu-latest]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@master
+        with:
+          fetch-depth: 1
+      - name: Install Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: ${{ matrix.go-version }}
+      - name: Run URL checker tests
+        run: make e2e

CONTRIBUTING.md

@@ -52,7 +52,12 @@ git remote add fork https://github.com/NAME/arkade
 ```bash
 gofmt -w -s ./pkg
 gofmt -w -s ./cmd
-go test ./...
+
+# Run all the unit tests
+make test
+
+# Use e2e tests ot check that URLs can be downloaded for all tools
+make e2e
 ```
 
 #### Checkout a branch to start work

Makefile

@@ -14,12 +14,16 @@ build:
 
 .PHONY: gofmt
 gofmt:
-	@test -z $(shell gofmt -l -s $(SOURCE_DIRS) ./ | tee /dev/stderr) || (echo "[WARN] Fix formatting issues with 'make fmt'" && exit 1)
+	@test -z $(shell gofmt -l -s $(SOURCE_DIRS) ./ | tee /dev/stderr) || (echo "[WARN] Fix formatting issues with 'make gofmt'" && exit 1)
 
 .PHONY: test
 test:
 	CGO_ENABLED=0 go test $(shell go list ./... | grep -v /vendor/|xargs echo) -cover
 
+.PHONY: e2e
+e2e:
+	CGO_ENABLED=0 go test github.com/alexellis/arkade/pkg/get -cover  --tags e2e
+
 .PHONY: dist
 dist:
 	mkdir -p bin

pkg/get/get.go

@@ -5,6 +5,7 @@ import (
 	"crypto/tls"
 	"errors"
 	"fmt"
+	"io/ioutil"
 	"log"
 	"net"
 	"net/http"
@@ -100,6 +101,26 @@ func getToolVersion(tool *Tool, version string) string {
 	return ver
 }
 
+func (tool Tool) Head(uri string) (int, string, http.Header, error) {
+	req, err := http.NewRequest(http.MethodHead, uri, nil)
+	if err != nil {
+		return http.StatusBadRequest, "", nil, err
+	}
+
+	res, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return http.StatusBadRequest, "", nil, err
+	}
+
+	var body string
+	if res.Body != nil {
+		b, _ := ioutil.ReadAll(res.Body)
+		body = string(b)
+	}
+
+	return res.StatusCode, body, res.Header, nil
+}
+
 func (tool Tool) GetURL(os, arch, version string) (string, error) {
 
 	if len(version) == 0 &&

pkg/get/url_checker_test.go

@@ -0,0 +1,44 @@
+//go:build e2e
+// +build e2e
+
+package get
+
+import (
+	"net/http"
+	"testing"
+)
+
+// Test_CheckTools runs end to end tests to verify the URLS for various tools using a HTTP head request.
+
+func Test_CheckTools(t *testing.T) {
+	tools := MakeTools()
+
+	os := "linux"
+	arch := "x86_64"
+
+	for _, toolV := range tools {
+		tool := toolV
+		t.Run("Download of "+tool.Name, func(t *testing.T) {
+			t.Parallel()
+
+			url, err := tool.GetURL(os, arch, tool.Version)
+			if err != nil {
+				t.Fatalf("Error getting url for %s: %s", tool.Name, err)
+			}
+			t.Logf("Checking %s via %s", tool.Name, url)
+
+			status, body, headers, err := tool.Head(url)
+			if err != nil {
+				t.Fatalf("Error with HTTP HEAD for %s, %s: %s", tool.Name, url, err)
+			}
+
+			if status != http.StatusOK {
+				t.Fatalf("Error with HTTP HEAD for %s, %s: status code: %d, body: %s", tool.Name, url, status, body)
+			}
+
+			if headers.Get("Content-Length") == "" {
+				t.Fatalf("Error with HTTP HEAD for %s, %s: content-length zero", tool.Name, url)
+			}
+		})
+	}
+}