fn: circleci and makefile adjustments (#625)

* fn: circleci and makefile adjustments

*) Moved more tasks into Makefile to allow for
parallelism and dependency checks.
*) Added cpu count in circleci make invocations
for parallelism

* fn: typo sqlite => sqlite3

* fn: removed unnecessary make pull & install

.circleci/config.yml

@@ -27,14 +27,9 @@ jobs:
           sudo service docker stop
           curl -fsSL https://get.docker.com/ | sudo sh
       - run: docker version
-      - run: make docker-build
-      - run: make install
-      - run: make test
-      - run: ./go-fmt.sh
-      # TODO these should be inside test.sh file ?
-      - run: ./api_test.sh mysql 4
-      - run: ./api_test.sh postgres 4
-      - run: ./api_test.sh sqlite 4
+      - run: nproc
+      - run: make full-test -j $(nproc)
+      - run: make docker-build -j $(nproc)
       - deploy:
           command: |
             if [[ "${CIRCLE_BRANCH}" == "master" && -z "${CIRCLE_PR_REPONAME}" ]]; then

Makefile

@@ -1,5 +1,5 @@
 # Just builds
-.PHONY: all test dep build test-log-datastore
+.PHONY: all test dep build test-log-datastore checkfmt pull-images api-test fn-test-utils test-middleware test-extensions test-basic test-api
 
 dep:
 	glide install -v
@@ -13,9 +13,47 @@ build:
 install:
 	go build -o ${GOPATH}/bin/fnserver
 
-test:
+checkfmt:
+	./go-fmt.sh
+
+fn-test-utils: checkfmt
+	cd images/fn-test-utils && ./build.sh
+
+test-middleware: test-basic
+	cd examples/middleware && go build
+
+test-extensions: test-basic
+	cd examples/extensions && go build
+
+test-basic: checkfmt pull-images fn-test-utils
 	./test.sh
 
+test: checkfmt pull-images test-basic test-middleware test-extensions
+
+test-api: test-basic
+	./api_test.sh mysql 4
+	./api_test.sh postgres 4
+	./api_test.sh sqlite3 4
+
+full-test: test test-api
+
+img-sleeper:
+	docker pull fnproject/sleeper
+img-error:
+	docker pull fnproject/error
+img-hello:
+	docker pull fnproject/hello
+img-swagger:
+	docker pull quay.io/goswagger/swagger
+img-mysql:
+	docker pull mysql
+img-postgres:
+	docker pull postgres
+img-minio:
+	docker pull minio/minio
+
+pull-images: img-sleeper img-error img-hello img-swagger img-mysql img-postgres img-minio
+
 test-datastore:
 	cd api/datastore && go test -v ./...
 

api_test.sh

@@ -1,4 +1,5 @@
-set -ex
+#!/bin/bash
+set -exuo pipefail
 
 function host {
     case ${DOCKER_LOCATION:-localhost} in

images/fn-test-utils/release.sh

@@ -1,4 +1,2 @@
 set -e
-
-./build.sh
 docker push fnproject/fn-test-utils:latest

test.sh

@@ -22,21 +22,18 @@ function host {
     esac
 }
 
-docker rm -fv func-postgres-test || echo No prev test db container
+function remove_containers {
+    docker rm -fv func-postgres-test 2>/dev/null || true
+    docker rm -fv func-mysql-test 2>/dev/null || true
+    docker rm -fv func-minio-test 2>/dev/null || true
+}
+
+remove_containers
+
 docker run --name func-postgres-test -e "POSTGRES_DB=funcs" -e "POSTGRES_PASSWORD=root" -p 5432:5432 -d postgres
-docker rm -fv func-mysql-test || echo No prev mysql test db container
 docker run --name func-mysql-test -p 3306:3306 -e MYSQL_DATABASE=funcs -e MYSQL_ROOT_PASSWORD=root -d mysql
-docker rm -fv func-minio-test || echo No prev minio test container
 docker run -d -p 9000:9000 --name func-minio-test -e "MINIO_ACCESS_KEY=admin" -e "MINIO_SECRET_KEY=password" minio/minio server /data
 
-# build test image locally first
-(cd images/fn-test-utils && ./build.sh)
-
-# pull all images used in tests so that tests don't time out and fail spuriously
-docker pull fnproject/sleeper
-docker pull fnproject/error
-docker pull fnproject/hello
-
 MYSQL_HOST=`host func-mysql-test`
 MYSQL_PORT=3306
 
@@ -52,17 +49,8 @@ export MINIO_URL="s3://admin:password@${MINIO_HOST}:${MINIO_PORT}/us-east-1/fnlo
 
 go test $(go list ./... | grep -v vendor | grep -v examples | grep -v test/fn-api-tests | grep -v images/fn-test-utils)
 go vet $(go list ./... | grep -v vendor)
-docker rm --force func-postgres-test
-docker rm --force func-mysql-test
-docker rm --force func-minio-test
+
+remove_containers
 
 docker run -v `pwd`:/go/src/github.com/fnproject/fn --rm  quay.io/goswagger/swagger validate /go/src/github.com/fnproject/fn/docs/swagger.yml
 
-# test middlware, extensions, examples, etc
-# TODO: do more here, maybe as part of fn tests
-cd examples/middleware
-go build
-cd ../..
-cd examples/extensions
-go build
-cd ../..