fn: test scripts should use well defined ports (#1077)

* fn: test scripts should use well defined ports

Moved allocation of listener ports for mysql/minio/postgres
to helper script with a list of service list names.

* fn: makefile docker pull mysql version must match tests

Makefile

@@ -60,7 +60,7 @@ img-busybox:
 img-hello:
 	docker pull fnproject/hello
 img-mysql:
-	docker pull mysql
+	docker pull mysql:5.7.22
 img-postgres:
 	docker pull postgres:9.3-alpine
 img-minio:

api/logs/log.go

@@ -26,7 +26,7 @@ var providers []Provider
 
 // AddProvider globally registers a new LogStore provider
 func AddProvider(pf Provider) {
-	logrus.Info("Adding log provider %s", pf)
+	logrus.Infof("Adding log provider %s", pf)
 	providers = append(providers, pf)
 }
 

api_test.sh

@@ -1,35 +1,12 @@
 #!/bin/bash
 set -exo pipefail
 
+export CONTEXT="fn_api_tests"
 source ./helpers.sh
+remove_containers ${CONTEXT}
 
-remove_containers
+DB_NAME=$1
-
+export FN_DB_URL=$(spawn_${DB_NAME} ${CONTEXT})
-case "$1" in
-    "sqlite3" )
-    rm -fr /tmp/fn_integration_tests.db
-    touch /tmp/fn_integration_tests.db
-    export FN_DB_URL="sqlite3:///tmp/fn_integration_tests.db"
-    ;;
-
-    "mysql" )
-    DB_CONTAINER="func-mysql-test"
-    docker rm -fv ${DB_CONTAINER} || echo No prev mysql test db container
-    docker run --name ${DB_CONTAINER} -p 3306:3306 -e MYSQL_DATABASE=funcs -e MYSQL_ROOT_PASSWORD=root -d mysql:5.7.22
-    MYSQL_HOST=`host ${DB_CONTAINER}`
-    MYSQL_PORT=3306
-    export FN_DB_URL="mysql://root:root@tcp(${MYSQL_HOST}:${MYSQL_PORT})/funcs"
-    ;;
-
-    "postgres" )
-    DB_CONTAINER="func-postgres-test"
-    docker rm -fv ${DB_CONTAINER} || echo No prev test db container
-    docker run --name ${DB_CONTAINER} -e "POSTGRES_DB=funcs" -e "POSTGRES_PASSWORD=root"  -p 5432:5432 -d postgres:9.3-alpine
-    POSTGRES_HOST=`host ${DB_CONTAINER}`
-    POSTGRES_PORT=5432
-    export FN_DB_URL="postgres://postgres:root@${POSTGRES_HOST}:${POSTGRES_PORT}/funcs?sslmode=disable"
-    ;;
-esac
 
 #test test/fn-api-tests/fn-api-tests.test
 #status=`echo $?`
@@ -49,7 +26,8 @@ esac
 #fi
 #pwd
 #./fn-api-tests.test -test.v  -test.parallel ${2:-1} ./...; cd ../../
+
 export FN_DS_DB_PING_MAX_RETRIES=60
 cd test/fn-api-tests && FN_API_URL="http://localhost:8080"  FN_DB_URL=${FN_DB_URL} go test -v  -parallel ${2:-1} ./...; cd ../../
 
-remove_containers
+remove_containers ${CONTEXT}

helpers.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 set -exo pipefail
 
-function host {
+function get_host {
     case ${DOCKER_LOCATION:-localhost} in
     localhost)
         echo "localhost"
@@ -20,8 +20,87 @@ function host {
     esac
 }
 
-function remove_containers {
+function get_port {
-    docker rm -fv func-postgres-test 2>/dev/null || true
+    local NAME=$1
-    docker rm -fv func-mysql-test 2>/dev/null || true
+    local PORT_START=${FN_TEST_PORT_RANGE_START:-33000}
-    docker rm -fv func-minio-test 2>/dev/null || true
+
+    local SERVICE_LIST=(
+        "fn_basic_tests_minio"
+        "fn_basic_tests_mysql"
+        "fn_basic_tests_postgres"
+        "fn_api_tests_minio"
+        "fn_api_tests_mysql"
+        "fn_api_tests_postgres"
+        "fn_system_tests_minio"
+        "fn_system_tests_mysql"
+        "fn_system_tests_postgres"
+    )
+
+    local IDX=0
+    while [ ${IDX} -lt ${#SERVICE_LIST[@]} ]
+    do
+        if [ ${SERVICE_LIST[$IDX]} = "${NAME}" ]; then
+            echo $((${PORT_START}+${IDX}))
+            return
+        fi
+        IDX=$(($IDX+1))
+    done
+
+    echo "Invalid context/component: ${NAME} not in service list"
+    exit -1
+}
+
+function spawn_sqlite3 {
+    local CONTEXT=$1
+    touch /tmp/${CONTEXT}_sqllite3.db
+    echo "sqlite3:///tmp/${CONTEXT}_sqllite3.db"
+}
+
+function spawn_mysql {
+    local CONTEXT=$1
+    local PORT=$(get_port ${CONTEXT}_mysql)
+    local HOST=$(get_host ${CONTEXT}_mysql)
+    local ID=$(docker run --name ${CONTEXT}_mysql \
+        -p ${PORT}:3306 \
+        -e MYSQL_DATABASE=funcs \
+        -e MYSQL_ROOT_PASSWORD=root \
+        -d mysql:5.7.22)
+
+    echo "mysql://root:root@tcp(${HOST}:${PORT})/funcs"
+}
+
+function spawn_postgres {
+    local CONTEXT=$1
+    local PORT=$(get_port ${CONTEXT}_postgres)
+    local HOST=$(get_host ${CONTEXT}_postgres)
+    local ID=$(docker run --name ${CONTEXT}_postgres \
+        -e "POSTGRES_DB=funcs" \
+        -e "POSTGRES_PASSWORD=root" \
+        -p ${PORT}:5432 \
+        -d postgres:9.3-alpine)
+
+    echo "postgres://postgres:root@${HOST}:${PORT}/funcs?sslmode=disable"
+}
+
+function spawn_minio {
+    local CONTEXT=$1
+    local PORT=$(get_port ${CONTEXT}_minio)
+    local HOST=$(get_host ${CONTEXT}_minio)
+    local ID=$(docker run --name ${CONTEXT}_minio \
+        -p ${PORT}:9000 \
+        -e "MINIO_ACCESS_KEY=admin" \
+        -e "MINIO_SECRET_KEY=password" \
+        -d minio/minio server /data)
+
+    echo "s3://admin:password@${HOST}:${PORT}/us-east-1/fnlogs"
+}
+
+function remove_containers {
+    local CONTEXT=$1
+    for i in mysql minio postgres
+    do
+        docker rm -fv ${CONTEXT}_${i} 2>/dev/null || true
+    done
+
+    rm -f /tmp/${CONTEXT}_sqllite3.db
 }

system_test.sh

@@ -1,42 +1,12 @@
 #!/bin/bash
 set -exo pipefail
 
+export CONTEXT="fn_system_tests"
 source ./helpers.sh
-
+remove_containers ${CONTEXT}
-function remove_system_containers {
-    docker rm -fv func-mysql-system-test 2>/dev/null || true
-    docker rm -fv func-postgres-system-test 2>/dev/null || true
-}
-
-remove_system_containers
 
 DB_NAME=$1
-
+export FN_DB_URL=$(spawn_${DB_NAME} ${CONTEXT})
-case "$DB_NAME" in
-    "sqlite3" )
-    rm -fr /tmp/fn_system_tests.db
-    touch /tmp/fn_system_tests.db
-    export FN_DB_URL="sqlite3:///tmp/fn_system_tests.db"
-    ;;
-
-    "mysql" )
-    DB_CONTAINER="func-mysql-system-test"
-    docker rm -fv ${DB_CONTAINER} || echo No prev mysql test db container
-    docker run --name ${DB_CONTAINER} -p 3307:3306 -e MYSQL_DATABASE=funcs -e MYSQL_ROOT_PASSWORD=root -d mysql:5.7.22
-    MYSQL_HOST=`host ${DB_CONTAINER}`
-    MYSQL_PORT=3307
-    export FN_DB_URL="mysql://root:root@tcp(${MYSQL_HOST}:${MYSQL_PORT})/funcs"
-    ;;
-
-    "postgres" )
-    DB_CONTAINER="func-postgres-system-test"
-    docker rm -fv ${DB_CONTAINER} || echo No prev test db container
-    docker run --name ${DB_CONTAINER} -e "POSTGRES_DB=funcs" -e "POSTGRES_PASSWORD=root"  -p 5433:5432 -d postgres:9.3-alpine
-    POSTGRES_HOST=`host ${DB_CONTAINER}`
-    POSTGRES_PORT=5433
-    export FN_DB_URL="postgres://postgres:root@${POSTGRES_HOST}:${POSTGRES_PORT}/funcs?sslmode=disable"
-    ;;
-esac
 
 # avoid port conflicts with api_test.sh which are run in parallel
 export FN_API_URL="http://localhost:8085"
@@ -54,4 +24,4 @@ export SYSTEM_TEST_PROMETHEUS_FILE=./prometheus.${DB_NAME}.txt
 
 cd test/fn-system-tests && FN_DB_URL=${FN_DB_URL} FN_API_URL=${FN_API_URL} go test -v -parallel ${2:-1} ./...; cd ../../
 
-remove_system_containers
+remove_containers ${CONTEXT}

test.sh

@@ -2,31 +2,18 @@
 # Top level test script to start all other tests
 set -exuo pipefail
 
-
+export CONTEXT="fn_basic_tests"
 source ./helpers.sh
+remove_containers ${CONTEXT}
 
-remove_containers
+export POSTGRES_URL=$(spawn_postgres ${CONTEXT})
-
+export MYSQL_URL=$(spawn_mysql ${CONTEXT})
-docker run --name func-postgres-test -e "POSTGRES_DB=funcs" -e "POSTGRES_PASSWORD=root" -p 5432:5432 -d postgres:9.3-alpine
+export MINIO_URL=$(spawn_minio ${CONTEXT})
-docker run --name func-mysql-test -p 3306:3306 -e MYSQL_DATABASE=funcs -e MYSQL_ROOT_PASSWORD=root -d mysql:5.7.22
+export FN_DS_DB_PING_MAX_RETRIES=60
-docker run -d -p 9000:9000 --name func-minio-test -e "MINIO_ACCESS_KEY=admin" -e "MINIO_SECRET_KEY=password" minio/minio server /data
-
-MYSQL_HOST=`host func-mysql-test`
-MYSQL_PORT=3306
-
-POSTGRES_HOST=`host func-postgres-test`
-POSTGRES_PORT=5432
-
-MINIO_HOST=`host func-minio-test`
-MINIO_PORT=9000
-
-export POSTGRES_URL="postgres://postgres:root@${POSTGRES_HOST}:${POSTGRES_PORT}/funcs?sslmode=disable"
-export MYSQL_URL="mysql://root:root@tcp(${MYSQL_HOST}:${MYSQL_PORT})/funcs"
-export MINIO_URL="s3://admin:password@${MINIO_HOST}:${MINIO_PORT}/us-east-1/fnlogs"
 
 go test -v $(go list ./... | grep -v vendor | grep -v examples | grep -v test/fn-api-tests | grep -v test/fn-system-tests | grep -v images/fn-test-utils)
 go vet $(go list ./... | grep -v vendor)
 
-remove_containers
+remove_containers ${CONTEXT}
 
 docker run -v `pwd`:/go/src/github.com/fnproject/fn --rm fnproject/swagger:0.0.1 /go/src/github.com/fnproject/fn/docs/swagger.yml