From bd7f67a74a7a027d7158ef135d97e9f6592d6e94 Mon Sep 17 00:00:00 2001 From: Tolga Ceylan Date: Wed, 20 Jun 2018 10:55:05 -0700 Subject: [PATCH] fn: test scripts should use well defined ports (#1077) * fn: test scripts should use well defined ports Moved allocation of listener ports for mysql/minio/postgres to helper script with a list of service list names. * fn: makefile docker pull mysql version must match tests --- Makefile | 2 +- api/logs/log.go | 2 +- api_test.sh | 34 ++++--------------- helpers.sh | 89 ++++++++++++++++++++++++++++++++++++++++++++++--- system_test.sh | 38 +++------------------ test.sh | 27 ++++----------- 6 files changed, 103 insertions(+), 89 deletions(-) diff --git a/Makefile b/Makefile index 10f1855c8..43c491a5e 100644 --- a/Makefile +++ b/Makefile @@ -60,7 +60,7 @@ img-busybox: img-hello: docker pull fnproject/hello img-mysql: - docker pull mysql + docker pull mysql:5.7.22 img-postgres: docker pull postgres:9.3-alpine img-minio: diff --git a/api/logs/log.go b/api/logs/log.go index ae7e350de..231baa913 100644 --- a/api/logs/log.go +++ b/api/logs/log.go @@ -26,7 +26,7 @@ var providers []Provider // AddProvider globally registers a new LogStore provider func AddProvider(pf Provider) { - logrus.Info("Adding log provider %s", pf) + logrus.Infof("Adding log provider %s", pf) providers = append(providers, pf) } diff --git a/api_test.sh b/api_test.sh index 1f5a7f679..52638078a 100755 --- a/api_test.sh +++ b/api_test.sh @@ -1,35 +1,12 @@ #!/bin/bash set -exo pipefail +export CONTEXT="fn_api_tests" source ./helpers.sh +remove_containers ${CONTEXT} -remove_containers - -case "$1" in - "sqlite3" ) - rm -fr /tmp/fn_integration_tests.db - touch /tmp/fn_integration_tests.db - export FN_DB_URL="sqlite3:///tmp/fn_integration_tests.db" - ;; - - "mysql" ) - DB_CONTAINER="func-mysql-test" - docker rm -fv ${DB_CONTAINER} || echo No prev mysql test db container - docker run --name ${DB_CONTAINER} -p 3306:3306 -e MYSQL_DATABASE=funcs -e MYSQL_ROOT_PASSWORD=root -d mysql:5.7.22 - MYSQL_HOST=`host ${DB_CONTAINER}` - MYSQL_PORT=3306 - export FN_DB_URL="mysql://root:root@tcp(${MYSQL_HOST}:${MYSQL_PORT})/funcs" - ;; - - "postgres" ) - DB_CONTAINER="func-postgres-test" - docker rm -fv ${DB_CONTAINER} || echo No prev test db container - docker run --name ${DB_CONTAINER} -e "POSTGRES_DB=funcs" -e "POSTGRES_PASSWORD=root" -p 5432:5432 -d postgres:9.3-alpine - POSTGRES_HOST=`host ${DB_CONTAINER}` - POSTGRES_PORT=5432 - export FN_DB_URL="postgres://postgres:root@${POSTGRES_HOST}:${POSTGRES_PORT}/funcs?sslmode=disable" - ;; -esac +DB_NAME=$1 +export FN_DB_URL=$(spawn_${DB_NAME} ${CONTEXT}) #test test/fn-api-tests/fn-api-tests.test #status=`echo $?` @@ -49,7 +26,8 @@ esac #fi #pwd #./fn-api-tests.test -test.v -test.parallel ${2:-1} ./...; cd ../../ + export FN_DS_DB_PING_MAX_RETRIES=60 cd test/fn-api-tests && FN_API_URL="http://localhost:8080" FN_DB_URL=${FN_DB_URL} go test -v -parallel ${2:-1} ./...; cd ../../ -remove_containers \ No newline at end of file +remove_containers ${CONTEXT} diff --git a/helpers.sh b/helpers.sh index 23affb3e8..d412723bc 100644 --- a/helpers.sh +++ b/helpers.sh @@ -1,7 +1,7 @@ #!/bin/bash set -exo pipefail -function host { +function get_host { case ${DOCKER_LOCATION:-localhost} in localhost) echo "localhost" @@ -20,8 +20,87 @@ function host { esac } -function remove_containers { - docker rm -fv func-postgres-test 2>/dev/null || true - docker rm -fv func-mysql-test 2>/dev/null || true - docker rm -fv func-minio-test 2>/dev/null || true +function get_port { + local NAME=$1 + local PORT_START=${FN_TEST_PORT_RANGE_START:-33000} + + local SERVICE_LIST=( + "fn_basic_tests_minio" + "fn_basic_tests_mysql" + "fn_basic_tests_postgres" + "fn_api_tests_minio" + "fn_api_tests_mysql" + "fn_api_tests_postgres" + "fn_system_tests_minio" + "fn_system_tests_mysql" + "fn_system_tests_postgres" + ) + + local IDX=0 + while [ ${IDX} -lt ${#SERVICE_LIST[@]} ] + do + if [ ${SERVICE_LIST[$IDX]} = "${NAME}" ]; then + echo $((${PORT_START}+${IDX})) + return + fi + IDX=$(($IDX+1)) + done + + echo "Invalid context/component: ${NAME} not in service list" + exit -1 +} + +function spawn_sqlite3 { + local CONTEXT=$1 + touch /tmp/${CONTEXT}_sqllite3.db + echo "sqlite3:///tmp/${CONTEXT}_sqllite3.db" +} + +function spawn_mysql { + local CONTEXT=$1 + local PORT=$(get_port ${CONTEXT}_mysql) + local HOST=$(get_host ${CONTEXT}_mysql) + local ID=$(docker run --name ${CONTEXT}_mysql \ + -p ${PORT}:3306 \ + -e MYSQL_DATABASE=funcs \ + -e MYSQL_ROOT_PASSWORD=root \ + -d mysql:5.7.22) + + echo "mysql://root:root@tcp(${HOST}:${PORT})/funcs" +} + +function spawn_postgres { + local CONTEXT=$1 + local PORT=$(get_port ${CONTEXT}_postgres) + local HOST=$(get_host ${CONTEXT}_postgres) + local ID=$(docker run --name ${CONTEXT}_postgres \ + -e "POSTGRES_DB=funcs" \ + -e "POSTGRES_PASSWORD=root" \ + -p ${PORT}:5432 \ + -d postgres:9.3-alpine) + + echo "postgres://postgres:root@${HOST}:${PORT}/funcs?sslmode=disable" +} + +function spawn_minio { + local CONTEXT=$1 + local PORT=$(get_port ${CONTEXT}_minio) + local HOST=$(get_host ${CONTEXT}_minio) + local ID=$(docker run --name ${CONTEXT}_minio \ + -p ${PORT}:9000 \ + -e "MINIO_ACCESS_KEY=admin" \ + -e "MINIO_SECRET_KEY=password" \ + -d minio/minio server /data) + + echo "s3://admin:password@${HOST}:${PORT}/us-east-1/fnlogs" +} + +function remove_containers { + local CONTEXT=$1 + for i in mysql minio postgres + do + docker rm -fv ${CONTEXT}_${i} 2>/dev/null || true + done + + rm -f /tmp/${CONTEXT}_sqllite3.db } diff --git a/system_test.sh b/system_test.sh index 2f5cb8991..7678f8469 100755 --- a/system_test.sh +++ b/system_test.sh @@ -1,42 +1,12 @@ #!/bin/bash set -exo pipefail +export CONTEXT="fn_system_tests" source ./helpers.sh - -function remove_system_containers { - docker rm -fv func-mysql-system-test 2>/dev/null || true - docker rm -fv func-postgres-system-test 2>/dev/null || true -} - -remove_system_containers +remove_containers ${CONTEXT} DB_NAME=$1 - -case "$DB_NAME" in - "sqlite3" ) - rm -fr /tmp/fn_system_tests.db - touch /tmp/fn_system_tests.db - export FN_DB_URL="sqlite3:///tmp/fn_system_tests.db" - ;; - - "mysql" ) - DB_CONTAINER="func-mysql-system-test" - docker rm -fv ${DB_CONTAINER} || echo No prev mysql test db container - docker run --name ${DB_CONTAINER} -p 3307:3306 -e MYSQL_DATABASE=funcs -e MYSQL_ROOT_PASSWORD=root -d mysql:5.7.22 - MYSQL_HOST=`host ${DB_CONTAINER}` - MYSQL_PORT=3307 - export FN_DB_URL="mysql://root:root@tcp(${MYSQL_HOST}:${MYSQL_PORT})/funcs" - ;; - - "postgres" ) - DB_CONTAINER="func-postgres-system-test" - docker rm -fv ${DB_CONTAINER} || echo No prev test db container - docker run --name ${DB_CONTAINER} -e "POSTGRES_DB=funcs" -e "POSTGRES_PASSWORD=root" -p 5433:5432 -d postgres:9.3-alpine - POSTGRES_HOST=`host ${DB_CONTAINER}` - POSTGRES_PORT=5433 - export FN_DB_URL="postgres://postgres:root@${POSTGRES_HOST}:${POSTGRES_PORT}/funcs?sslmode=disable" - ;; -esac +export FN_DB_URL=$(spawn_${DB_NAME} ${CONTEXT}) # avoid port conflicts with api_test.sh which are run in parallel export FN_API_URL="http://localhost:8085" @@ -54,4 +24,4 @@ export SYSTEM_TEST_PROMETHEUS_FILE=./prometheus.${DB_NAME}.txt cd test/fn-system-tests && FN_DB_URL=${FN_DB_URL} FN_API_URL=${FN_API_URL} go test -v -parallel ${2:-1} ./...; cd ../../ -remove_system_containers +remove_containers ${CONTEXT} diff --git a/test.sh b/test.sh index d5dde653c..8c5100eb1 100755 --- a/test.sh +++ b/test.sh @@ -2,31 +2,18 @@ # Top level test script to start all other tests set -exuo pipefail - +export CONTEXT="fn_basic_tests" source ./helpers.sh +remove_containers ${CONTEXT} -remove_containers - -docker run --name func-postgres-test -e "POSTGRES_DB=funcs" -e "POSTGRES_PASSWORD=root" -p 5432:5432 -d postgres:9.3-alpine -docker run --name func-mysql-test -p 3306:3306 -e MYSQL_DATABASE=funcs -e MYSQL_ROOT_PASSWORD=root -d mysql:5.7.22 -docker run -d -p 9000:9000 --name func-minio-test -e "MINIO_ACCESS_KEY=admin" -e "MINIO_SECRET_KEY=password" minio/minio server /data - -MYSQL_HOST=`host func-mysql-test` -MYSQL_PORT=3306 - -POSTGRES_HOST=`host func-postgres-test` -POSTGRES_PORT=5432 - -MINIO_HOST=`host func-minio-test` -MINIO_PORT=9000 - -export POSTGRES_URL="postgres://postgres:root@${POSTGRES_HOST}:${POSTGRES_PORT}/funcs?sslmode=disable" -export MYSQL_URL="mysql://root:root@tcp(${MYSQL_HOST}:${MYSQL_PORT})/funcs" -export MINIO_URL="s3://admin:password@${MINIO_HOST}:${MINIO_PORT}/us-east-1/fnlogs" +export POSTGRES_URL=$(spawn_postgres ${CONTEXT}) +export MYSQL_URL=$(spawn_mysql ${CONTEXT}) +export MINIO_URL=$(spawn_minio ${CONTEXT}) +export FN_DS_DB_PING_MAX_RETRIES=60 go test -v $(go list ./... | grep -v vendor | grep -v examples | grep -v test/fn-api-tests | grep -v test/fn-system-tests | grep -v images/fn-test-utils) go vet $(go list ./... | grep -v vendor) -remove_containers +remove_containers ${CONTEXT} docker run -v `pwd`:/go/src/github.com/fnproject/fn --rm fnproject/swagger:0.0.1 /go/src/github.com/fnproject/fn/docs/swagger.yml