From b9765a4beefcd87dd5d5a11f020c05a58705816d Mon Sep 17 00:00:00 2001 From: Rodolfo Napoles Date: Wed, 8 Sep 2021 08:39:21 -0400 Subject: [PATCH] Modifies and adds scripts to run K8S tests on IBM Cloud cluster (#5022) * Scripts for PSI minishift tests * added line at end of script * added line at end of script * emoved info as requested * Updates as per review feedback * deleting unwanted script as it was replaced with other files * Scripts for PSI minishift tests * Applied workaround to use env variable to select minikube or minishift * Set CLUSTER env variable to appply workaround for firewall not taking parameters * added curl for missing script and added misiing /, removed unnecessary cd * Separating minishift from merged minikube-minishift scripts due to firewall not taking parameters for script execution * Changes to run tests in IBM Cloud, changed TEST_EXEX_NODES=24 * remove non required changes (old stuff) and remove non required scripts (old stuff) * Modified and added scripts to run tests in IBM Cloud cluster * Additional changes for to run K8S test in IBM Cloud * Removed unnecessary line * changes to run tests on K8S cluster in IBM Cloud * ncorportaed changes based on feedback for IBM Cloud tests * Chnages based in feedback for IBM Cloud * updated string for home dir * removing test-integration-devfile as it ends with failures * Fixed if statement * changes to Makefile for number of nodes * added requested comment to Makefile about nodes if running in IBM Cloud --- Makefile | 9 +++- scripts/k8sibmc-all-tests.sh | 41 +++++++++++++++ scripts/k8sibmc-setup-env.sh | 76 +++++++++++++++++++++++++++ scripts/minici-presubmit-all-tests.sh | 17 +++--- 4 files changed, 133 insertions(+), 10 deletions(-) create mode 100644 scripts/k8sibmc-all-tests.sh create mode 100644 scripts/k8sibmc-setup-env.sh diff --git a/Makefile b/Makefile index eb2bba423..ce0cb6ff4 100644 --- a/Makefile +++ b/Makefile @@ -15,10 +15,15 @@ TIMEOUT ?= 14400s # Env variable TEST_EXEC_NODES is used to pass spec execution type # (parallel or sequential) for ginkgo tests. To run the specs sequentially use -# TEST_EXEC_NODES=1, otherwise by default the specs are run in parallel on 4 ginkgo test node. +# TEST_EXEC_NODES=1, otherwise by default the specs are run in parallel on 4 ginkgo test node if running on PSI cluster or 24 nodes if running on IBM Cloud cluster. + # NOTE: Any TEST_EXEC_NODES value greater than one runs the spec in parallel # on the same number of ginkgo test nodes. -TEST_EXEC_NODES ?= 4 +ifdef TEST_EXEC_NODES + TEST_EXEC_NODES := $(TEST_EXEC_NODES) +else + TEST_EXEC_NODES := 4 +endif # Slow spec threshold for ginkgo tests. After this time (in second), ginkgo marks test as slow SLOW_SPEC_THRESHOLD := 120 diff --git a/scripts/k8sibmc-all-tests.sh b/scripts/k8sibmc-all-tests.sh new file mode 100644 index 000000000..498c38fe3 --- /dev/null +++ b/scripts/k8sibmc-all-tests.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# Runs integration tests on K8S cluster hosted in IBM Cloud + +shout() { + set +x + echo -e "\n.---------------------------------------\n${1}\n'---------------------------------------\n" + set -x +} + +set -ex + +# This is one of the variables injected by ci-firewall. Its purpose is to allow scripts to handle uniqueness as needed +SCRIPT_IDENTITY=${SCRIPT_IDENTITY:-"def-id"} + +case ${1} in + k8s) + export TEST_EXEC_NODES="24" + ibmcloud login --apikey $IBMC_DEVELOPER_OCLOGIN_APIKEY -a cloud.ibm.com -r eu-de -g "Developer-CI-and-QE" + ibmcloud ks cluster config --cluster $IBMC_K8S_CLUSTER_ID + + # Integration tests + shout "| Running integration Tests on Kubernetes cluster in IBM Cloud" + make test-cmd-project + + shout "Cleaning up some leftover namespaces" + + set +x + for i in $(kubectl get namespace -o name); do + if [[ $i == "namespace/${SCRIPT_IDENTITY}"* ]]; then + kubectl delete $i + fi + done + set -x + + odo logout + ;; + *) + echo "Need parameter set to k8s" + exit 1 + ;; +esac diff --git a/scripts/k8sibmc-setup-env.sh b/scripts/k8sibmc-setup-env.sh new file mode 100644 index 000000000..0b4267a7d --- /dev/null +++ b/scripts/k8sibmc-setup-env.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash +#Sets up requirements to run tests in K8S cluster hosted in the IBM Cloud + +shout() { + set +x + echo -e "\n.---------------------------------------\n${1}\n'---------------------------------------\n" + set -x +} + +# Create a bin directory whereever script runs. This will be where all binaries that need to be in PATH will reside. +export HOME="~/" +export GOPATH="~/go" +export GOBIN="$GOPATH/bin" +mkdir -p $GOBIN +# This is one of the variables injected by ci-firewall. Its purpose is to allow scripts to handle uniqueness as needed +SCRIPT_IDENTITY=${SCRIPT_IDENTITY:-"def-id"} + +# Add GOBIN which is the bin dir we created earlier to PATH so any binaries there are automatically available in PATH +export PATH=$PATH:$GOBIN + +# Prep for integration/e2e +shout "Building odo binaries" +make bin + +# copy built odo to GOBIN +cp -avrf ./odo $GOBIN/ + +setup_kubeconfig() { + # Login as admin to IBM Cloud and get kubeconfig file for K8S cluster + ibmcloud login --apikey $IBMC_ADMIN_OCLOGIN_APIKEY -a cloud.ibm.com -r eu-de -g "Developer-CI-and-QE" + ibmcloud ks cluster config --cluster $IBMC_K8S_CLUSTER_ID + + export KUBECONFIG=$HOME/.kube/config + if [[ ! -f $KUBECONFIG ]]; then + echo "Could not find kubeconfig file" + exit 1 + fi + if [[ ! -z $KUBECONFIG ]]; then + # Copy kubeconfig to current directory, to avoid clashes with other test runs + # Read and Write permission to current kubeconfig file + cp $KUBECONFIG "`pwd`/config" + chmod 640 "`pwd`/config" + export KUBECONFIG="`pwd`/config" + fi +} + +case ${1} in + k8s) + setup_kubeconfig + ;; + *) + echo "<<< Need parameter set to K8S >>>" + exit 1 + ;; +esac + +### Applies to both K8S and minikube +# Setup to find nessasary data from cluster setup +## Constants +SETUP_OPERATORS="./scripts/configure-cluster/common/setup-operators.sh" + +# The OLM Version +export OLM_VERSION="v0.18.3" +# Enable OLM for running operator tests +curl -sL https://github.com/operator-framework/operator-lifecycle-manager/releases/download/$OLM_VERSION/install.sh | bash -s $OLM_VERSION + +set +x +# Get kubectl cluster info +kubectl cluster-info + +set -x +# Set kubernetes env var as true, to distinguish the platform inside the tests +export KUBERNETES=true + +# Create Operators for Operator tests +sh $SETUP_OPERATORS diff --git a/scripts/minici-presubmit-all-tests.sh b/scripts/minici-presubmit-all-tests.sh index 3c968865b..007a78963 100755 --- a/scripts/minici-presubmit-all-tests.sh +++ b/scripts/minici-presubmit-all-tests.sh @@ -22,17 +22,18 @@ case $1 in export RUN_SCRIPT="scripts/minikube-minishift-all-tests.sh minikube" export TIMEOUT="4h00m" ;; - minishift) - export JOB_NAME="odo-minishift-pr-tests" - export SENDQUEUE="amqp.ci.queue.minishift.send" - export SENDTOPIC="amqp.ci.topic.minishift.send" - export EXCHANGE="amqp.ci.exchange.minishift.send" - export SETUP_SCRIPT="scripts/minikube-minishift-setup-env.sh minishift" - export RUN_SCRIPT="scripts/minikube-minishift-all-tests.sh minishift" + k8s) + #Removing minishift (dropped support for 3.11) and adding K8S on IBM Cloud + export JOB_NAME="odo-k8s-ibmc-pr-tests" + export SENDQUEUE="amqp.ci.queue.k8sibmc.send" + export SENDTOPIC="amqp.ci.topic.k8sibmc.send" + export EXCHANGE="amqp.ci.exchange.k8sibmc.send" + export SETUP_SCRIPT="scripts/k8sibmc-setup-env.sh k8s" + export RUN_SCRIPT="scripts/k8sibmc-all-tests.sh k8s" export TIMEOUT="4h00m" ;; *) - echo "Must pass minikube or minishift as paramater" + echo "Must pass minikube or k8s as paramater" exit 1 ;; esac