Modifies and adds scripts to run K8S tests on IBM Cloud cluster (#5022)

* Scripts for PSI minishift tests * added line at end of script * added line at end of script * emoved info as requested * Updates as per review feedback * deleting unwanted script as it was replaced with other files * Scripts for PSI minishift tests * Applied workaround to use env variable to select minikube or minishift * Set CLUSTER env variable to appply workaround for firewall not taking parameters * added curl for missing script and added misiing /, removed unnecessary cd * Separating minishift from merged minikube-minishift scripts due to firewall not taking parameters for script execution * Changes to run tests in IBM Cloud, changed TEST_EXEX_NODES=24 * remove non required changes (old stuff) and remove non required scripts (old stuff) * Modified and added scripts to run tests in IBM Cloud cluster * Additional changes for to run K8S test in IBM Cloud * Removed unnecessary line * changes to run tests on K8S cluster in IBM Cloud * ncorportaed changes based on feedback for IBM Cloud tests * Chnages based in feedback for IBM Cloud * updated string for home dir * removing test-integration-devfile as it ends with failures * Fixed if statement * changes to Makefile for number of nodes * added requested comment to Makefile about nodes if running in IBM Cloud
2025-10-19 03:06:19 +03:00 · 2021-09-08 08:39:21 -04:00
parent b34a68e7e2
commit b9765a4bee
4 changed files with 133 additions and 10 deletions
--- a/9
+++ b/9
@@ -15,10 +15,15 @@ TIMEOUT ?= 14400s

 # Env variable TEST_EXEC_NODES is used to pass spec execution type
 # (parallel or sequential) for ginkgo tests. To run the specs sequentially use
-# TEST_EXEC_NODES=1, otherwise by default the specs are run in parallel on 4 ginkgo test node.
+# TEST_EXEC_NODES=1, otherwise by default the specs are run in parallel on 4 ginkgo test node if running on PSI cluster or 24 nodes if running on IBM Cloud cluster.
+
 # NOTE: Any TEST_EXEC_NODES value greater than one runs the spec in parallel
 # on the same number of ginkgo test nodes.
-TEST_EXEC_NODES ?= 4
+ifdef TEST_EXEC_NODES
+   TEST_EXEC_NODES := $(TEST_EXEC_NODES)
+else
+   TEST_EXEC_NODES := 4
+endif

 # Slow spec threshold for ginkgo tests. After this time (in second), ginkgo marks test as slow
 SLOW_SPEC_THRESHOLD := 120
--- a/scripts/k8sibmc-all-tests.sh
+++ b/scripts/k8sibmc-all-tests.sh
@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+# Runs integration tests on K8S cluster hosted in IBM Cloud
+
+shout() {
+  set +x
+  echo -e "\n.---------------------------------------\n${1}\n'---------------------------------------\n"
+  set -x
+}
+
+set -ex
+
+# This is one of the variables injected by ci-firewall. Its purpose is to allow scripts to handle uniqueness as needed
+SCRIPT_IDENTITY=${SCRIPT_IDENTITY:-"def-id"}
+
+case ${1} in
+    k8s)
+        export TEST_EXEC_NODES="24"
+        ibmcloud login --apikey $IBMC_DEVELOPER_OCLOGIN_APIKEY -a cloud.ibm.com -r eu-de -g "Developer-CI-and-QE"
+        ibmcloud ks cluster config --cluster $IBMC_K8S_CLUSTER_ID
+
+        # Integration tests
+        shout "| Running integration Tests on Kubernetes cluster in IBM Cloud"
+        make test-cmd-project
+
+        shout "Cleaning up some leftover namespaces"
+
+        set +x
+        for i in $(kubectl get namespace -o name); do
+	        if [[ $i == "namespace/${SCRIPT_IDENTITY}"* ]]; then
+	            kubectl delete $i
+            fi
+        done
+        set -x
+
+        odo logout
+        ;;
+    *)
+        echo "Need parameter set to k8s"
+        exit 1
+        ;;
+esac
--- a/scripts/k8sibmc-setup-env.sh
+++ b/scripts/k8sibmc-setup-env.sh
@@ -0,0 +1,76 @@
+#!/usr/bin/env bash
+#Sets up requirements to run tests in K8S cluster hosted in the IBM Cloud
+
+shout() {
+  set +x
+  echo -e "\n.---------------------------------------\n${1}\n'---------------------------------------\n"
+  set -x
+}
+
+# Create a bin directory whereever script runs. This will be where all binaries that need to be in PATH will reside.
+export HOME="~/"
+export GOPATH="~/go"
+export GOBIN="$GOPATH/bin"
+mkdir -p $GOBIN
+# This is one of the variables injected by ci-firewall. Its purpose is to allow scripts to handle uniqueness as needed
+SCRIPT_IDENTITY=${SCRIPT_IDENTITY:-"def-id"}
+
+# Add GOBIN which is the bin dir we created earlier to PATH so any binaries there are automatically available in PATH
+export PATH=$PATH:$GOBIN
+
+# Prep for integration/e2e
+shout "Building odo binaries"
+make bin
+
+# copy built odo to GOBIN
+cp -avrf ./odo $GOBIN/
+
+setup_kubeconfig() {
+    # Login as admin to IBM Cloud and get kubeconfig file for K8S cluster
+    ibmcloud login --apikey $IBMC_ADMIN_OCLOGIN_APIKEY -a cloud.ibm.com -r eu-de -g "Developer-CI-and-QE"
+    ibmcloud ks cluster config --cluster $IBMC_K8S_CLUSTER_ID
+
+    export KUBECONFIG=$HOME/.kube/config
+    if [[ ! -f $KUBECONFIG ]]; then
+        echo "Could not find kubeconfig file"
+        exit 1
+    fi
+    if [[ ! -z $KUBECONFIG ]]; then
+        # Copy kubeconfig to current directory, to avoid clashes with other test runs
+        # Read and Write permission to current kubeconfig file
+        cp $KUBECONFIG "`pwd`/config"
+        chmod 640 "`pwd`/config"
+        export KUBECONFIG="`pwd`/config"
+    fi
+}
+
+case ${1} in
+    k8s)
+        setup_kubeconfig
+        ;;
+    *)
+        echo "<<< Need parameter set to K8S >>>"
+        exit 1
+        ;;
+esac
+
+### Applies to both K8S and minikube
+# Setup to find nessasary data from cluster setup
+## Constants
+SETUP_OPERATORS="./scripts/configure-cluster/common/setup-operators.sh"
+
+# The OLM Version
+export OLM_VERSION="v0.18.3"
+# Enable OLM for running operator tests
+curl -sL https://github.com/operator-framework/operator-lifecycle-manager/releases/download/$OLM_VERSION/install.sh | bash -s $OLM_VERSION
+
+set +x
+# Get kubectl cluster info
+kubectl cluster-info
+        
+set -x
+# Set kubernetes env var as true, to distinguish the platform inside the tests
+export KUBERNETES=true
+
+# Create Operators for Operator tests
+sh $SETUP_OPERATORS
--- a/scripts/minici-presubmit-all-tests.sh
+++ b/scripts/minici-presubmit-all-tests.sh
@@ -22,17 +22,18 @@ case $1 in
        export RUN_SCRIPT="scripts/minikube-minishift-all-tests.sh minikube"
        export TIMEOUT="4h00m"
        ;;
-    minishift)
-        export JOB_NAME="odo-minishift-pr-tests"
-        export SENDQUEUE="amqp.ci.queue.minishift.send"
-        export SENDTOPIC="amqp.ci.topic.minishift.send"
-        export EXCHANGE="amqp.ci.exchange.minishift.send"
-        export SETUP_SCRIPT="scripts/minikube-minishift-setup-env.sh minishift"
-        export RUN_SCRIPT="scripts/minikube-minishift-all-tests.sh minishift"
+    k8s)
+        #Removing minishift (dropped support for 3.11) and adding K8S on IBM Cloud
+        export JOB_NAME="odo-k8s-ibmc-pr-tests"
+        export SENDQUEUE="amqp.ci.queue.k8sibmc.send"
+        export SENDTOPIC="amqp.ci.topic.k8sibmc.send"
+        export EXCHANGE="amqp.ci.exchange.k8sibmc.send"
+        export SETUP_SCRIPT="scripts/k8sibmc-setup-env.sh k8s"
+        export RUN_SCRIPT="scripts/k8sibmc-all-tests.sh k8s"
        export TIMEOUT="4h00m"
        ;;
    *)
-        echo "Must pass minikube or minishift as paramater"
+        echo "Must pass minikube or k8s as paramater"
        exit 1
        ;;
 esac