diff --git a/Makefile b/Makefile index 1a87a0fea..466bb9ace 100644 --- a/Makefile +++ b/Makefile @@ -213,6 +213,10 @@ openshiftci-presubmit-unittests: test-integration-cluster: $(RUN_GINKGO) $(GINKGO_FLAGS) --junit-report="test-integration.xml" --label-filter="!unauth && !nocluster && !podman" tests/integration +.PHONY: test-integration-openshift +test-integration-openshift: + $(RUN_GINKGO) $(GINKGO_FLAGS) --junit-report="test-integration.xml" --label-filter="!unauth && !nocluster && !podman && !skiponopenshift" tests/integration + .PHONY: test-integration-cluster-no-service-binding test-integration-cluster-no-service-binding: $(RUN_GINKGO) $(GINKGO_FLAGS) --junit-report="test-integration.xml" --label-filter="!unauth && !nocluster && !podman &&!servicebinding" tests/integration diff --git a/scripts/openshiftci-Nightly-SBO-tests.sh b/scripts/openshiftci-Nightly-SBO-tests.sh index ac56b1aef..38799f110 100755 --- a/scripts/openshiftci-Nightly-SBO-tests.sh +++ b/scripts/openshiftci-Nightly-SBO-tests.sh @@ -28,7 +28,12 @@ oc login -u developer -p password@123 --insecure-skip-tls-verify oc whoami # Cluster related integration tests -make test-integration-cluster -make test-e2e +make test-integration-openshift || error=true + +make test-e2e || error=true + +if [ $error ]; then + exit -1 +fi oc logout diff --git a/scripts/openshiftci-periodic-tests.sh b/scripts/openshiftci-periodic-tests.sh index 30db6af51..efdb336cf 100755 --- a/scripts/openshiftci-periodic-tests.sh +++ b/scripts/openshiftci-periodic-tests.sh @@ -27,7 +27,7 @@ oc login -u developer -p password@123 --insecure-skip-tls-verify oc whoami # Cluster related Integration tests -make test-integration-cluster || error=true +make test-integration-openshift || error=true # E2e tests make test-e2e || error=true @@ -36,6 +36,4 @@ if [ $error ]; then exit -1 fi - - oc logout diff --git a/scripts/openshiftci-presubmit-all-tests.sh b/scripts/openshiftci-presubmit-all-tests.sh index cc43c8ab4..60edf100c 100755 --- a/scripts/openshiftci-presubmit-all-tests.sh +++ b/scripts/openshiftci-presubmit-all-tests.sh @@ -36,32 +36,22 @@ oc whoami # We want to use a stable Devfile registry for InterOP testing, and so we use the custom Devfile Registry setup on IBM cloud source ./scripts/openshiftci-config.sh -if [ "${ARCH}" == "s390x" ]; then - # Integration tests - make test-integration - # E2e tests - make test-e2e -elif [ "${ARCH}" == "ppc64le" ]; then - # Integration tests - make test-integration - # E2e tests - make test-e2e -else - # Integration tests - make test-integration || error=true - # E2e tests - make test-e2e || error=true +# Integration tests +make test-integration-openshift || error=true - # Fail the build if there is any error while test execution - if [ $error ]; then - exit -1 - fi +# E2e tests +make test-e2e || error=true + +# Fail the build if there is any error while test execution +if [ $error ]; then + exit -1 fi if [ ! -z "$ARTIFACT_DIR" ] then #copy artifact to $ARTIFACT_DIR if ARTIFACT_DIR var is exposed - cp -r test-*.xml $ARTIFACT_DIR + cp -vr test-*.xml $ARTIFACT_DIR || true fi + oc logout diff --git a/tests/helper/labels.go b/tests/helper/labels.go index e90f43174..eb1972216 100644 --- a/tests/helper/labels.go +++ b/tests/helper/labels.go @@ -5,10 +5,11 @@ import ( ) const ( - LabelNoCluster = "nocluster" - LabelUnauth = "unauth" - LabelPodman = "podman" - LabelServiceBinding = "servicebinding" + LabelNoCluster = "nocluster" + LabelUnauth = "unauth" + LabelPodman = "podman" + LabelServiceBinding = "servicebinding" + LabelSkipOnOpenShift = "skiponopenshift" ) func NeedsCluster(labels []string) bool { diff --git a/tests/integration/cmd_add_binding_test.go b/tests/integration/cmd_add_binding_test.go index d395956b1..67ea47a2b 100644 --- a/tests/integration/cmd_add_binding_test.go +++ b/tests/integration/cmd_add_binding_test.go @@ -11,7 +11,7 @@ import ( "github.com/redhat-developer/odo/tests/helper" ) -var _ = Describe("odo add binding command tests", Label(helper.LabelServiceBinding), func() { +var _ = Describe("odo add binding command tests", Label(helper.LabelServiceBinding), Label(helper.LabelSkipOnOpenShift), func() { // TODO: Add integration tests for the following cases when SBO is not installed // * servicebinding with envvars does not send `odo dev` in an infinite loop // * `odo dev` deletes service binding secrets for SB that is not present locally diff --git a/tests/integration/cmd_delete_test.go b/tests/integration/cmd_delete_test.go index 3170b189e..971c64584 100644 --- a/tests/integration/cmd_delete_test.go +++ b/tests/integration/cmd_delete_test.go @@ -15,7 +15,7 @@ import ( "github.com/redhat-developer/odo/tests/helper" ) -var _ = Describe("odo delete command tests", func() { +var _ = Describe("odo delete command tests", Label(helper.LabelSkipOnOpenShift), func() { var commonVar helper.CommonVar var cmpName, deploymentName, serviceName string var getDeployArgs, getSVCArgs []string diff --git a/tests/integration/cmd_describe_component_test.go b/tests/integration/cmd_describe_component_test.go index d20e498f2..713aacefb 100644 --- a/tests/integration/cmd_describe_component_test.go +++ b/tests/integration/cmd_describe_component_test.go @@ -13,7 +13,7 @@ import ( "github.com/redhat-developer/odo/tests/helper" ) -var _ = Describe("odo describe component command tests", func() { +var _ = Describe("odo describe component command tests", Label(helper.LabelSkipOnOpenShift), func() { var commonVar helper.CommonVar var cmpName string diff --git a/tests/integration/cmd_describe_list_binding_test.go b/tests/integration/cmd_describe_list_binding_test.go index 3e4ff26c1..026c21235 100644 --- a/tests/integration/cmd_describe_list_binding_test.go +++ b/tests/integration/cmd_describe_list_binding_test.go @@ -12,7 +12,7 @@ import ( "github.com/redhat-developer/odo/tests/helper" ) -var _ = Describe("odo describe/list binding command tests", Label(helper.LabelServiceBinding), func() { +var _ = Describe("odo describe/list binding command tests", Label(helper.LabelServiceBinding), Label(helper.LabelSkipOnOpenShift), func() { var commonVar helper.CommonVar // This is run before every Spec (It) diff --git a/tests/integration/cmd_dev_api_server_test.go b/tests/integration/cmd_dev_api_server_test.go index 66c622aef..cbeacf4ec 100644 --- a/tests/integration/cmd_dev_api_server_test.go +++ b/tests/integration/cmd_dev_api_server_test.go @@ -17,7 +17,7 @@ import ( "github.com/redhat-developer/odo/tests/helper" ) -var _ = Describe("odo dev command with api server tests", func() { +var _ = Describe("odo dev command with api server tests", Label(helper.LabelSkipOnOpenShift), func() { var cmpName string var commonVar helper.CommonVar diff --git a/tests/integration/cmd_dev_debug_test.go b/tests/integration/cmd_dev_debug_test.go index 088bcbd73..9d3b8c836 100644 --- a/tests/integration/cmd_dev_debug_test.go +++ b/tests/integration/cmd_dev_debug_test.go @@ -36,7 +36,7 @@ var _ = Describe("odo dev debug command tests", func() { for _, podman := range []bool{false, true} { podman := podman - When("a component is bootstrapped", func() { + When("a component is bootstrapped", Label(helper.LabelSkipOnOpenShift), func() { BeforeEach(func() { helper.CopyExample(filepath.Join("source", "devfiles", "nodejs", "project"), commonVar.Context) helper.Cmd("odo", "init", "--name", cmpName, "--devfile-path", helper.GetExamplePath("source", "devfiles", "nodejs", "devfile-with-debugrun.yaml")).ShouldPass() @@ -132,7 +132,7 @@ var _ = Describe("odo dev debug command tests", func() { for _, podman := range []bool{false, true} { podman := podman - When("creating nodejs component, doing odo dev and run command has dev.odo.push.path attribute", helper.LabelPodmanIf(podman, func() { + When("creating nodejs component, doing odo dev and run command has dev.odo.push.path attribute", Label(helper.LabelSkipOnOpenShift), helper.LabelPodmanIf(podman, func() { var devSession helper.DevSession var devStarted bool BeforeEach(func() { @@ -191,7 +191,7 @@ var _ = Describe("odo dev debug command tests", func() { devfileHandlerCtx := devfileHandlerCtx for _, podman := range []bool{false, true} { podman := podman - When("a composite command is used as debug command - "+devfileHandlerCtx.name, helper.LabelPodmanIf(podman, func() { + When("a composite command is used as debug command - "+devfileHandlerCtx.name, Label(helper.LabelSkipOnOpenShift), helper.LabelPodmanIf(podman, func() { var devfileCmpName string var devSession helper.DevSession @@ -264,7 +264,7 @@ var _ = Describe("odo dev debug command tests", func() { } } - When("a composite apply command is used as debug command", func() { + When("a composite apply command is used as debug command", Label(helper.LabelSkipOnOpenShift), func() { deploymentNames := []string{"my-openshift-component", "my-k8s-component"} var devSession helper.DevSession @@ -350,7 +350,7 @@ var _ = Describe("odo dev debug command tests", func() { devfileHandlerCtx := devfileHandlerCtx for _, podman := range []bool{false, true} { podman := podman - When("running build and debug commands as composite in different containers and a shared volume - "+devfileHandlerCtx.name, helper.LabelPodmanIf(podman, func() { + When("running build and debug commands as composite in different containers and a shared volume - "+devfileHandlerCtx.name, Label(helper.LabelSkipOnOpenShift), helper.LabelPodmanIf(podman, func() { var devfileCmpName string var devSession helper.DevSession @@ -434,7 +434,7 @@ var _ = Describe("odo dev debug command tests", func() { for _, podman := range []bool{false, true} { podman := podman - When("a component without debug command is bootstrapped", helper.LabelPodmanIf(podman, func() { + When("a component without debug command is bootstrapped", Label(helper.LabelSkipOnOpenShift), helper.LabelPodmanIf(podman, func() { BeforeEach(func() { helper.CopyExample(filepath.Join("source", "devfiles", "nodejs", "project"), commonVar.Context) helper.Cmd("odo", "init", "--name", cmpName, "--devfile-path", helper.GetExamplePath("source", "devfiles", "nodejs", "devfile-without-debugrun.yaml")).ShouldPass() diff --git a/tests/integration/cmd_dev_test.go b/tests/integration/cmd_dev_test.go index 56798226a..df44ba21b 100644 --- a/tests/integration/cmd_dev_test.go +++ b/tests/integration/cmd_dev_test.go @@ -38,7 +38,7 @@ import ( "github.com/redhat-developer/odo/tests/helper" ) -var _ = Describe("odo dev command tests", func() { +var _ = Describe("odo dev command tests", Label(helper.LabelSkipOnOpenShift), func() { var cmpName string var commonVar helper.CommonVar diff --git a/tests/integration/cmd_devfile_build_images_test.go b/tests/integration/cmd_devfile_build_images_test.go index 6f20c0131..774b6012d 100644 --- a/tests/integration/cmd_devfile_build_images_test.go +++ b/tests/integration/cmd_devfile_build_images_test.go @@ -15,7 +15,7 @@ import ( "github.com/redhat-developer/odo/tests/helper" ) -var _ = Describe("odo devfile build-images command tests", func() { +var _ = Describe("odo devfile build-images command tests", Label(helper.LabelSkipOnOpenShift), func() { var commonVar helper.CommonVar var cmpName string diff --git a/tests/integration/cmd_devfile_deploy_test.go b/tests/integration/cmd_devfile_deploy_test.go index a7790de00..7333bc85e 100644 --- a/tests/integration/cmd_devfile_deploy_test.go +++ b/tests/integration/cmd_devfile_deploy_test.go @@ -382,7 +382,7 @@ ComponentSettings: }) } - When("deploying a ServiceBinding k8s resource", Label(helper.LabelServiceBinding), func() { + When("deploying a ServiceBinding k8s resource", Label(helper.LabelServiceBinding), Label(helper.LabelSkipOnOpenShift), func() { const serviceBindingName = "my-nodejs-app-cluster-sample" // hard-coded from devfile-deploy-with-SB.yaml BeforeEach(func() { skipLogin := os.Getenv("SKIP_SERVICE_BINDING_TESTS") diff --git a/tests/integration/cmd_devfile_init_test.go b/tests/integration/cmd_devfile_init_test.go index 56769c63f..a2a7f3265 100644 --- a/tests/integration/cmd_devfile_init_test.go +++ b/tests/integration/cmd_devfile_init_test.go @@ -24,7 +24,7 @@ import ( "github.com/redhat-developer/odo/tests/helper" ) -var _ = Describe("odo devfile init command tests", func() { +var _ = Describe("odo devfile init command tests", Label(helper.LabelSkipOnOpenShift), func() { var commonVar helper.CommonVar var _ = BeforeEach(func() { diff --git a/tests/integration/cmd_devfile_list_test.go b/tests/integration/cmd_devfile_list_test.go index 513e0171a..9b7ad66a2 100644 --- a/tests/integration/cmd_devfile_list_test.go +++ b/tests/integration/cmd_devfile_list_test.go @@ -14,7 +14,7 @@ import ( . "github.com/onsi/gomega" ) -var _ = Describe("odo list with devfile", func() { +var _ = Describe("odo list with devfile", Label(helper.LabelSkipOnOpenShift), func() { var commonVar helper.CommonVar // This is run before every Spec (It) diff --git a/tests/integration/cmd_devfile_registry_test.go b/tests/integration/cmd_devfile_registry_test.go index e079fc3f5..1fa832d47 100644 --- a/tests/integration/cmd_devfile_registry_test.go +++ b/tests/integration/cmd_devfile_registry_test.go @@ -11,7 +11,7 @@ import ( "github.com/redhat-developer/odo/tests/helper" ) -var _ = Describe("odo devfile registry command tests", func() { +var _ = Describe("odo devfile registry command tests", Label(helper.LabelSkipOnOpenShift), func() { var commonVar helper.CommonVar diff --git a/tests/integration/cmd_list_services_test.go b/tests/integration/cmd_list_services_test.go index 1c1bbfcb1..320ca9937 100644 --- a/tests/integration/cmd_list_services_test.go +++ b/tests/integration/cmd_list_services_test.go @@ -10,7 +10,7 @@ import ( "github.com/redhat-developer/odo/tests/helper" ) -var _ = Describe("odo list services tests", Label(helper.LabelServiceBinding), func() { +var _ = Describe("odo list services tests", Label(helper.LabelServiceBinding), Label(helper.LabelSkipOnOpenShift), func() { var commonVar helper.CommonVar var randomProject string diff --git a/tests/integration/cmd_logs_test.go b/tests/integration/cmd_logs_test.go index f267a4dda..c0948214d 100644 --- a/tests/integration/cmd_logs_test.go +++ b/tests/integration/cmd_logs_test.go @@ -24,7 +24,7 @@ func getLogCommand(podman bool, otherArgs ...string) *helper.CmdWrapper { return cmd } -var _ = Describe("odo logs command tests", func() { +var _ = Describe("odo logs command tests", Label(helper.LabelSkipOnOpenShift), func() { var componentName string var commonVar helper.CommonVar diff --git a/tests/integration/cmd_namespace_test.go b/tests/integration/cmd_namespace_test.go index 090305354..3e89dff51 100644 --- a/tests/integration/cmd_namespace_test.go +++ b/tests/integration/cmd_namespace_test.go @@ -15,7 +15,7 @@ import ( . "github.com/onsi/gomega" ) -var _ = Describe("odo create/delete/list/set namespace/project tests", func() { +var _ = Describe("odo create/delete/list/set namespace/project tests", Label(helper.LabelSkipOnOpenShift), func() { var commonVar helper.CommonVar BeforeEach(func() { diff --git a/tests/integration/cmd_pref_config_test.go b/tests/integration/cmd_pref_config_test.go index ded0d9923..bd038fc9e 100644 --- a/tests/integration/cmd_pref_config_test.go +++ b/tests/integration/cmd_pref_config_test.go @@ -18,7 +18,7 @@ import ( const promptMessageSubString = "Help odo improve by allowing it to collect usage data." -var _ = Describe("odo preference and config command tests", func() { +var _ = Describe("odo preference and config command tests", Label(helper.LabelSkipOnOpenShift), func() { // TODO: A neater way to provide odo path. Currently we assume odo and oc in $PATH already. var commonVar helper.CommonVar diff --git a/tests/integration/cmd_remove_binding_test.go b/tests/integration/cmd_remove_binding_test.go index 1391bbc29..f5b3fb44a 100644 --- a/tests/integration/cmd_remove_binding_test.go +++ b/tests/integration/cmd_remove_binding_test.go @@ -11,7 +11,7 @@ import ( "github.com/redhat-developer/odo/tests/helper" ) -var _ = Describe("odo remove binding command tests", Label(helper.LabelServiceBinding), func() { +var _ = Describe("odo remove binding command tests", Label(helper.LabelServiceBinding), Label(helper.LabelSkipOnOpenShift), func() { var commonVar helper.CommonVar var _ = BeforeEach(func() { diff --git a/tests/integration/cmd_run_test.go b/tests/integration/cmd_run_test.go index cb2c1a676..080dca06a 100644 --- a/tests/integration/cmd_run_test.go +++ b/tests/integration/cmd_run_test.go @@ -10,7 +10,7 @@ import ( . "github.com/onsi/gomega" ) -var _ = Describe("odo run command tests", func() { +var _ = Describe("odo run command tests", Label(helper.LabelSkipOnOpenShift), func() { var cmpName string var commonVar helper.CommonVar diff --git a/tests/integration/generic_test.go b/tests/integration/generic_test.go index 60c7903af..a3939ba49 100644 --- a/tests/integration/generic_test.go +++ b/tests/integration/generic_test.go @@ -7,7 +7,7 @@ import ( "github.com/redhat-developer/odo/tests/helper" ) -var _ = Describe("odo generic", func() { +var _ = Describe("odo generic", Label(helper.LabelSkipOnOpenShift), func() { // TODO: A neater way to provide odo path. Currently we assume \ // odo and oc in $PATH already var oc helper.OcRunner diff --git a/tests/integration/interactive_add_binding_test.go b/tests/integration/interactive_add_binding_test.go index d001ee9f9..fffb8eae4 100644 --- a/tests/integration/interactive_add_binding_test.go +++ b/tests/integration/interactive_add_binding_test.go @@ -13,7 +13,7 @@ import ( . "github.com/onsi/gomega" ) -var _ = Describe("odo add binding interactive command tests", Label(helper.LabelServiceBinding), func() { +var _ = Describe("odo add binding interactive command tests", Label(helper.LabelServiceBinding), Label(helper.LabelSkipOnOpenShift), func() { var commonVar helper.CommonVar var serviceName string diff --git a/tests/integration/interactive_deploy_test.go b/tests/integration/interactive_deploy_test.go index d46b7374d..e1f5957fc 100644 --- a/tests/integration/interactive_deploy_test.go +++ b/tests/integration/interactive_deploy_test.go @@ -11,7 +11,7 @@ import ( "github.com/redhat-developer/odo/tests/helper" ) -var _ = Describe("odo deploy interactive command tests", func() { +var _ = Describe("odo deploy interactive command tests", Label(helper.LabelSkipOnOpenShift), func() { var commonVar helper.CommonVar diff --git a/tests/integration/interactive_dev_test.go b/tests/integration/interactive_dev_test.go index 199275b55..379c89a04 100644 --- a/tests/integration/interactive_dev_test.go +++ b/tests/integration/interactive_dev_test.go @@ -11,7 +11,7 @@ import ( "github.com/redhat-developer/odo/tests/helper" ) -var _ = Describe("odo dev interactive command tests", func() { +var _ = Describe("odo dev interactive command tests", Label(helper.LabelSkipOnOpenShift), func() { var commonVar helper.CommonVar diff --git a/tests/integration/interactive_init_test.go b/tests/integration/interactive_init_test.go index 8fd67c459..5754f162e 100644 --- a/tests/integration/interactive_init_test.go +++ b/tests/integration/interactive_init_test.go @@ -22,7 +22,7 @@ import ( "github.com/redhat-developer/odo/tests/helper" ) -var _ = Describe("odo init interactive command tests", func() { +var _ = Describe("odo init interactive command tests", Label(helper.LabelSkipOnOpenShift), func() { var commonVar helper.CommonVar // This is run before every Spec (It) diff --git a/tests/integration/loginlogout/cmd_login_logout_test.go b/tests/integration/loginlogout/cmd_login_logout_test.go index 2762d452f..7737726cc 100644 --- a/tests/integration/loginlogout/cmd_login_logout_test.go +++ b/tests/integration/loginlogout/cmd_login_logout_test.go @@ -2,6 +2,7 @@ package integration import ( "os" + "strings" "time" . "github.com/onsi/ginkgo/v2" @@ -46,6 +47,9 @@ var _ = Describe("odo login and logout command tests", func() { Context("when running login tests", func() { It("should successful with correct credentials and fails with incorrect token", func() { + if strings.ToLower(os.Getenv("CI")) != "openshift" { + Skip("Skipping if not running on OpenShift. Set CI environment variable to openshift.") + } // skip if requested skipLogin := os.Getenv("SKIP_USER_LOGIN_TESTS") if skipLogin == "true" { diff --git a/tests/integration/plugin_handler_test.go b/tests/integration/plugin_handler_test.go index 482625ec8..152402df4 100644 --- a/tests/integration/plugin_handler_test.go +++ b/tests/integration/plugin_handler_test.go @@ -11,6 +11,7 @@ import ( . "github.com/onsi/gomega" "github.com/redhat-developer/odo/pkg/odo/cli/plugins" + "github.com/redhat-developer/odo/tests/helper" ) var sampleScript = []byte(` @@ -18,7 +19,7 @@ var sampleScript = []byte(` echo 'hello' `) -var _ = Describe("odo plugin functionality", func() { +var _ = Describe("odo plugin functionality", Label(helper.LabelSkipOnOpenShift), func() { var tempDir string var origPath = os.Getenv("PATH") var handler plugins.PluginHandler