Isolate Podman tests in namespaces (#6499)

* Rename SetProjectName into GetProjectName

Co-authored-by: Anand Singh <ansingh@redhat.com>
Co-authored-by: Parthvi Vala <pvala@redhat.com>
Co-authored-by: Philippe Martin <phmartin@redhat.com>

* Generate specific containers.conf file for each test spec using a dedicated engine namespace

Co-authored-by: Anand Singh <ansingh@redhat.com>
Co-authored-by: Parthvi Vala <pvala@redhat.com>
Co-authored-by: Philippe Martin <phmartin@redhat.com>

* Listen on random ports on Podman when '--random-ports' is used

This reduces the risks of port conflicts when running test specs in parallel.

Co-authored-by: Philippe Martin <phmartin@redhat.com>

* Exclude Gosec G404 (use of math/rand) rule

Co-authored-by: Philippe Martin <phmartin@redhat.com>

* Run Podman specs in parallel

* Output the Pod spec to be played by Podman depending on verbosity level

This will help debug potential issues.

* Use random name in 'using devfile that contains K8s resource to run it on podman' test

* Use random component name in sample java-quarkus project used in 'a hotReload capable project is used with odo dev' test

* Revert "Run Podman specs in parallel"

Parallelization works great on GitHub Actions, but I experimented a lot
of issues when running locally with a lot (~11) of parallel test nodes.
Not sure why exactly, but some containers created by Podman had a lot of
networking issues.
We can look into parallelizing the runs later in a subsequent PR.

This reverts commit 64d5d31248a62f355a32ca245ba399a723fdb22f.

* Allow overridding the number of parallel nodes for Podman integration tests

This way, we could be able to run them in parallel on GitHub
but sequentially (default) locally.
This will still benefit us by reducing the time it takes to run such tests on GitHub.

Meanwhile, we can look into the issues we have locally with parallelization.

Note that it is still possible to run them locally in parallel via
the PODMAN_EXEC_NODE env var.

Co-authored-by: Anand Singh <ansingh@redhat.com>
Co-authored-by: Parthvi Vala <pvala@redhat.com>
Co-authored-by: Philippe Martin <phmartin@redhat.com>

.github/workflows/podman-test.yaml

@@ -4,6 +4,7 @@ on:
     branches:    
       - main
 
+
 jobs:
   ODO-PODMAN-TEST:
     runs-on: ubuntu-latest
@@ -21,5 +22,6 @@ jobs:
       run: make install
 
     - name: Run Integration tests
-      run: |
-        make test-integration-podman
+      env:
+        PODMAN_EXEC_NODES: ${{ vars.PODMAN_TEST_EXEC_NODES }}
+      run: make test-integration-podman

Makefile

@@ -47,12 +47,20 @@ UNIT_TEST_ARGS ?=
 
 export ARTIFACT_DIR ?= .
 
+ifdef PODMAN_EXEC_NODES
+   PODMAN_EXEC_NODES := $(PODMAN_EXEC_NODES)
+else
+   PODMAN_EXEC_NODES := 1
+endif
+
 GINKGO_FLAGS_ALL = $(GINKGO_TEST_ARGS) --randomize-all --slow-spec-threshold=$(SLOW_SPEC_THRESHOLD) -timeout $(TIMEOUT) --no-color
 
 # Flags to run one test per core.
 GINKGO_FLAGS_AUTO = $(GINKGO_FLAGS_ALL) -p
 # Flags for tests that may be run in parallel
 GINKGO_FLAGS=$(GINKGO_FLAGS_ALL) -nodes=$(TEST_EXEC_NODES)
+# Flags for Podman tests that may be run in parallel
+GINKGO_FLAGS_PODMAN=$(GINKGO_FLAGS_ALL) -nodes=$(PODMAN_EXEC_NODES)
 # Flags for tests that must not be run in parallel
 GINKGO_FLAGS_ONE=$(GINKGO_FLAGS_ALL) -nodes=1
 # GolangCi version for unit-validate test
@@ -202,9 +210,12 @@ test-integration-openshift-unauth:
 test-integration-no-cluster:
 	$(RUN_GINKGO) $(GINKGO_FLAGS_AUTO)  --junit-report="test-integration-nc.xml" --label-filter=nocluster tests/integration
 
+# Running by default on 1 node because of issues when running too many Podman containers locally,
+# but you can override this behavior by setting the PODMAN_EXEC_NODES env var if needed.
+# For example: "PODMAN_EXEC_NODES=5".
 .PHONY: test-integration-podman
 test-integration-podman:
-	$(RUN_GINKGO) $(GINKGO_FLAGS_ONE)  --junit-report="test-integration-podman.xml" --label-filter=podman tests/integration
+	$(RUN_GINKGO) $(GINKGO_FLAGS_PODMAN) --junit-report="test-integration-podman.xml" --label-filter=podman tests/integration
 
 .PHONY: test-integration
 test-integration: test-integration-no-cluster test-integration-cluster

pkg/dev/podmandev/pod.go

@@ -2,6 +2,8 @@ package podmandev
 
 import (
 	"fmt"
+	"math/rand" //#nosec
+	"time"
 
 	"github.com/devfile/library/pkg/devfile/generator"
 	"github.com/devfile/library/pkg/devfile/parser"
@@ -28,6 +30,7 @@ func createPodFromComponent(
 	buildCommand string,
 	runCommand string,
 	debugCommand string,
+	randomPorts bool,
 	usedPorts []int,
 ) (*corev1.Pod, []api.ForwardedPort, error) {
 	containers, err := generator.GetContainers(devfileObj, common.DevfileOptions{})
@@ -45,7 +48,7 @@ func createPodFromComponent(
 	utils.AddOdoProjectVolume(&containers)
 	utils.AddOdoMandatoryVolume(&containers)
 
-	fwPorts := addHostPorts(containers, debug, usedPorts)
+	fwPorts := addHostPorts(containers, debug, randomPorts, usedPorts)
 
 	volumes := []corev1.Volume{
 		{
@@ -111,10 +114,12 @@ func getVolumeName(volume string, componentName string, appName string) string {
 	return volume + "-" + componentName + "-" + appName
 }
 
-func addHostPorts(containers []corev1.Container, debug bool, usedPorts []int) []api.ForwardedPort {
+func addHostPorts(containers []corev1.Container, debug bool, randomPorts bool, usedPorts []int) []api.ForwardedPort {
 	var result []api.ForwardedPort
 	startPort := 40001
 	endPort := startPort + 10000
+	usedPortsCopy := make([]int, len(usedPorts))
+	copy(usedPortsCopy, usedPorts)
 	for i := range containers {
 		var ports []corev1.ContainerPort
 		for _, port := range containers[i].Ports {
@@ -124,11 +129,30 @@ func addHostPorts(containers []corev1.Container, debug bool, usedPorts []int) []
 					containers[i].Name, portName, port.ContainerPort)
 				continue
 			}
-			freePort, err := util.NextFreePort(startPort, endPort, usedPorts)
+			var freePort int
+			if randomPorts {
+				if len(usedPortsCopy) != 0 {
+					freePort = usedPortsCopy[0]
+					usedPortsCopy = usedPortsCopy[1:]
+				} else {
+					rand.Seed(time.Now().UnixNano()) //#nosec
+					for {
+						freePort = rand.Intn(endPort-startPort+1) + startPort //#nosec
+						if util.IsPortFree(freePort) {
+							break
+						}
+						time.Sleep(100 * time.Millisecond)
+					}
+				}
+			} else {
+				var err error
+				freePort, err = util.NextFreePort(startPort, endPort, usedPorts)
 				if err != nil {
 					klog.Infof("%s", err)
 					continue
 				}
+				startPort = freePort + 1
+			}
 			result = append(result, api.ForwardedPort{
 				Platform:      commonflags.PlatformPodman,
 				ContainerName: containers[i].Name,
@@ -138,7 +162,6 @@ func addHostPorts(containers []corev1.Container, debug bool, usedPorts []int) []
 			})
 			port.HostPort = int32(freePort)
 			ports = append(ports, port)
-			startPort = freePort + 1
 		}
 		containers[i].Ports = ports
 	}

pkg/dev/podmandev/pod_test.go

@@ -388,6 +388,7 @@ func Test_createPodFromComponent(t *testing.T) {
 				tt.args.buildCommand,
 				tt.args.runCommand,
 				tt.args.debugCommand,
+				false,
 				[]int{40001, 40002},
 			)
 			if (err != nil) != tt.wantErr {

pkg/dev/podmandev/reconcile.go

@@ -160,6 +160,7 @@ func (o *DevClient) deployPod(ctx context.Context, options dev.StartOptions) (*c
 		options.BuildCommand,
 		options.RunCommand,
 		"",
+		options.RandomPorts,
 		o.usedPorts,
 	)
 	if err != nil {

pkg/podman/podman.go

@@ -63,6 +63,12 @@ func (o *PodmanCli) PlayKube(pod *corev1.Pod) error {
 		return err
 	}
 
+	if klog.V(4) {
+		var sb strings.Builder
+		_ = serializer.Encode(pod, &sb)
+		klog.Infof("Pod spec to play: \n---\n%s\n---\n", sb.String())
+	}
+
 	err = serializer.Encode(pod, stdin)
 	if err != nil {
 		return err

tests/helper/helper_generic.go

@@ -195,9 +195,10 @@ func CommonBeforeEach() CommonVar {
 	commonVar.ConfigDir = CreateNewContext()
 	commonVar.CliRunner = GetCliRunner()
 	commonVar.OriginalKubeconfig = os.Getenv("KUBECONFIG")
-	if NeedsCluster(CurrentSpecReport().Labels()) {
+	specLabels := CurrentSpecReport().Labels()
+	if NeedsCluster(specLabels) {
 		LocalKubeconfigSet(commonVar.ConfigDir)
-		if IsAuth(CurrentSpecReport().Labels()) {
+		if IsAuth(specLabels) {
 			commonVar.Project = commonVar.CliRunner.CreateAndSetRandNamespaceProject()
 		} else {
 			commonVar.CliRunner.AssertNonAuthenticated()
@@ -211,6 +212,11 @@ func CommonBeforeEach() CommonVar {
 		err = kubeconfig.Close()
 		Expect(err).To(BeNil())
 		os.Setenv("KUBECONFIG", kubeconfig.Name())
+
+		if NeedsPodman(specLabels) {
+			// Generate a dedicated containers.conf with a specific namespace
+			GenerateAndSetContainersConf(commonVar.ConfigDir)
+		}
 	}
 	commonVar.OriginalWorkingDirectory = Getwd()
 
@@ -319,8 +325,8 @@ func JsonPathContentIsValidUserPort(json string, path string) {
 	))
 }
 
-// SetProjectName sets projectNames based on the name of the test file name (without path and replacing _ with -), line number of current ginkgo execution, and a random string of 3 letters
-func SetProjectName() string {
+// GetProjectName sets projectNames based on the name of the test file name (without path and replacing _ with -), line number of current ginkgo execution, and a random string of 3 letters
+func GetProjectName() string {
 	//Get current test filename and remove file path, file extension and replace undescores with hyphens
 	currGinkgoTestFileName := strings.Replace(strings.Split(strings.Split(CurrentSpecReport().
 		ContainerHierarchyLocations[0].FileName, "/")[len(strings.Split(CurrentSpecReport().ContainerHierarchyLocations[0].FileName, "/"))-1], ".")[0], "_", "-", -1)

tests/helper/helper_kubectl.go

@@ -151,7 +151,7 @@ func (kubectl KubectlRunner) GetServices(namespace string) string {
 
 // CreateAndSetRandNamespaceProject create and set new project
 func (kubectl KubectlRunner) CreateAndSetRandNamespaceProject() string {
-	projectName := SetProjectName()
+	projectName := GetProjectName()
 	kubectl.createAndSetRandNamespaceProject(projectName)
 	return projectName
 }

tests/helper/helper_oc.go

@@ -312,7 +312,7 @@ func (oc OcRunner) VerifyResourceToBeDeleted(ri ResourceInfo) {
 
 // CreateAndSetRandNamespaceProject create and set new project
 func (oc OcRunner) CreateAndSetRandNamespaceProject() string {
-	projectName := SetProjectName()
+	projectName := GetProjectName()
 	oc.createAndSetRandNamespaceProject(projectName)
 	return projectName
 }

tests/helper/helper_podman.go

@@ -0,0 +1,20 @@
+package helper
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+
+	. "github.com/onsi/gomega"
+)
+
+func GenerateAndSetContainersConf(dir string) {
+	ns := GetProjectName()
+	containersConfPath := filepath.Join(dir, "containers.conf")
+	err := CreateFileWithContent(containersConfPath, fmt.Sprintf(`
+[engine]
+namespace=%q
+`, ns))
+	Expect(err).ShouldNot(HaveOccurred())
+	os.Setenv("CONTAINERS_CONF", containersConfPath)
+}

tests/helper/labels.go

@@ -22,6 +22,15 @@ func NeedsCluster(labels []string) bool {
 	return true
 }
 
+func NeedsPodman(labels []string) bool {
+	for _, label := range labels {
+		if label == LabelPodman {
+			return true
+		}
+	}
+	return false
+}
+
 func IsAuth(labels []string) bool {
 	for _, label := range labels {
 		if label == LabelUnauth {

tests/helper/odo_utils.go

@@ -34,7 +34,7 @@ func GetPreferenceValue(key string) string {
 // CreateRandProject create new project with random name (10 letters)
 // without writing to the config file (without switching project)
 func CreateRandProject() string {
-	projectName := SetProjectName()
+	projectName := GetProjectName()
 	fmt.Fprintf(GinkgoWriter, "Creating a new project: %s\n", projectName)
 	session := Cmd("odo", "create", "project", projectName, "-w", "-v4").ShouldPass().Out()
 	Expect(session).To(ContainSubstring("New project created"))

tests/integration/cmd_dev_test.go

@@ -2899,7 +2899,10 @@ CMD ["npm", "start"]
 		When("using devfile that contains K8s resource to run it on podman", Label(helper.LabelPodman), func() {
 			BeforeEach(func() {
 				helper.CopyExample(filepath.Join("source", "devfiles", "nodejs", "project"), commonVar.Context)
-				helper.CopyExampleDevFile(filepath.Join("source", "devfiles", "nodejs", "devfile-composite-apply-different-commandgk.yaml"), filepath.Join(commonVar.Context, "devfile.yaml"))
+				helper.CopyExampleDevFile(
+					filepath.Join("source", "devfiles", "nodejs", "devfile-composite-apply-different-commandgk.yaml"),
+					filepath.Join(commonVar.Context, "devfile.yaml"),
+					helper.DevfileMetadataNameSetter(cmpName))
 			})
 			It(fmt.Sprintf("should show warning about being unable to create the resource when running odo dev %s on podman", ctx.title), func() {
 				err := helper.RunDevMode(helper.DevSessionOpts{RunOnPodman: true, CmdlineArgs: ctx.args}, func(session *gexec.Session, outContents, errContents []byte, ports map[string]string) {
@@ -2922,6 +2925,7 @@ CMD ["npm", "start"]
 			var executeRunCommand = "Executing the application (command: dev-run)"
 			BeforeEach(func() {
 				helper.CopyExample(filepath.Join("source", "java-quarkus"), commonVar.Context)
+				helper.UpdateDevfileContent(filepath.Join(commonVar.Context, "devfile.yaml"), []helper.DevfileUpdater{helper.DevfileMetadataNameSetter(cmpName)})
 				var err error
 				devSession, stdout, _, _, err = helper.StartDevMode(helper.DevSessionOpts{
 					RunOnPodman: podman,