Fix potential port conflict issue with the API server tests by using a random server port if --random-ports is set (#6995)

* Let the OS assign a random ephemeral port if `--random-ports` is specified when running `odo dev` or `odo api-server`

This should hopefully fix the potential port conflict flaky
issue we are seeing, especially on Windows.

* Do not allow setting `--random-ports` and a specific port value

This applies to:
- `odo dev --random-ports --api-server --api-server-port=<port>`
- `odo api-server --random-ports --port=<port>`

pkg/apiserver-impl/starterserver.go

@@ -30,6 +30,7 @@ type ApiServer struct {
 func StartServer(
 	ctx context.Context,
 	cancelFunc context.CancelFunc,
+	randomPort bool,
 	port int,
 	devfilePath string,
 	devfileFiles []string,
@@ -65,7 +66,7 @@ func StartServer(
 	staticServer := http.FileServer(http.FS(fSys))
 	router.PathPrefix("/").Handler(staticServer)
 
-	if port == 0 {
+	if port == 0 && !randomPort {
 		port, err = util.NextFreePort(20000, 30001, nil, "")
 		if err != nil {
 			klog.V(0).Infof("Unable to start the API server; encountered error: %v", err)
@@ -73,23 +74,36 @@ func StartServer(
 		}
 	}
 
-	err = stateClient.SetAPIServerPort(ctx, port)
+	listener, err := net.Listen("tcp", fmt.Sprintf(":%d", port))
+	if err != nil {
+		return ApiServer{}, fmt.Errorf("unable to start API Server listener on port %d: %w", port, err)
+	}
+
+	server := &http.Server{
+		BaseContext: func(net.Listener) context.Context {
+			return ctx
+		},
+		Handler: router,
+	}
+	var errChan = make(chan error)
+	go func() {
+		errChan <- server.Serve(listener)
+	}()
+
+	// Get the actual port value assigned by the operating system
+	listeningPort := listener.Addr().(*net.TCPAddr).Port
+	if port != 0 && port != listeningPort {
+		panic(fmt.Sprintf("requested port (%d) not the same as the actual port the API Server is bound to (%d)", port, listeningPort))
+	}
+
+	err = stateClient.SetAPIServerPort(ctx, listeningPort)
 	if err != nil {
 		klog.V(0).Infof("Unable to start the API server; encountered error: %v", err)
 		cancelFunc()
 	}
 
-	klog.V(0).Infof("API Server started at localhost:%d/api/v1", port)
+	klog.V(0).Infof("API Server started at localhost:%d/api/v1", listeningPort)
 
-	server := &http.Server{Addr: fmt.Sprintf(":%d", port), Handler: router}
-	var errChan = make(chan error)
-	go func() {
-		server.BaseContext = func(net.Listener) context.Context {
-			return ctx
-		}
-		err = server.ListenAndServe()
-		errChan <- err
-	}()
 	go func() {
 		select {
 		case <-ctx.Done():

pkg/odo/cli/apiserver/apiserver.go

@@ -2,6 +2,7 @@ package apiserver
 
 import (
 	"context"
+	"errors"
 	"fmt"
 
 	"github.com/spf13/cobra"
@@ -21,7 +22,10 @@ const (
 
 type ApiServerOptions struct {
 	clientset *clientset.Clientset
-	portFlag  int
+
+	// Flags
+	randomPortsFlag bool
+	portFlag        int
 }
 
 func NewApiServerOptions() *ApiServerOptions {
@@ -41,6 +45,9 @@ func (o *ApiServerOptions) Complete(ctx context.Context, cmdline cmdline.Cmdline
 }
 
 func (o *ApiServerOptions) Validate(ctx context.Context) error {
+	if o.randomPortsFlag && o.portFlag != 0 {
+		return errors.New("--random-ports and --port cannot be used together")
+	}
 	return nil
 }
 
@@ -67,6 +74,7 @@ func (o *ApiServerOptions) Run(ctx context.Context) (err error) {
 	_, err = apiserver_impl.StartServer(
 		ctx,
 		cancel,
+		o.randomPortsFlag,
 		o.portFlag,
 		devfilePath,
 		devfileFiles,
@@ -114,6 +122,7 @@ func NewCmdApiServer(ctx context.Context, name, fullName string, testClientset c
 		clientset.STATE,
 		clientset.PREFERENCE,
 	)
+	apiserverCmd.Flags().BoolVar(&o.randomPortsFlag, "random-ports", false, "Assign a random API Server port.")
 	apiserverCmd.Flags().IntVar(&o.portFlag, "port", 0, "Define custom port for API Server.")
 	return apiserverCmd
 }

pkg/odo/cli/dev/dev.go

@@ -181,6 +181,9 @@ func (o *DevOptions) Validate(ctx context.Context) error {
 	}
 
 	if o.apiServerFlag && o.apiServerPortFlag != 0 {
+		if o.randomPortsFlag {
+			return errors.New("--random-ports and --api-server-port cannot be used together")
+		}
 		if !util.IsPortFree(o.apiServerPortFlag, "") {
 			return fmt.Errorf("port %d is not free; please try another port", o.apiServerPortFlag)
 		}
@@ -271,6 +274,7 @@ func (o *DevOptions) Run(ctx context.Context) (err error) {
 		apiServer, err = apiserver_impl.StartServer(
 			ctx,
 			o.cancel,
+			o.randomPortsFlag,
 			o.apiServerPortFlag,
 			devfilePath,
 			devfileFiles,

tests/integration/cmd_dev_api_server_test.go

@@ -42,6 +42,23 @@ var _ = Describe("odo dev command with api server tests", func() {
 					helper.CopyExample(filepath.Join("source", "devfiles", "nodejs", "project"), commonVar.Context)
 					helper.CopyExampleDevFile(filepath.Join("source", "devfiles", "nodejs", "devfile.yaml"), filepath.Join(commonVar.Context, "devfile.yaml"), cmpName)
 				})
+
+				if customPort {
+					It("should fail if --random-ports and --api-server-port are used together", func() {
+						args := []string{
+							"dev",
+							"--random-ports",
+							"--api-server",
+							fmt.Sprintf("--api-server-port=%d", helper.GetCustomStartPort()),
+						}
+						if podman {
+							args = append(args, "--platform=podman")
+						}
+						errOut := helper.Cmd("odo", args...).AddEnv("ODO_EXPERIMENTAL_MODE=true").ShouldFail().Err()
+						Expect(errOut).Should(ContainSubstring("--random-ports and --api-server-port cannot be used together"))
+					})
+				}
+
 				When(fmt.Sprintf("odo dev is run with --api-server flag (custom api server port=%v)", customPort), func() {
 					var devSession helper.DevSession
 					var localPort int
@@ -53,6 +70,7 @@ var _ = Describe("odo dev command with api server tests", func() {
 						if customPort {
 							localPort = helper.GetCustomStartPort()
 							opts.APIServerPort = localPort
+							opts.NoRandomPorts = true
 						}
 						var err error
 						devSession, err = helper.StartDevMode(opts)