Install and use a reverse proxy for staging devfile registry (#5922)

* Kubernetes Manifests to deploy nginx reverse proxy for staging devfile registry

* Get the external address of the loadBalancer service for devfile proxy

* If DEVFILE_PROXY is defined, use it as Devfile registry

* Fix get address from Windows

* Add some log displaying proxy address found

* Filter requests on user agent

* Add Go-http-client user-agent

* Add doc

.ibm/pipelines/kubernetes-tests.sh

@@ -12,6 +12,8 @@ cleanup_namespaces
 export SKIP_USER_LOGIN_TESTS=true
 (
     set -e
+    export DEVFILE_PROXY="$(kubectl get svc -n devfile-proxy nginx -o jsonpath='{.status.loadBalancer.ingress[0].hostname}' || true)"
+    echo Using Devfile proxy: ${DEVFILE_PROXY}
     make install
     make test-integration
     make test-e2e

.ibm/pipelines/openshift-tests.sh

@@ -13,6 +13,8 @@ cleanup_namespaces
 
 (
     set -e
+    export DEVFILE_PROXY="$(kubectl get svc -n devfile-proxy nginx -o jsonpath='{.status.loadBalancer.ingress[0].hostname}' || true)"
+    echo Using Devfile proxy: ${DEVFILE_PROXY}
     make install
     make test-integration
     make test-e2e

.ibm/pipelines/windows-test-script.ps1

@@ -64,6 +64,14 @@ function Run-Test {
     oc login -u apikey -p ${API_KEY} ${IBM_OPENSHIFT_ENDPOINT}
     Check-ExitCode $LASTEXITCODE
 
+    Shout "Getting Devfile proxy address"
+    $DEVFILE_PROXY=$(oc get svc -n devfile-proxy nginx -o jsonpath='{.status.loadBalancer.ingress[0].hostname}')
+    if ( $LASTEXITCODE -eq 0 )
+    {
+        Shout "Using Devfile proxy: $DEVFILE_PROXY"
+        [Environment]::SetEnvironmentVariable("DEVFILE_PROXY", "$DEVFILE_PROXY")
+    }
+
     Shout "Create Binary"
     make install 
     Shout "Running test"

scripts/ansible/kubernetes-cluster/README.md

@@ -155,3 +155,20 @@ $ helm install nfs-subdir-external-provisioner \
     --set storageClass.defaultClass=true \
     --set storageClass.onDelete=delete
 ```
+
+## Devfile registry reverse proxy
+
+To install a reverse proxy caching the requests to the Staging Devfile registry (https://registry.stage.devfile.io),
+you can run the following command:
+
+```
+kubectl apply -f devfile-proxy.yaml
+```
+
+This will install an nginx install configured as a reverse proxy with the Staging Devfile registry as only backend. 
+
+A Load Balancer service will be created accessible publicly. To limit requests on the proxy, the requests are limited
+to user agents beginning with `containerd` or `Go-http-client`.
+
+The integration tests are able to detect the presence of the Load Balancer service and use the proxy if the service is present
+and providing an external address.

scripts/ansible/kubernetes-cluster/devfile-proxy.yaml

@@ -0,0 +1,125 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: devfile-proxy
+
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nginx
+  namespace: devfile-proxy
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: nginx
+  template:
+    metadata:
+      labels:
+        app: nginx
+    spec:
+      containers:
+      - name: nginx
+        image: nginx
+        imagePullPolicy: IfNotPresent
+        ports:
+        - containerPort: 8080
+        volumeMounts:
+        - mountPath: /etc/nginx # mount nginx-conf volumn to /etc/nginx
+          readOnly: true
+          name: nginx-conf
+        - mountPath: /var/log/nginx
+          name: log
+        - mountPath: /var/cache/nginx
+          name: cache
+        - mountPath: /var/run
+          name: run
+        - mountPath: /data/nginx/cache
+          name: nginx-cache
+        resources:
+          requests:
+            memory: 256Mi
+            cpu: 256m
+          limits:
+            memory: 256Mi
+            cpu: 256m
+      volumes:
+      - name: nginx-conf
+        configMap:
+          name: nginx-conf # place ConfigMap `nginx-conf` on /etc/nginx
+          items:
+            - key: nginx.conf
+              path: nginx.conf
+      - name: log
+        emptyDir: {}
+      - name: cache
+        emptyDir: {}
+      - name: run
+        emptyDir: {}
+      - name: nginx-cache
+        emptyDir: {}
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: nginx
+  namespace: devfile-proxy
+spec:
+  type: LoadBalancer
+  ports:
+  - port: 80
+    targetPort: 8080
+  selector:
+    app: nginx
+
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: nginx-conf
+  namespace: devfile-proxy
+data:
+  nginx.conf: |
+    events {
+      worker_connections  1024;
+    }
+
+    http {
+      proxy_cache_path 
+        /data/nginx/cache 
+        levels=1:2 
+        keys_zone=app:1M 
+        max_size=100M;
+
+      log_format cacheStatus '$host $server_name $server_port $remote_addr $upstream_cache_status $remote_user [$time_local] " $request " '
+                             '$status $body_bytes_sent "$http_referer" '
+                             '"$http_user_agent" "$http_x_forwarded_for"';
+
+      # Need to have a DNS server to resolve the FQDNs provided to proxy_pass
+      # Use the DNS resolver provided to the container
+      resolver 172.21.0.10;
+
+      map "$http_user_agent" $proxybackend {
+        default             "";
+        "~^containerd"      https://registry.stage.devfile.io;
+        "~^Go-http-client"  https://registry.stage.devfile.io;
+      }
+
+      server {
+        listen 8080;
+
+        error_log   /dev/stderr error;
+        access_log  /dev/stdout cacheStatus;
+
+        location / {
+          proxy_cache           app;
+          proxy_pass             $proxybackend;
+          proxy_set_header       Host registry.stage.devfile.io;
+          proxy_ignore_headers   Set-Cookie;
+          proxy_ignore_headers   Cache-Control;
+          proxy_cache_valid     any 30m;
+        }
+      }
+    }

tests/helper/helper_generic.go

@@ -325,7 +325,11 @@ type ResourceInfo struct {
 
 func SetDefaultDevfileRegistryAsStaging() {
 	const registryName string = "DefaultDevfileRegistry"
-	const addRegistryURL string = "https://registry.stage.devfile.io"
+	addRegistryURL := "https://registry.stage.devfile.io"
+	proxy := os.Getenv("DEVFILE_PROXY")
+	if proxy != "" {
+		addRegistryURL = "http://" + proxy
+	}
 	Cmd("odo", "preference", "remove", "registry", registryName, "-f").ShouldPass()
 	Cmd("odo", "preference", "add", "registry", registryName, addRegistryURL).ShouldPass()
 }