Merge pull request #768 from rancher-sandbox/752-install-buildkitd-with-kim

Reinstate using `kim builder install` to get buildkit running correctly on the server

background.ts

@@ -259,7 +259,7 @@ async function relayImageProcessorNamespaces() {
 }
 
 Electron.ipcMain.on('images-namespaces-read', (event) => {
-  if ([K8s.State.VM_STARTED, K8s.State.STARTED].includes(k8smanager.state)) {
+  if (k8smanager.state === K8s.State.STARTED) {
     relayImageProcessorNamespaces().catch();
   }
 });
@@ -352,7 +352,6 @@ Electron.ipcMain.on('k8s-restart', async() => {
   try {
     switch (k8smanager.state) {
     case K8s.State.STOPPED:
-    case K8s.State.VM_STARTED:
     case K8s.State.STARTED:
       // Calling start() will restart the backend, possible switching versions
       // as a side-effect.

src/k8s-engine/images/imageProcessor.ts

@@ -1,18 +1,30 @@
 import { Buffer } from 'buffer';
 import { ChildProcess, spawn } from 'child_process';
 import { EventEmitter } from 'events';
+import net from 'net';
 import os from 'os';
 import timers from 'timers';
+import tls from 'tls';
+import util from 'util';
 
+import * as k8s from '@kubernetes/client-node';
+
+import * as childProcess from '@/utils/childProcess';
 import * as K8s from '@/k8s-engine/k8s';
 import mainEvents from '@/main/mainEvents';
 import Logging from '@/utils/logging';
+import resources from '@/resources';
 import LimaBackend from '@/k8s-engine/lima';
 
 const REFRESH_INTERVAL = 5 * 1000;
 const APP_NAME = 'rancher-desktop';
+const KUBE_CONTEXT = 'rancher-desktop';
 const console = Logging.images;
 
+function defined<T>(input: T | undefined | null): input is T {
+  return typeof input !== 'undefined' && input !== null;
+}
+
 /**
  * The fields that cover the results of a finished process.
  * Not all fields are set for every process.
@@ -70,9 +82,21 @@ export abstract class ImageProcessor extends EventEmitter {
         this.updateWatchStatus();
       }
     });
-    mainEvents.on('k8s-check-state', (mgr: K8s.KubernetesBackend) => {
-      this.isK8sReady = [K8s.State.VM_STARTED, K8s.State.STARTED].includes(mgr.state);
+    mainEvents.on('k8s-check-state', async(mgr: K8s.KubernetesBackend) => {
+      this.isK8sReady = mgr.state === K8s.State.STARTED;
       this.updateWatchStatus();
+      if (this.isK8sReady) {
+        let endpoint: string | undefined;
+
+        // XXX temporary hack: use a fixed address for kim endpoint
+        if (mgr.backend === 'lima') {
+          endpoint = '127.0.0.1';
+        }
+
+        const needsForce = !(await this.isInstallValid(mgr, endpoint));
+
+        this.install(mgr, needsForce, endpoint);
+      }
     });
   }
 
@@ -259,6 +283,203 @@ export abstract class ImageProcessor extends EventEmitter {
     });
   }
 
+  /**
+   * Determine if the Kim service needs to be reinstalled.
+   */
+  protected async isInstallValid(mgr: K8s.KubernetesBackend, endpoint?: string): Promise<boolean> {
+    const host = await mgr.ipAddress;
+
+    if (!host) {
+      return false;
+    }
+
+    const client = new k8s.KubeConfig();
+
+    client.loadFromDefault();
+    client.setCurrentContext(KUBE_CONTEXT);
+    const api = client.makeApiClient(k8s.CoreV1Api);
+
+    // Remove any stale pods; do this first, as we may end up having an invalid
+    // configuration but with stale pods.  Note that `kim builder install --force`
+    // will _not_ fix any stale pods.  We need to wait for the node IP to be
+    // correct first, though, to ensure that we don't end up with a recreated
+    // pod with the stale address.
+    await this.waitForNodeIP(api, host);
+    await this.removeStalePods(api);
+
+    const wantedEndpoint = endpoint || host;
+
+    // Check if the endpoint has the correct address
+    try {
+      const { body: endpointBody } = await api.readNamespacedEndpoints('builder', 'kube-image');
+      const subset = endpointBody.subsets?.find(subset => subset.ports?.some(port => port.name === 'kim'));
+
+      if (!(subset?.addresses || []).some(address => address.ip === wantedEndpoint)) {
+        console.log('Existing kim install invalid: incorrect endpoint address.');
+
+        return false;
+      }
+    } catch (ex) {
+      if (ex.statusCode === 404) {
+        console.log('Existing kim install invalid: missing endpoint');
+
+        return false;
+      }
+      console.error('Error looking for endpoints:', ex);
+      throw ex;
+    }
+
+    // Check if the certificate has the correct address
+    const { body: secretBody } = await api.readNamespacedSecret('kim-tls-server', 'kube-image');
+    const encodedCert = (secretBody.data || {})['tls.crt'];
+
+    // If we don't have a cert, that's fine — kim will fix it.
+    if (encodedCert) {
+      const cert = Buffer.from(encodedCert, 'base64');
+      const secureContext = tls.createSecureContext({ cert });
+      const socket = new tls.TLSSocket(new net.Socket(), { secureContext });
+      const parsedCert = socket.getCertificate();
+
+      console.log(parsedCert);
+      if (parsedCert && 'subjectaltname' in parsedCert) {
+        const { subjectaltname } = parsedCert;
+        const names = subjectaltname.split(',').map(s => s.trim());
+        const acceptable = [`IP Address:${ wantedEndpoint }`, `DNS:${ wantedEndpoint }`];
+
+        if (!names.some(name => acceptable.includes(name))) {
+          console.log(`Existing kim install invalid: incorrect certificate (${ subjectaltname } does not contain ${ wantedEndpoint }).`);
+
+          return false;
+        }
+      }
+    }
+
+    return true;
+  }
+
+  /**
+   * Wait for the Kubernetes node to have the expected IP address.
+   *
+   * When the (single-node) cluster initially starts up, the node (internal)
+   * address can take a while to be updated.
+   * @param api API to communicate with Kubernetes.
+   * @param hostAddr The expected node address.
+   */
+  protected async waitForNodeIP(api: k8s.CoreV1Api, hostAddr: string) {
+    console.log(`Waiting for Kubernetes node IP to become ${ hostAddr }...`);
+    const startTime = Date.now();
+    const MAX_WAIT_TIME = 60_000;
+
+    while (true) {
+      const { body: nodeList } = await api.listNode();
+      const addresses = nodeList.items
+        .flatMap(node => node.status?.addresses)
+        .filter(defined)
+        .filter(address => address.type === 'InternalIP')
+        .flatMap(address => address.address);
+
+      if (addresses.includes(hostAddr)) {
+        break;
+      }
+      if (Date.now() - startTime > MAX_WAIT_TIME) {
+        console.log(`Stop waiting for the IP after ${ MAX_WAIT_TIME / 1000 } seconds`);
+        break;
+      }
+      await util.promisify(setTimeout)(1_000);
+    }
+  }
+
+  /**
+   * When we start the cluster, we may have leftover pods from the builder
+   * daemonset that have stale addresses.  They will not work correctly (not
+   * listening on the new address), but their existence will prevent a new,
+   * correct pod from being created.
+   *
+   * @param api API to communicate with Kubernetes.
+   * @param hostAddr The expected node address.
+   */
+  protected async removeStalePods(api: k8s.CoreV1Api) {
+    const { body: nodeList } = await api.listNode();
+    const addresses = nodeList.items
+      .flatMap(node => node.status?.addresses)
+      .filter(defined)
+      .filter(address => address.type === 'InternalIP')
+      .flatMap(address => address.address);
+
+    const { body: podList } = await api.listNamespacedPod(
+      'kube-image', undefined, undefined, undefined, undefined,
+      'app.kubernetes.io/name=kim,app.kubernetes.io/component=builder');
+
+    for (const pod of podList.items) {
+      const { namespace, name } = pod.metadata || {};
+
+      if (!namespace || !name) {
+        continue;
+      }
+      const currentAddress = pod.status?.podIP;
+
+      if (currentAddress && !addresses.includes(currentAddress)) {
+        console.log(`Deleting stale builder pod ${ namespace }:${ name } - pod IP ${ currentAddress } not in ${ addresses }`);
+        api.deleteNamespacedPod(name, namespace);
+      } else {
+        console.log(`Keeping builder pod ${ namespace }:${ name } - pod IP ${ currentAddress } in ${ addresses }`);
+      }
+    }
+  }
+
+  /**
+   * Install the kim backend if required; this returns when the backend is ready.
+   * @param force If true, force a reinstall of the backend.
+   */
+  async install(backend: K8s.KubernetesBackend, force = false, address?: string) {
+    if (!force && await backend.isServiceReady('kube-image', 'builder')) {
+      console.log('Skipping kim reinstall: service is ready, and without --force');
+
+      return;
+    }
+
+    const startTime = Date.now();
+    const maxWaitTime = 300_000;
+    const waitTime = 3_000;
+    const args = ['builder', 'install'];
+
+    if (force) {
+      args.push('--force');
+    }
+
+    if (address) {
+      args.push('--endpoint-addr', address);
+    }
+
+    console.log(`Installing kim: kim ${ args.join(' ') }`);
+
+    try {
+      await childProcess.spawnFile(
+        resources.executable('kim'),
+        args,
+        {
+          stdio:       ['ignore', console, console],
+          windowsHide: true,
+        });
+
+      while (true) {
+        const currentTime = Date.now();
+
+        if ((currentTime - startTime) > maxWaitTime) {
+          console.log(`Waited more than ${ maxWaitTime / 1000 } secs, it might start up later`);
+          break;
+        }
+        if (await backend.isServiceReady('kube-image', 'builder')) {
+          break;
+        }
+        await util.promisify(setTimeout)(waitTime);
+      }
+    } catch (e) {
+      console.error(`Failed to restart the kim builder: ${ e.message }.`);
+      console.error('The images page will probably be empty');
+    }
+  }
+
   get namespace() {
     return this.currentNamespace;
   }

src/k8s-engine/k8s.ts

@@ -10,7 +10,6 @@ export { KubeClient as Client, ServiceEntry } from './client';
 export enum State {
   STOPPED = 0, // The engine is not running.
   STARTING, // The engine is attempting to start.
-  VM_STARTED, // The VM is up, but k8s hasn't started yet
   STARTED, // The engine is started; the dashboard is not yet ready.
   STOPPING, // The engine is attempting to stop.
   ERROR, // There is an error and we cannot recover automatically.

src/k8s-engine/lima.ts

@@ -746,7 +746,6 @@ export default class LimaBackend extends events.EventEmitter implements K8s.Kube
             console.debug('/etc/rancher/k3s/k3s.yaml is ready.');
           }
         );
-        this.setState(K8s.State.VM_STARTED);
         await this.progressTracker.action(
           'Updating kubeconfig',
           50,

src/k8s-engine/wsl.ts

@@ -809,7 +809,6 @@ export default class WSLBackend extends events.EventEmitter implements K8s.Kuber
           }
         });
 
-        this.setState(K8s.State.VM_STARTED);
         await this.progressTracker.action('Starting guest agent', 100, this.launchAgent());
 
         await this.progressTracker.action(

src/main/tray.ts

@@ -176,7 +176,6 @@ export class Tray {
     const labels = {
       [State.STOPPED]:    'Kubernetes is stopped',
       [State.STARTING]:   'Kubernetes is starting',
-      [State.VM_STARTED]: 'VM is ready',
       [State.STARTED]:    'Kubernetes is running',
       [State.STOPPING]:   'Kubernetes is shutting down',
       [State.ERROR]:      'Kubernetes has encountered an error',

src/pages/Images.vue

@@ -32,7 +32,7 @@ export default {
 
   computed: {
     state() {
-      if (![K8s.State.VM_STARTED, K8s.State.STARTED].includes(this.k8sState)) {
+      if (this.k8sState !== K8s.State.STARTED) {
         return 'IMAGE_MANAGER_UNREADY';
       }