alihan/microk8s
1
0
Fork 0
mirror of https://github.com/ubuntu/microk8s.git synced 2021-05-23 02:23:41 +03:00
Code Issues Projects Releases Wiki Activity

Do not re-issue certificates on clusters (#2217)

Browse Source
This commit is contained in:
Konstantinos Tsakalozos
2021-05-05 08:30:38 +03:00
committed by GitHub
parent 9526eeda66
commit f53908083d
7 changed files with 154 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
microk8s-resources/actions/common/utils.sh
View File

@@ -702,4 +702,3 @@ mark_boot_time() {
now=$(date +%s)
echo "$now" > "$1"/last-start-date
}

3
microk8s-resources/wrappers/apiservice-kicker
View File

@@ -53,7 +53,8 @@ do
chgrp microk8s -R ${SNAP_DATA}/var/kubernetes/backend || true
fi
if ! grep -E "(--advertise-address|--bind-address)" $SNAP_DATA/args/kube-apiserver &> /dev/null &&
if ! [ -e "${SNAP_DATA}/var/lock/no-cert-reissue" ] &&
! grep -E "(--advertise-address|--bind-address)" $SNAP_DATA/args/kube-apiserver &> /dev/null &&
ip route | grep default &> /dev/null
then
if snapctl services microk8s.daemon-kubelite | grep active &> /dev/null

5
microk8s-resources/wrappers/microk8s-reset.wrapper
View File

@@ -111,6 +111,11 @@ clean_cluster() {
then
run_with_sudo rm -rf "$SNAP_DATA/bin/"
fi
if [ -e "${SNAP_DATA}/var/lock/no-cert-reissue" ]
then
run_with_sudo rm -rf "${SNAP_DATA}/var/lock/no-cert-reissue"
fi
}
apply_cni() {

4
scripts/cluster/agent.py
View File

@@ -24,6 +24,7 @@ from .common.utils import (
get_cluster_agent_port,
try_initialise_cni_autodetect_for_clustering,
service,
mark_no_cert_reissue,
)
from flask import Flask, jsonify, request, Response
@@ -319,6 +320,8 @@ def join_node_etcd():
else:
kubelet_args = read_kubelet_args_file()
mark_no_cert_reissue()
return jsonify(
ca=ca,
etcd=etcd_ep,
@@ -608,6 +611,7 @@ def join_node_dqlite():
cluster_cert, cluster_key = get_cluster_certs()
# Make sure calico can autodetect the right interface for packet routing
try_initialise_cni_autodetect_for_clustering(node_addr, apply_cni=True)
mark_no_cert_reissue()
return jsonify(
ca=get_cert("ca.crt"),

22
scripts/cluster/common/utils.py
View File

@@ -276,3 +276,25 @@ def service(operation, service_name):
subprocess.check_call(
"snapctl {} microk8s.daemon-{}".format(operation, service_name).split()
)
def mark_no_cert_reissue():
"""
Mark a node as being part of a cluster that should not re-issue certs
on network changes
"""
snap_data = os.environ.get("SNAP_DATA")
lock_file = "{}/var/lock/no-cert-reissue".format(snap_data)
open(lock_file, "a").close()
os.chmod(lock_file, 0o700)
def unmark_no_cert_reissue():
"""
Unmark a node as being part of a cluster. The node should now re-issue certs
on network changes
"""
snap_data = os.environ.get("SNAP_DATA")
lock_file = "{}/var/lock/no-cert-reissue".format(snap_data)
if os.path.exists(lock_file):
os.unlink(lock_file)

14
scripts/cluster/join.py
View File

@@ -25,6 +25,8 @@ from common.utils import (
get_cluster_agent_port,
try_initialise_cni_autodetect_for_clustering,
service,
mark_no_cert_reissue,
unmark_no_cert_reissue,
)
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
@@ -324,9 +326,10 @@ def store_remote_ca(ca):
try_set_file_permissions(ca_cert_file)
def mark_cluster_node():
def mark_worker_node():
"""
Mark a node as being part of a cluster by creating a var/lock/clustered.lock
Mark a node as being part of a cluster not running the control plane
by creating a var/lock/clustered.lock
"""
lock_file = "{}/var/lock/clustered.lock".format(snapdata_path)
open(lock_file, "a").close()
@@ -397,6 +400,8 @@ def reset_current_etcd_installation():
time.sleep(5)
waits -= 1
unmark_no_cert_reissue()
def reset_current_dqlite_installation():
"""
@@ -499,6 +504,7 @@ def reset_current_dqlite_installation():
time.sleep(5)
waits -= 1
print(" ")
unmark_no_cert_reissue()
restart_all_services()
@@ -981,6 +987,7 @@ def join_dqlite(connection_parts, verify=False):
# We want to update the local CNI yaml but we do not want to apply it.
# The cni is applied already in the cluster we join
try_initialise_cni_autodetect_for_clustering(master_ip, apply_cni=False)
mark_no_cert_reissue()
def join_etcd(connection_parts, verify=True):
@@ -1003,7 +1010,8 @@ def join_etcd(connection_parts, verify=True):
update_flannel(info["etcd"], master_ip, master_port, token)
update_kubeproxy(info["kubeproxy"], info["ca"], master_ip, info["apiport"], hostname_override)
update_kubelet(info["kubelet"], info["ca"], master_ip, info["apiport"])
mark_cluster_node()
mark_worker_node()
mark_no_cert_reissue()
if __name__ == "__main__":

122
tests/test-cluster.py
View File

@@ -11,7 +11,9 @@ from os import path
# the test will attempt a refresh to the channel requested for testing
# reuse_vms = ['vm-ldzcjb', 'vm-nfpgea', 'vm-pkgbtw']
reuse_vms = None
channel_to_test = os.environ.get("CHANNEL_TO_TEST", "edge/ha-preview")
# Channel we want to test. A full path to a local snap can be used for local builds
channel_to_test = os.environ.get("CHANNEL_TO_TEST", "latest/edge")
backend = os.environ.get("BACKEND", None)
@@ -20,42 +22,40 @@ class VM:
This class abstracts the backend we are using. It could be either multipass or lxc.
"""
def __init__(self, attach_vm=None):
def __init__(self, backend=None, attach_vm=None):
"""Detect the available backends and instantiate a VM.
If `attach_vm` is provided we just make sure the right MicroK8s is deployed.
:param backend: either multipass of lxc
:param attach_vm: the name of the VM we want to reuse
"""
rnd_letters = "".join(random.choice(string.ascii_lowercase) for i in range(6))
self.backend = "none"
self.backend = backend
self.vm_name = "vm-{}".format(rnd_letters)
self.attached = False
if attach_vm:
self.attached = True
self.vm_name = attach_vm
if path.exists("/snap/bin/multipass") or backend == "multipass":
def setup(self, channel_or_snap):
"""Setup the VM with the right snap.
:param channel_or_snap: the snap channel or the path to the local snap build
"""
if (path.exists("/snap/bin/multipass") and not self.backend) or self.backend == "multipass":
print("Creating mulitpass VM")
self.backend = "multipass"
if not attach_vm:
subprocess.check_call(
"/snap/bin/multipass launch 18.04 -n {} -m 2G".format(self.vm_name).split()
)
subprocess.check_call(
"/snap/bin/multipass exec {} -- sudo "
"snap install microk8s --classic --channel {}".format(
self.vm_name, channel_to_test
).split()
)
else:
subprocess.check_call(
"/snap/bin/multipass exec {} -- sudo "
"snap refresh microk8s --channel {}".format(
self.vm_name, channel_to_test
).split()
)
self._setup_multipass(channel_or_snap)
elif path.exists("/snap/bin/lxc") or backend == "lxc":
elif (path.exists("/snap/bin/lxc") and not self.backend) or self.backend == "lxc":
print("Creating lxc VM")
self.backend = "lxc"
if not attach_vm:
self._setup_lxc(channel_or_snap)
else:
raise Exception("Need to install multipass of lxc")
def _setup_lxc(self, channel_or_snap):
if not self.attached:
profiles = subprocess.check_output("/snap/bin/lxc profile list".split())
if "microk8s" not in profiles.decode():
subprocess.check_call("/snap/bin/lxc profile copy default microk8s".split())
@@ -74,17 +74,71 @@ class VM:
self.vm_name
).split()
)
time.sleep(20)
if channel_or_snap.startswith("/"):
self._transfer_install_local_snap_lxc(channel_or_snap)
else:
cmd_prefix = "/snap/bin/lxc exec {} -- script -e -c".format(self.vm_name).split()
cmd = ["snap install microk8s --classic --channel {}".format(channel_to_test)]
cmd = ["snap install microk8s --classic --channel {}".format(channel_or_snap)]
time.sleep(20)
subprocess.check_output(cmd_prefix + cmd)
else:
if channel_or_snap.startswith("/"):
self._transfer_install_local_snap_lxc(channel_or_snap)
else:
cmd = "/snap/bin/lxc exec {} -- ".format(self.vm_name).split()
cmd.append("sudo snap refresh microk8s --channel {}".format(channel_to_test))
cmd.append("sudo snap refresh microk8s --channel {}".format(channel_or_snap))
subprocess.check_call(cmd)
def _transfer_install_local_snap_lxc(self, channel_or_snap):
print("Installing snap from {}".format(channel_or_snap))
cmd_prefix = "/snap/bin/lxc exec {} -- script -e -c".format(self.vm_name).split()
cmd = ["rm -rf /var/tmp/microk8s.snap"]
subprocess.check_output(cmd_prefix + cmd)
cmd = "lxc file push {} {}/var/tmp/microk8s.snap".format(
channel_or_snap, self.vm_name
).split()
subprocess.check_output(cmd)
cmd = ["snap install /var/tmp/microk8s.snap --classic --dangerous"]
subprocess.check_output(cmd_prefix + cmd)
time.sleep(20)
def _setup_multipass(self, channel_or_snap):
if not self.attached:
subprocess.check_call(
"/snap/bin/multipass launch 18.04 -n {} -m 2G".format(self.vm_name).split()
)
if channel_or_snap.startswith("/"):
self._transfer_install_local_snap_multipass(channel_or_snap)
else:
raise Exception("Need to install multipass of lxc")
subprocess.check_call(
"/snap/bin/multipass exec {} -- sudo "
"snap install microk8s --classic --channel {}".format(
self.vm_name, channel_or_snap
).split()
)
else:
if channel_or_snap.startswith("/"):
self._transfer_install_local_snap_multipass(channel_or_snap)
else:
subprocess.check_call(
"/snap/bin/multipass exec {} -- sudo "
"snap refresh microk8s --channel {}".format(
self.vm_name, channel_or_snap
).split()
)
def _transfer_install_local_snap_multipass(self, channel_or_snap):
print("Installing snap from {}".format(channel_or_snap))
subprocess.check_call(
"/snap/bin/multipass transfer {} {}:/var/tmp/microk8s.snap".format(
channel_or_snap, self.vm_name
).split()
)
subprocess.check_call(
"/snap/bin/multipass exec {} -- sudo "
"snap install /var/tmp/microk8s.snap --classic --dangerous".format(self.vm_name).split()
)
def run(self, cmd):
"""
@@ -131,13 +185,16 @@ class TestCluster(object):
size = 3
for i in range(0, size):
print("Creating machine {}".format(i))
vm = VM()
vm = VM(backend)
vm.setup(channel_to_test)
print("Waiting for machine {}".format(i))
vm.run("/snap/bin/microk8s.status --wait-ready --timeout 120")
self.VM.append(vm)
else:
for vm_name in reuse_vms:
self.VM.append(VM(vm_name))
vm = VM(backend, vm_name)
vm.setup(channel_to_test)
self.VM.append(vm)
# Form cluster
vm_master = self.VM[0]
@@ -296,3 +353,12 @@ class TestCluster(object):
time.sleep(2)
continue
break
def test_no_cert_reissue_in_nodes(self):
"""
Test that each node has the cert no-reissue lock.
"""
print("Checking for the no re-issue lock")
for vm in self.VM:
lock_files = vm.run("ls /var/snap/microk8s/current/var/lock/")
assert "no-cert-reissue" in lock_files.decode()