alihan/microk8s
1
0
Fork 0
mirror of https://github.com/ubuntu/microk8s.git synced 2021-05-23 02:23:41 +03:00
Code Issues Projects Releases Wiki Activity

Add flake8 to CI checks (#1482)

Browse Source 
* Add flake8 to CI checks

black doesn't catch some things such as unused variables or trailing
spaces in docstrings, so add flake8 to the CI process.

flake8 and black disagree on a few things, and black is probably in the
right, at least according to pep8. So, add a few lint ignores, as in
https://github.com/psf/black/blob/master/docs/compatible_configs.md
This commit is contained in:
Kenneth Koski
2020-09-01 03:33:22 -05:00
committed by GitHub
parent 6acd4820b4
commit 7c5607a6b6
21 changed files with 101 additions and 94 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
.flake8 Normal file
View File

@@ -0,0 +1,3 @@
[flake8]
max-line-length = 100
extend-ignore = E203, W503

7
.github/workflows/check-formatting.yml vendored
View File

@@ -14,7 +14,10 @@ jobs:
uses: actions/checkout@v2
- name: Install dependencies
run: sudo pip3 install black
run: sudo pip3 install black flake8
- name: Check formatting
run: black --check . microk8s-resources/actions/enable.kubeflow.sh
run: |
set -eux
black --check . microk8s-resources/actions/enable.kubeflow.sh
flake8 . microk8s-resources/actions/enable.kubeflow.sh

8
installer/cli/microk8s.py
View File

@@ -103,7 +103,7 @@ def _show_install_help():
--help Show this message and exit.
--cpu Cores used by MicroK8s (default={})
--mem RAM in GB used by MicroK8s (default={})
--disk Maximum volume in GB of the dynamically expandable hard disk to be used (default={})
--disk Max volume in GB of the dynamically expandable hard disk to be used (default={})
--channel Kubernetes version to install (default={})
-y, --assume-yes Automatic yes to prompts"""
Echo.info(
@@ -260,7 +260,7 @@ def dashboard_proxy() -> None:
instance.run(command)
except KeyboardInterrupt:
return
except ProviderInstanceNotFoundError as provider_error:
except ProviderInstanceNotFoundError:
_not_installed(echo)
return 1
@@ -299,7 +299,7 @@ def run(cmd) -> None:
command = cmd[0]
cmd[0] = "microk8s.{}".format(command)
instance.run(cmd)
except ProviderInstanceNotFoundError as provider_error:
except ProviderInstanceNotFoundError:
_not_installed(echo)
return 1
@@ -331,7 +331,7 @@ def _get_microk8s_commands() -> List:
return complete
else:
return ["start", "stop"]
except ProviderNotFound as provider_error:
except ProviderNotFound:
return ["start", "stop"]

2
installer/setup.py
View File

@@ -7,7 +7,7 @@ setup(
license='Apache-2.0',
author='Joe Borg',
author_email='joseph.borg@canonical.com',
description='MicroK8s is a small, fast, single-package Kubernetes for developers, IoT and edge.',
description='MicroK8s is a small, fast, single-package Kubernetes for developers, IoT and edge',
packages=['cli', 'common', 'vm_providers', 'vm_providers._multipass', 'vm_providers.repo'],
install_requires=[
'setuptools',

3
installer/vm_providers/_multipass/_multipass.py
View File

@@ -15,7 +15,6 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import logging
import os
import sys
from typing import Dict, Optional, Sequence
@@ -48,8 +47,6 @@ class Multipass(Provider):
return "multipass"
def run(self, command: Sequence[str], hide_output: bool = False) -> Optional[bytes]:
env_command = self._get_env_command()
cmd = ["sudo"]
cmd.extend(command)
self._log_run(cmd)

2
microk8s-resources/actions/enable.kubeflow.sh
View File

@@ -211,7 +211,7 @@ def main():
args[arg] = strtobool(args[arg])
with open("/proc/meminfo") as f:
memtotal_lines = [l for l in f.readlines() if "MemTotal" in l]
memtotal_lines = [line for line in f.readlines() if "MemTotal" in line]
try:
total_mem = int(memtotal_lines[0].split(" ")[-2])

11
scripts/cluster/agent.py
View File

@@ -9,6 +9,7 @@ import string
import subprocess
import sys
import time
from typing import List
import yaml
@@ -24,7 +25,7 @@ from .common.utils import (
try_initialise_cni_autodetect_for_clustering,
)
from flask import Flask, jsonify, request, abort, Response
from flask import Flask, jsonify, request, Response
app = Flask(__name__)
CLUSTER_API = "cluster/api/v1.0"
@@ -292,8 +293,8 @@ def join_node_etcd():
if is_node_running_dqlite():
msg = (
"Failed to join the cluster. This is an HA dqlite cluster. \n"
"Please, retry after enabling HA on this joining node with 'microk8s enable ha-cluster'."
"Failed to join the cluster. This is an HA dqlite cluster.\n"
"Please retry after enabling HA on this joining node with 'microk8s enable ha-cluster'."
)
error_msg = {"error": msg}
return Response(json.dumps(error_msg), mimetype='application/json', status=501)
@@ -558,12 +559,8 @@ def join_node_dqlite():
"""
if request.headers['Content-Type'] == 'application/json':
token = request.json['token']
hostname = request.json['hostname']
port = request.json['port']
else:
token = request.form['token']
hostname = request.form['hostname']
port = request.form['port']
if not is_valid(token):
error_msg = {"error": "Invalid token"}

4
scripts/cluster/common/utils.py
View File

@@ -20,7 +20,7 @@ def try_set_file_permissions(file):
os.chmod(file, 0o660)
try:
shutil.chown(file, group='microk8s')
except:
except LookupError:
# not setting the group means only the current user can access the file
pass
@@ -169,7 +169,7 @@ def is_same_server(hostname, ip):
hname, _, _ = socket.gethostbyaddr(ip)
if hname == hostname:
return True
except:
except socket.error:
# Ignore any unresolvable IP by host, surely this is not from the same node.
pass

26
scripts/cluster/join.py
View File

@@ -7,6 +7,7 @@ import os
import getopt
import sys
import time
from typing import Dict
import netifaces
import requests
@@ -19,7 +20,6 @@ import json
from common.utils import (
try_set_file_permissions,
is_node_running_dqlite,
get_dqlite_port,
get_cluster_agent_port,
try_initialise_cni_autodetect_for_clustering,
)
@@ -352,10 +352,12 @@ def reset_current_dqlite_installation():
"""
if is_leader_without_successor():
print(
"This node currently holds the only copy of the Kubernetes database so it cannot leave the cluster."
"This node currently holds the only copy of the Kubernetes "
"database so it cannot leave the cluster."
)
print(
"To remove this node you can either first remove all other nodes with 'microk8s remove-node' or"
"To remove this node you can either first remove all other "
"nodes with 'microk8s remove-node' or"
)
print("form a highly available cluster by adding at least three nodes.")
exit(3)
@@ -504,9 +506,10 @@ def get_dqlite_endpoints():
def is_leader_without_successor():
"""
Check if the current node is safe to be removed. Check if this node acts as a leader to a cluster
with more than one nodes where there is no other node to take over the leadership.
"""Checks if the current node is safe to be removed.
Check if this node acts as a leader to a cluster with more than one nodes where there
is no other node to take over the leadership.
:return: True if this node is the leader without a successor.
"""
@@ -534,7 +537,6 @@ def is_leader_without_successor():
is_voter = False
for ep in ep_addresses:
found = False
for ip in local_ips:
if "{}:".format(ip) in ep[0]:
# ep[1] == ep[Role] == 0 means we are voters
@@ -663,8 +665,10 @@ def remove_dqlite_node(node, force=False):
delete_dqlite_node([node_ep], my_ep)
elif node_ep and not force:
print(
"Removal failed. Node {} is registered with dqlite. Please, run first 'microk8s leave' on the departing node. \n"
"If the node is not available anymore and will never attempt to join the cluster in the future use the '--force' flag \n"
"Removal failed. Node {} is registered with dqlite. "
"Please, run first 'microk8s leave' on the departing node. \n"
"If the node is not available anymore and will never attempt to join the cluster "
"in the future use the '--force' flag \n"
"to unregister the node while removing it.".format(node)
)
exit(1)
@@ -894,8 +898,8 @@ def join_dqlite(connection_parts):
store_callback_token(info["callback_token"])
update_dqlite(info["cluster_cert"], info["cluster_key"], info["voters"], hostname_override)
# We want to update the local CNI yaml but we do not want to apply it. The cni is applied already
# in the cluster we join
# We want to update the local CNI yaml but we do not want to apply it.
# The cni is applied already in the cluster we join
try_initialise_cni_autodetect_for_clustering(master_ip, apply_cni=False)

5
scripts/wrappers/common/utils.py
View File

@@ -105,7 +105,8 @@ def is_cluster_locked():
clusterLockFile = os.path.expandvars("${SNAP_DATA}/var/lock/clustered.lock")
if os.path.isfile(clusterLockFile):
print(
"This MicroK8s deployment is acting as a node in a cluster. Please use the microk8s status on the master."
"This MicroK8s deployment is acting as a node in a cluster. "
"Please use the microk8s status on the master."
)
exit(0)
@@ -129,7 +130,7 @@ def exit_if_no_permission():
user = getpass.getuser()
# test if we can access the default kubeconfig
clientConfigFile = os.path.expandvars("${SNAP_DATA}/credentials/client.config")
if os.access(clientConfigFile, os.R_OK) == False:
if not os.access(clientConfigFile, os.R_OK):
print("Insufficient permissions to access MicroK8s.")
print(
"You can either try again with sudo or add the user {} to the 'microk8s' group:".format(

2
scripts/wrappers/dbctl.py
View File

@@ -57,7 +57,6 @@ def backup(fname=None, debug=False):
:param debug: show debug output
"""
snap_path = os.environ.get('SNAP')
snapdata_path = os.environ.get('SNAP_DATA')
# snap_path = '/snap/microk8s/current'
# snapdata_path = '/var/snap/microk8s/current'
@@ -84,7 +83,6 @@ def backup(fname=None, debug=False):
arcname=os.path.basename("{}/{}".format(tmpdirname, fname)),
)
target_file = '{}/var/tmp/{}'.format(snapdata_path, fname_tar)
print("The backup is: {}".format(fname_tar))
except subprocess.CalledProcessError as e:
print("Backup process failed. {}".format(e))

21
scripts/wrappers/refresh-certs.py
View File

@@ -49,18 +49,18 @@ def undo_refresh():
try:
subprocess.check_call('cp -r {}/certs {}/'.format(backup_dir, snapdata_path).split())
subprocess.check_call('cp -r {}/credentials {}'.format(backup_dir, snapdata_path).split())
except subprocess.CalledProcessError as e:
except subprocess.CalledProcessError:
click.echo('Failed to recover certificates')
exit(4)
try:
subprocess.check_call('{}/microk8s-stop.wrapper'.format(snap_path).split())
except:
except subprocess.CalledProcessError:
pass
try:
subprocess.check_call('{}/microk8s-start.wrapper'.format(snap_path).split())
except subprocess.CalledProcessError as e:
except subprocess.CalledProcessError:
click.echo('Failed to start MicroK8s after reverting the certificates')
exit(4)
@@ -111,7 +111,7 @@ def refresh_ca():
click.echo("Creating new certificates")
try:
produce_certs()
except subprocess.CalledProcessError as e:
except subprocess.CalledProcessError:
click.echo("Failed to produce new certificates. Reverting.")
undo_refresh()
exit(20)
@@ -119,7 +119,8 @@ def refresh_ca():
update_configs()
msg = """
The CA certificates have been replaced. Kubernetes will restart the pods of your workloads.
Any worker nodes you may have in your cluster need to be removed and re-joined to become aware of the new CA.
Any worker nodes you may have in your cluster need to be removed and \
re-joined to become aware of the new CA.
"""
click.echo(msg)
@@ -179,7 +180,7 @@ def install_ca(ca_dir):
click.echo("Installing provided certificates")
try:
install_certs(ca_dir)
except subprocess.CalledProcessError as e:
except subprocess.CalledProcessError:
click.echo("Failed to produce new certificates. Reverting.")
undo_refresh()
exit(20)
@@ -187,7 +188,8 @@ def install_ca(ca_dir):
update_configs()
msg = """
The CA certificates have been replaced. Kubernetes will restart the pods of your workloads.
Any worker nodes you may have in your cluster need to be removed and re-joined to become aware of the new CA.
Any worker nodes you may have in your cluster need to be removed and \
re-joined to become aware of the new CA.
"""
click.echo(msg)
@@ -216,7 +218,7 @@ def refresh_certs(ca_dir, undo, check, help):
show_help()
exit(0)
if not ca_dir is None and (undo or check):
if ca_dir is not None and (undo or check):
click.echo("Please do not set any options in combination with the CA_DIR.")
exit(1)
@@ -224,7 +226,8 @@ def refresh_certs(ca_dir, undo, check, help):
click.echo("Please select either one of the options -c or -u, not both.")
exit(2)
# Operations here will need root privileges as some of the credentials and certificates are used by system services.
# Operations here will need root privileges as some of the credentials
# and certificates are used by system services.
exit_if_no_root()
if check:
check_certificate()

2
scripts/wrappers/upgrade.py
View File

@@ -144,7 +144,7 @@ def get_nodes_info(safe=True):
continue
node_info = [(parts[0], parts[1])]
except subprocess.CalledProcessError:
print("Error in gathering cluster node information. Upgrade aborted.".format(host))
print("Error in gathering cluster node information. Upgrade aborted.")
exit(1)
else:
if os.path.isfile(callback_tokens_file):

22
tests/test-addons.py
View File

@@ -13,7 +13,6 @@ from validators import (
validate_registry,
validate_forward,
validate_metrics_server,
validate_prometheus,
validate_fluentd,
validate_jaeger,
validate_linkerd,
@@ -30,7 +29,7 @@ from utils import (
microk8s_disable,
microk8s_reset,
)
from subprocess import Popen, PIPE, STDOUT, CalledProcessError, check_call
from subprocess import PIPE, STDOUT, CalledProcessError, check_call, run
class TestAddons(object):
@@ -96,7 +95,7 @@ class TestAddons(object):
"""
try:
print("Enabling gpu")
gpu_enable_outcome = microk8s_enable("gpu")
microk8s_enable("gpu")
except CalledProcessError:
# Failed to enable gpu. Skip the test.
print("Could not enable GPU support")
@@ -142,10 +141,13 @@ class TestAddons(object):
Sets up and validates Cilium.
"""
print("Enabling Cilium")
p = Popen(
"/snap/bin/microk8s.enable cilium".split(), stdout=PIPE, stdin=PIPE, stderr=STDOUT
run(
"/snap/bin/microk8s.enable cilium".split(),
stdout=PIPE,
input=b'N\n',
stderr=STDOUT,
check=True,
)
p.communicate(input=b'N\n')[0]
print("Validating Cilium")
validate_cilium()
print("Disabling Cilium")
@@ -163,8 +165,12 @@ class TestAddons(object):
Sets up and validates Multus.
"""
print("Enabling Multus")
p = Popen(
"/snap/bin/microk8s.enable multus".split(), stdout=PIPE, stdin=PIPE, stderr=STDOUT
run(
"/snap/bin/microk8s.enable multus".split(),
stdout=PIPE,
stdin=PIPE,
stderr=STDOUT,
check=True,
)
print("Validating Multus")
validate_multus()

6
tests/test-cluster.py
View File

@@ -21,9 +21,9 @@ class VM:
"""
def __init__(self, attach_vm=None):
"""
Detect the available backends and instantiate a VM. If attach_vm is provided we just make sure the right
MicroK8s is deployed.
"""Detect the available backends and instantiate a VM.
If `attach_vm` is provided we just make sure the right MicroK8s is deployed.
:param attach_vm: the name of the VM we want to reuse
"""
rnd_letters = ''.join(random.choice(string.ascii_lowercase) for i in range(6))

2
tests/test-live-addons.py
View File

@@ -8,8 +8,6 @@ from validators import (
validate_registry,
validate_forward,
validate_metrics_server,
validate_prometheus,
validate_fluentd,
validate_jaeger,
validate_cilium,
validate_multus,

1
tests/test-upgrade-path.py
View File

@@ -3,7 +3,6 @@ import os
import platform
import time
import requests
from subprocess import check_call, CalledProcessError, check_output
from utils import (
wait_for_installation,
run_until_success,

29
tests/test-upgrade.py
View File

@@ -1,21 +1,16 @@
import pytest
import os
import platform
import time
import requests
from validators import (
validate_dns_dashboard,
validate_storage,
validate_ingress,
validate_ambassador,
validate_gpu,
validate_registry,
validate_forward,
validate_metrics_server,
validate_prometheus,
validate_fluentd,
validate_jaeger,
validate_kubeflow,
validate_cilium,
validate_metallb_config,
validate_multus,
@@ -68,7 +63,7 @@ class TestUpgrade(object):
assert "Nothing to do for" not in enable
validate_dns_dashboard()
test_matrix['dns_dashboard'] = validate_dns_dashboard
except:
except CalledProcessError:
print('Will not test dns-dashboard')
try:
@@ -76,7 +71,7 @@ class TestUpgrade(object):
assert "Nothing to do for" not in enable
validate_storage()
test_matrix['storage'] = validate_storage
except:
except CalledProcessError:
print('Will not test storage')
try:
@@ -84,7 +79,7 @@ class TestUpgrade(object):
assert "Nothing to do for" not in enable
validate_ingress()
test_matrix['ingress'] = validate_ingress
except:
except CalledProcessError:
print('Will not test ingress')
try:
@@ -92,7 +87,7 @@ class TestUpgrade(object):
assert "Nothing to do for" not in enable
validate_gpu()
test_matrix['gpu'] = validate_gpu
except:
except CalledProcessError:
print('Will not test gpu')
try:
@@ -100,13 +95,13 @@ class TestUpgrade(object):
assert "Nothing to do for" not in enable
validate_registry()
test_matrix['registry'] = validate_registry
except:
except CalledProcessError:
print('Will not test registry')
try:
validate_forward()
test_matrix['forward'] = validate_forward
except:
except CalledProcessError:
print('Will not test port forward')
try:
@@ -114,7 +109,7 @@ class TestUpgrade(object):
assert "Nothing to do for" not in enable
validate_metrics_server()
test_matrix['metrics_server'] = validate_metrics_server
except:
except CalledProcessError:
print('Will not test the metrics server')
# AMD64 only tests
@@ -146,7 +141,7 @@ class TestUpgrade(object):
assert "Nothing to do for" not in enable
validate_fluentd()
test_matrix['fluentd'] = validate_fluentd
except:
except CalledProcessError:
print('Will not test the fluentd')
try:
@@ -154,7 +149,7 @@ class TestUpgrade(object):
assert "Nothing to do for" not in enable
validate_jaeger()
test_matrix['jaeger'] = validate_jaeger
except:
except CalledProcessError:
print('Will not test the jaeger addon')
try:
@@ -162,7 +157,7 @@ class TestUpgrade(object):
assert "Nothing to do for" not in enable
validate_cilium()
test_matrix['cilium'] = validate_cilium
except:
except CalledProcessError:
print('Will not test the cilium addon')
try:
ip_ranges = (
@@ -172,7 +167,7 @@ class TestUpgrade(object):
assert "MetalLB is enabled" in enable and "Nothing to do for" not in enable
validate_metallb_config(ip_ranges)
test_matrix['metallb'] = validate_metallb_config
except:
except CalledProcessError:
print("Will not test the metallb addon")
try:
@@ -180,7 +175,7 @@ class TestUpgrade(object):
assert "Nothing to do for" not in enable
validate_multus()
test_matrix['multus'] = validate_multus
except:
except CalledProcessError:
print('Will not test the multus addon')
# Refresh the snap to the target

4
tests/utils.py
View File

@@ -152,9 +152,9 @@ def wait_for_namespace_termination(namespace, timeout_insec=360):
while True:
try:
cmd = '/snap/bin/microk8s.kubectl get ns {}'.format(namespace)
output = check_output(cmd.split()).strip().decode('utf8')
check_output(cmd.split()).strip().decode('utf8')
print('Waiting...')
except CalledProcessError as err:
except CalledProcessError:
if datetime.datetime.now() > deadline:
raise
else:

18
tests/validators.py
View File

@@ -4,6 +4,7 @@ import re
import requests
import platform
import yaml
import subprocess
from utils import (
kubectl,
@@ -27,11 +28,13 @@ def validate_dns_dashboard():
while attempt > 0:
try:
output = kubectl(
"get --raw /api/v1/namespaces/kube-system/services/https:kubernetes-dashboard:/proxy/"
"get "
"--raw "
"/api/v1/namespaces/kube-system/services/https:kubernetes-dashboard:/proxy/"
)
if "Kubernetes Dashboard" in output:
break
except:
except subprocess.CalledProcessError:
pass
time.sleep(10)
attempt -= 1
@@ -99,7 +102,7 @@ def common_ingress():
if resp.status_code == 200 and "microbot.png" in resp.content.decode("utf-8"):
service_ok = True
break
except:
except requests.RequestException:
time.sleep(5)
attempt -= 1
if resp.status_code != 200 or "microbot.png" not in resp.content.decode("utf-8"):
@@ -110,7 +113,7 @@ def common_ingress():
if resp.status_code == 200 and "microbot.png" in resp.content.decode("utf-8"):
service_ok = True
break
except:
except requests.RequestException:
time.sleep(5)
attempt -= 1
@@ -278,7 +281,7 @@ def validate_forward():
resp = requests.get("http://localhost:5123")
if resp.status_code == 200:
break
except:
except requests.RequestException:
pass
attempt -= 1
time.sleep(2)
@@ -297,7 +300,7 @@ def validate_metrics_server():
output = kubectl("get --raw /apis/metrics.k8s.io/v1beta1/pods")
if "PodMetricsList" in output:
break
except:
except subprocess.CalledProcessError:
pass
time.sleep(10)
attempt -= 1
@@ -345,7 +348,7 @@ def validate_jaeger():
output = kubectl("get ingress")
if "simplest-query" in output:
break
except:
except subprocess.CalledProcessError:
pass
time.sleep(2)
attempt -= 1
@@ -378,7 +381,6 @@ def validate_linkerd():
timeout_insec=300,
)
print("Linkerd proxy injector up and running.")
### Disabling this test because the deletion of the namespace get stuck.
here = os.path.dirname(os.path.abspath(__file__))
manifest = os.path.join(here, "templates", "emojivoto.yaml")
kubectl("apply -f {}".format(manifest))

17
tests/verify-branches.py
View File

@@ -4,15 +4,15 @@ from subprocess import check_output
class TestMicrok8sBranches(object):
def test_branches(self):
"""
We need to make sure the LP builders pointing to the master github branch are only pushing to the
latest and current k8s stable snap tracks. An indication that this is not enforced is that
we do not have a branch for the k8s release for the previous stable release. Let me clarify
with an example.
"""Ensures LP builders push to correct snap tracks.
Assuming upstream stable k8s release is v1.12.x, there has to be a 1.11 github branch used by
the respective LP builders for building the v1.11.y.
We need to make sure the LP builders pointing to the master github branch are only pushing
to the latest and current k8s stable snap tracks. An indication that this is not enforced is
that we do not have a branch for the k8s release for the previous stable release. Let me
clarify with an example.
Assuming upstream stable k8s release is v1.12.x, there has to be a 1.11 github branch used
by the respective LP builders for building the v1.11.y.
"""
upstream_version = self._upstream_release()
assert upstream_version
@@ -48,7 +48,8 @@ class TestMicrok8sBranches(object):
def _get_max_minor(self, major):
"""Get the latest minor release of the provided major.
For example if you use 1 as major you will get back X where X gives you latest 1.X release."""
For example if you use 1 as major you will get back X where X gives you latest 1.X release.
"""
minor = 0
while self._upstream_release_exists(major, minor):
minor += 1