alihan/buildbot
1
0
Fork 0
mirror of https://github.com/buildbot/buildbot.git synced 2021-05-20 10:47:20 +03:00
Code Issues Projects Releases Wiki Activity

worker: add option to delete leftover dirs

Browse Source 
Removing these leftover directories can be forgotten resulting in
increasing disk usage.

Add an option to create-workers to remove these directories when
detected (disabled by default to keep current behaviour).

> $ buildbot-worker create-worker --delete-leftover-dirs <dir> <server> <name> <password>
> mkdir -p <dir>/TFA-test
> 2020-01-17 14:38:55+0100 [Broker,client] Deleting directory 'TFA-test' that is not being used by the buildmaster

Signed-off-by: Robin Jarry <robin@jarry.cc>
Signed-off-by: Thomas Faivre <thomas.faivre@6wind.com>
This commit is contained in:
Robin Jarry
2020-01-16 13:05:31 +01:00
committed by Thomas Faivre
parent 86d8fcf957
commit 4c0fbc8d2d
9 changed files with 69 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
master/buildbot/newsfragments/worker_deleteleftoverdirs.feature Normal file
View File

@@ -0,0 +1 @@
New ``buildbot-worker create-worker --delete-leftover-dirs`` option to automatically remove obsolete builder directories

6
master/docs/manual/installation/worker.rst
View File

@@ -171,6 +171,12 @@ To use these, just include them on the ``buildbot-worker create-worker`` command
If set, the generated connection string starts with ``tls`` instead of with ``tcp``, allowing encrypted connection to the buildmaster.
Make sure the worker trusts the buildmasters certificate. If you have an non-authoritative certificate (CA is self-signed) see ``connection_string`` below.
.. option:: --delete-leftover-dirs
Can also be passed directly to the Worker constructor in :file:`buildbot.tac`.
If set, unexpected directories in worker base directory will be removed.
Otherwise, a warning will be displayed in :file:`twistd.log` so that you can manually remove them.
.. _Other-Worker-Configuration:
Other Worker Configuration

25
worker/buildbot_worker/base.py
View File

@@ -18,6 +18,7 @@ from __future__ import print_function
import multiprocessing
import os.path
import shutil
import socket
import sys
@@ -243,12 +244,13 @@ class BotBase(service.MultiService):
os_release_file = "/etc/os-release"
def __init__(self, basedir, unicode_encoding=None):
def __init__(self, basedir, unicode_encoding=None, delete_leftover_dirs=False):
service.MultiService.__init__(self)
self.basedir = basedir
self.numcpus = None
self.unicode_encoding = unicode_encoding or sys.getfilesystemencoding(
) or 'ascii'
self.delete_leftover_dirs = delete_leftover_dirs
self.builders = {}
# for testing purposes
@@ -302,9 +304,18 @@ class BotBase(service.MultiService):
for dir in os.listdir(self.basedir):
if os.path.isdir(os.path.join(self.basedir, dir)):
if dir not in wanted_dirs:
log.msg("I have a leftover directory '{0}' that is not "
"being used by the buildmaster: you can delete "
"it now".format(dir))
if self.delete_leftover_dirs:
log.msg("Deleting directory '{0}' that is not being "
"used by the buildmaster".format(dir))
try:
shutil.rmtree(dir)
except OSError as e:
log.msg("Cannot remove directory '{0}': "
"{1}".format(dir, e))
else:
log.msg("I have a leftover directory '{0}' that is not "
"being used by the buildmaster: you can delete "
"it now".format(dir))
defer.returnValue(retval)
@@ -381,11 +392,13 @@ class WorkerBase(service.MultiService):
def __init__(self, name, basedir,
umask=None,
unicode_encoding=None):
unicode_encoding=None,
delete_leftover_dirs=False):
service.MultiService.__init__(self)
self.name = name
bot = self.Bot(basedir, unicode_encoding=unicode_encoding)
bot = self.Bot(basedir, unicode_encoding=unicode_encoding,
delete_leftover_dirs=delete_leftover_dirs)
bot.setServiceParent(self)
self.bot = bot
self.umask = umask

6
worker/buildbot_worker/pb.py
View File

@@ -179,7 +179,8 @@ class Worker(WorkerBase, service.MultiService):
def __init__(self, buildmaster_host, port, name, passwd, basedir,
keepalive, usePTY=None, keepaliveTimeout=None, umask=None,
maxdelay=None, numcpus=None, unicode_encoding=None, useTls=None,
allow_shutdown=None, maxRetries=None, connection_string=None):
allow_shutdown=None, maxRetries=None, connection_string=None,
delete_leftover_dirs=False):
assert usePTY is None, "worker-side usePTY is not supported anymore"
assert (connection_string is None or
@@ -189,7 +190,8 @@ class Worker(WorkerBase, service.MultiService):
service.MultiService.__init__(self)
WorkerBase.__init__(
self, name, basedir, umask=umask, unicode_encoding=unicode_encoding)
self, name, basedir, umask=umask, unicode_encoding=unicode_encoding,
delete_leftover_dirs=delete_leftover_dirs)
if keepalive == 0:
keepalive = None

4
worker/buildbot_worker/scripts/create_worker.py
View File

@@ -59,11 +59,13 @@ numcpus = %(numcpus)s
allow_shutdown = %(allow-shutdown)s
maxretries = %(maxretries)s
use_tls = %(use-tls)s
delete_leftover_dirs = %(delete-leftover-dirs)s
s = Worker(buildmaster_host, port, workername, passwd, basedir,
keepalive, umask=umask, maxdelay=maxdelay,
numcpus=numcpus, allow_shutdown=allow_shutdown,
maxRetries=maxretries, useTls=use_tls)
maxRetries=maxretries, useTls=use_tls,
delete_leftover_dirs=delete_leftover_dirs)
s.setServiceParent(application)
"""]

2
worker/buildbot_worker/scripts/runner.py
View File

@@ -108,6 +108,8 @@ class CreateWorkerOptions(MakerBase):
"Do not permit buildmaster rotate logs by itself"],
['use-tls', None,
"Uses TLS to connect to master"],
['delete-leftover-dirs', None,
'Delete folders that are not required by the master on connection'],
]
optParameters = [
["keepalive", "k", 600,

25
worker/buildbot_worker/test/unit/test_scripts_create_worker.py
View File

@@ -480,6 +480,7 @@ class TestCreateWorker(misc.StdoutAssertionsMixin, unittest.TestCase):
"relocatable": False,
"quiet": False,
"use-tls": False,
"delete-leftover-dirs": False,
# options
"basedir": "bdir",
"allow-shutdown": None,
@@ -623,6 +624,7 @@ class TestCreateWorker(misc.StdoutAssertionsMixin, unittest.TestCase):
allow_shutdown=options["allow-shutdown"],
maxRetries=options["maxretries"],
useTls=options["use-tls"],
delete_leftover_dirs=options["delete-leftover-dirs"],
)
# check that Worker instance attached to application
@@ -802,3 +804,26 @@ class TestCreateWorker(misc.StdoutAssertionsMixin, unittest.TestCase):
# there should be no output on stdout
self.assertWasQuiet()
def testDeleteLeftoverDirs(self):
"""
test calling createWorker() with --delete-leftover-dirs flag
"""
options = self.options.copy()
options["delete-leftover-dirs"] = True
# patch _make*() functions to do nothing
self.setUpMakeFunctions()
# call createWorker() and check that we get success exit code
self.assertEqual(create_worker.createWorker(options), 0,
"unexpected exit code")
# check _make*() functions were called with correct arguments
expected_tac_contents = ("".join(create_worker.workerTACTemplate)) % options
self.assertMakeFunctionsCalls(self.options["basedir"],
expected_tac_contents,
self.options["quiet"])
# check that correct info message was printed
self.assertStdoutEqual("worker configured in bdir\n")

4
worker/docker/buildbot.tac
View File

@@ -33,8 +33,10 @@ umask = None
maxdelay = 300
allow_shutdown = None
maxretries = 10
delete_leftover_dirs = False
s = Worker(buildmaster_host, port, workername, passwd, basedir,
keepalive, umask=umask, maxdelay=maxdelay,
allow_shutdown=allow_shutdown, maxRetries=maxretries)
allow_shutdown=allow_shutdown, maxRetries=maxretries,
delete_leftover_dirs=delete_leftover_dirs)
s.setServiceParent(application)

6
worker/docs/buildbot-worker.1
View File

@@ -62,6 +62,9 @@ create-worker
.I COUNT
]
[
.BR \-\-delete\-leftover\-dirs
]
[
.BR \-\-verbose
]
.I PATH
@@ -168,6 +171,9 @@ Set whether child processes should be run in a pty (0 means do not run in a
pty).
Default value is 0.
.TP
.BR \-\-delete\-leftover\-dirs
Set to remove unexpected directories in worker base directory.
.TP
.I PATH
Path to worker base directory.
.TP