alihan/buildbot
1
0
Fork 0
mirror of https://github.com/buildbot/buildbot.git synced 2021-05-20 10:47:20 +03:00
Code Issues Projects Releases Wiki Activity

Remove buildbot-slave.

Browse Source 
There have been no substantive changes to the slave in 0.9.x, and
buildbot-worker provides equivalent functionality, so get rid of the
duplication.
This commit is contained in:
Tom Prince
2016-06-24 10:16:38 -06:00
parent 4a94332eec
commit 240442faf4
93 changed files with 20 additions and 12430 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
.travis.yml
View File

@@ -51,9 +51,6 @@ env:
# Configuration to run `python setup.py test` to check this test runner.
# - TWISTED=latest SQLALCHEMY=latest TESTS=setuppy_test
# Configuration to run tests without installed buildbot-slave.
- TWISTED=latest SQLALCHEMY=latest TESTS=trial_worker_only
cache:
directories:
- $HOME/.cache/pip
@@ -71,7 +68,7 @@ cache:
matrix:
fast_finish: true
include:
# Tests of buildbot-worker and buildbot-slave on python 2.6
# Tests of buildbot-worker on python 2.6
# Specify SQLALCHEMY=latest to avoid errors installing.
- python: "2.6"
env: TWISTED=14.0.2 TESTS=trial_worker SQLALCHEMY=latest
@@ -95,7 +92,6 @@ install:
[ $TESTS != trial -a $TESTS != coverage -a $TESTS != lint -a $TESTS != js ] || \
pip install -e pkg \
-e master[tls,test] \
-e slave \
-e worker \
MySQL-python \
psycopg2 \
@@ -103,10 +99,7 @@ install:
- |
[ $TESTS != trial_worker ] || \
pip install -e slave \
-e worker \
- "[ $TESTS != trial_worker_only ] || pip install -e pkg -e master[tls,test] -e worker"
pip install -e worker
# install buildbot_www from pip in order to run the www tests
- "[ $TESTS != trial -a $TESTS != coverage ] || pip install --pre buildbot_www"
@@ -115,8 +108,7 @@ install:
# for python 3 we dont install everything yet..
- |
[ $TESTS != py3 ] || \
pip install -e slave \
-e worker \
pip install -e worker \
future
# Run additional tests only in latest configuration
@@ -147,13 +139,13 @@ before_script:
# Tests running commands
script:
- "[ $TESTS != js ] || make frontend_install_tests"
- "[ $TESTS != trial ] || trial --reporter=text --rterrors buildbot.test buildslave.test buildbot_worker.test"
- "[ $TESTS != trial_worker ] || trial --reporter=text --rterrors buildslave.test buildbot_worker.test"
- "[ $TESTS != trial ] || trial --reporter=text --rterrors buildbot.test buildbot_worker.test"
- "[ $TESTS != trial_worker ] || trial --reporter=text --rterrors buildbot_worker.test"
# run tests under coverage for latest only (it's slower..)
- "[ $TESTS != coverage ] || coverage run --rcfile=.coveragerc $(which trial) --reporter=text --rterrors buildbot.test buildslave.test buildbot_worker.test"
- "[ $TESTS != coverage ] || coverage run --rcfile=.coveragerc $(which trial) --reporter=text --rterrors buildbot.test buildbot_worker.test"
# run tests that are know to work on py3
- "[ $TESTS != py3 ] || trial --reporter=text --rterrors buildslave.test buildbot_worker.test"
- "[ $TESTS != py3 ] || trial --reporter=text --rterrors buildbot_worker.test"
# Run additional tests in their separate job
- "[ $TESTS != lint ] || make pylint"
@@ -167,11 +159,8 @@ script:
- "[ $TESTS != docs ] || make -C master/docs SPHINXOPTS=-q linkcheck"
- "[ $TESTS != setuppy_test ] || (cd master; python setup.py test)"
- "[ $TESTS != setuppy_test ] || (cd slave; python setup.py test)"
- "[ $TESTS != setuppy_test ] || (cd worker; python setup.py test)"
- "[ $TESTS != trial_worker_only ] || trial --reporter=text --rterrors buildbot.test buildbot_worker.test"
notifications:
email: false

4
Makefile
View File

@@ -11,14 +11,12 @@ docs:
# pylint the whole sourcecode (validate.sh will do that as well, but only process the modified files)
pylint:
$(MAKE) -C master pylint; master_res=$$?; \
$(MAKE) -C slave pylint; slave_res=$$?; \
$(MAKE) -C worker pylint; worker_res=$$?; \
if [ $$master_res != 0 ] || [ $$slave_res != 0 ] || [ $$worker_res != 0 ]; then exit 1; fi
if [ $$master_res != 0 ] || [ $$worker_res != 0 ]; then exit 1; fi
# flake8 the whole sourcecode (validate.sh will do that as well, but only process the modified files)
flake8:
$(MAKE) -C master flake8
$(MAKE) -C slave flake8
$(MAKE) -C worker flake8
flake8 --config=common/flake8rc www/*/buildbot_*/
flake8 --config=common/flake8rc www/*/setup.py

3
appveyor.yml
View File

@@ -23,14 +23,13 @@ install:
- "%PYTHON%\\python.exe -m pip install -e pkg"
- "%PYTHON%\\python.exe -m pip install -e master[tls,test]"
- "%PYTHON%\\python.exe -m pip install -e slave"
- "%PYTHON%\\python.exe -m pip install -e worker"
- "%PYTHON%\\python.exe -m pip list"
build: false
test_script:
- "%PYTHON%\\python.exe %PYTHON%\\Scripts\\trial.py --reporter=text --rterrors buildbot.test buildslave.test buildbot_worker.test"
- "%PYTHON%\\python.exe %PYTHON%\\Scripts\\trial.py --reporter=text --rterrors buildbot.test buildbot_worker.test"
on_failure:
# Store _trial_temp directory as artifact on build failure.

2
circle.yml
View File

@@ -8,7 +8,7 @@ machine:
dependencies:
override:
- pyenv global 2.7.11
- pip install -e pkg -e master[docs] -e slave -e worker
- pip install -e pkg -e master[docs] -e worker
test:
override:

1
common/coveragerc
View File

@@ -25,7 +25,6 @@ exclude_lines =
include =
master/*
slave/*
worker/*
omit =

2
common/merge_and_pep8.sh
View File

@@ -21,7 +21,7 @@ function unittests()
{
status run the whole test suite as a double check
find . -name \*.pyc -exec rm {} \;
trial --reporter=text buildslave buildbot_worker buildbot
trial --reporter=text buildbot_worker buildbot
if [[ $? != 0 ]]
then
echo "Oups.. the tests are failing, better resolve them now before the big autopep8 work"

4
common/validate.sh
View File

@@ -1,5 +1,5 @@
#! /bin/bash
TEST='buildbot.test buildslave.test buildbot_worker.test'
TEST='buildbot.test buildbot_worker.test'
# if stdout is a terminal define some colors
# validate.sh can be run as hook from GUI git clients, such as git-gui
@@ -162,7 +162,7 @@ if ! $quick; then
elif [ -z `command -v cctrial` ]; then
warning "Skipping Python Tests ('pip install cctrial' for quick tests)"
else
cctrial -H buildbot buildslave buildbot_worker || not_ok "Python tests failed"
cctrial -H buildbot buildbot_worker || not_ok "Python tests failed"
fi
status "checking formatting"

2
master/docs/developer/tests.rst
View File

@@ -1,7 +1,7 @@
Buildbot's Test Suite
=====================
Buildbot's master tests are under ``buildbot.test``, ``buildbot-worker`` package tests are under ``buildbot_worker.test``, and ``buildbot-slave`` package tests are under ``buildslave.test``.
Buildbot's master tests are under ``buildbot.test``, ``buildbot-worker`` package tests are under ``buildbot_worker.test``.
Tests for the workers are similar to the master, although in some cases helpful functionality on the master is not re-implemented on the worker.
Suites

2
master/docs/manual/installation/worker.rst
View File

@@ -185,7 +185,7 @@ During project lifetime worker has transitioned over few states:
1. Before Buildbot version 0.8.1 worker were integral part of ``buildbot`` package distribution.
2. Starting from Buildbot version 0.8.1 worker were extracted from ``buildbot`` package to ``buildbot-slave`` package.
3. Starting from Buildbot version 0.9.0 ``buildbot-slave`` were deprecated in favor of ``buildbot-worker`` package.
3. Starting from Buildbot version 0.9.0 the ``buildbot-slave`` package was renamed to ``buildbot-worker``.
Upgrading a Worker to buildbot-slave 0.8.1
''''''''''''''''''''''''''''''''''''''''''

4
master/docs/manual/worker-transition.rst
View File

@@ -536,10 +536,10 @@ List of database-related changes in API (fallback for old API is provided):
``buildbot-worker``
-------------------
``buildbot-slave`` package has been deprecated in favor of ``buildbot-worker`` package.
``buildbot-slave`` package has been renamed to ``buildbot-worker``.
``buildbot-worker`` has backward incompatible changes and requires buildmaster >= 0.9.0b8.
``buildbot-slave`` will work with both 0.8.x and 0.9.x versions of buildmaster, so there is no need to upgrade currently deployed buildbot-slaves during switch from 0.8.x to 0.9.x.
``buildbot-slave`` from 0.8.x will work with both 0.8.x and 0.9.x versions of buildmaster, so there is no need to upgrade currently deployed buildbot-slaves during switch from 0.8.x to 0.9.x.
.. list-table:: Master/worker compatibility table
:header-rows: 1

6
master/docs/relnotes/index.rst
View File

@@ -68,10 +68,10 @@ Deprecations, Removals, and Non-Compatible Changes
Buildslave
----------
Fixes
~~~~~
Deprecations, Removals, and Non-Compatible Changes
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
* ``buildslave`` script now outputs messages to the terminal.
* The ``buildbot-slave`` package has finished being renamed to ``buildbot-worker``.
Worker

280
slave/COPYING
View File

@@ -1,280 +0,0 @@
GNU GENERAL PUBLIC LICENSE
Version 2, June 1991
Copyright (C) 1989, 1991 Free Software Foundation, Inc.,
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
Everyone is permitted to copy and distribute verbatim copies
of this license document, but changing it is not allowed.
Preamble
The licenses for most software are designed to take away your
freedom to share and change it. By contrast, the GNU General Public
License is intended to guarantee your freedom to share and change free
software--to make sure the software is free for all its users. This
General Public License applies to most of the Free Software
Foundation's software and to any other program whose authors commit to
using it. (Some other Free Software Foundation software is covered by
the GNU Lesser General Public License instead.) You can apply it to
your programs, too.
When we speak of free software, we are referring to freedom, not
price. Our General Public Licenses are designed to make sure that you
have the freedom to distribute copies of free software (and charge for
this service if you wish), that you receive source code or can get it
if you want it, that you can change the software or use pieces of it
in new free programs; and that you know you can do these things.
To protect your rights, we need to make restrictions that forbid
anyone to deny you these rights or to ask you to surrender the rights.
These restrictions translate to certain responsibilities for you if you
distribute copies of the software, or if you modify it.
For example, if you distribute copies of such a program, whether
gratis or for a fee, you must give the recipients all the rights that
you have. You must make sure that they, too, receive or can get the
source code. And you must show them these terms so they know their
rights.
We protect your rights with two steps: (1) copyright the software, and
(2) offer you this license which gives you legal permission to copy,
distribute and/or modify the software.
Also, for each author's protection and ours, we want to make certain
that everyone understands that there is no warranty for this free
software. If the software is modified by someone else and passed on, we
want its recipients to know that what they have is not the original, so
that any problems introduced by others will not reflect on the original
authors' reputations.
Finally, any free program is threatened constantly by software
patents. We wish to avoid the danger that redistributors of a free
program will individually obtain patent licenses, in effect making the
program proprietary. To prevent this, we have made it clear that any
patent must be licensed for everyone's free use or not licensed at all.
The precise terms and conditions for copying, distribution and
modification follow.
GNU GENERAL PUBLIC LICENSE
TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
0. This License applies to any program or other work which contains
a notice placed by the copyright holder saying it may be distributed
under the terms of this General Public License. The "Program", below,
refers to any such program or work, and a "work based on the Program"
means either the Program or any derivative work under copyright law:
that is to say, a work containing the Program or a portion of it,
either verbatim or with modifications and/or translated into another
language. (Hereinafter, translation is included without limitation in
the term "modification".) Each licensee is addressed as "you".
Activities other than copying, distribution and modification are not
covered by this License; they are outside its scope. The act of
running the Program is not restricted, and the output from the Program
is covered only if its contents constitute a work based on the
Program (independent of having been made by running the Program).
Whether that is true depends on what the Program does.
1. You may copy and distribute verbatim copies of the Program's
source code as you receive it, in any medium, provided that you
conspicuously and appropriately publish on each copy an appropriate
copyright notice and disclaimer of warranty; keep intact all the
notices that refer to this License and to the absence of any warranty;
and give any other recipients of the Program a copy of this License
along with the Program.
You may charge a fee for the physical act of transferring a copy, and
you may at your option offer warranty protection in exchange for a fee.
2. You may modify your copy or copies of the Program or any portion
of it, thus forming a work based on the Program, and copy and
distribute such modifications or work under the terms of Section 1
above, provided that you also meet all of these conditions:
a) You must cause the modified files to carry prominent notices
stating that you changed the files and the date of any change.
b) You must cause any work that you distribute or publish, that in
whole or in part contains or is derived from the Program or any
part thereof, to be licensed as a whole at no charge to all third
parties under the terms of this License.
c) If the modified program normally reads commands interactively
when run, you must cause it, when started running for such
interactive use in the most ordinary way, to print or display an
announcement including an appropriate copyright notice and a
notice that there is no warranty (or else, saying that you provide
a warranty) and that users may redistribute the program under
these conditions, and telling the user how to view a copy of this
License. (Exception: if the Program itself is interactive but
does not normally print such an announcement, your work based on
the Program is not required to print an announcement.)
These requirements apply to the modified work as a whole. If
identifiable sections of that work are not derived from the Program,
and can be reasonably considered independent and separate works in
themselves, then this License, and its terms, do not apply to those
sections when you distribute them as separate works. But when you
distribute the same sections as part of a whole which is a work based
on the Program, the distribution of the whole must be on the terms of
this License, whose permissions for other licensees extend to the
entire whole, and thus to each and every part regardless of who wrote it.
Thus, it is not the intent of this section to claim rights or contest
your rights to work written entirely by you; rather, the intent is to
exercise the right to control the distribution of derivative or
collective works based on the Program.
In addition, mere aggregation of another work not based on the Program
with the Program (or with a work based on the Program) on a volume of
a storage or distribution medium does not bring the other work under
the scope of this License.
3. You may copy and distribute the Program (or a work based on it,
under Section 2) in object code or executable form under the terms of
Sections 1 and 2 above provided that you also do one of the following:
a) Accompany it with the complete corresponding machine-readable
source code, which must be distributed under the terms of Sections
1 and 2 above on a medium customarily used for software interchange; or,
b) Accompany it with a written offer, valid for at least three
years, to give any third party, for a charge no more than your
cost of physically performing source distribution, a complete
machine-readable copy of the corresponding source code, to be
distributed under the terms of Sections 1 and 2 above on a medium
customarily used for software interchange; or,
c) Accompany it with the information you received as to the offer
to distribute corresponding source code. (This alternative is
allowed only for noncommercial distribution and only if you
received the program in object code or executable form with such
an offer, in accord with Subsection b above.)
The source code for a work means the preferred form of the work for
making modifications to it. For an executable work, complete source
code means all the source code for all modules it contains, plus any
associated interface definition files, plus the scripts used to
control compilation and installation of the executable. However, as a
special exception, the source code distributed need not include
anything that is normally distributed (in either source or binary
form) with the major components (compiler, kernel, and so on) of the
operating system on which the executable runs, unless that component
itself accompanies the executable.
If distribution of executable or object code is made by offering
access to copy from a designated place, then offering equivalent
access to copy the source code from the same place counts as
distribution of the source code, even though third parties are not
compelled to copy the source along with the object code.
4. You may not copy, modify, sublicense, or distribute the Program
except as expressly provided under this License. Any attempt
otherwise to copy, modify, sublicense or distribute the Program is
void, and will automatically terminate your rights under this License.
However, parties who have received copies, or rights, from you under
this License will not have their licenses terminated so long as such
parties remain in full compliance.
5. You are not required to accept this License, since you have not
signed it. However, nothing else grants you permission to modify or
distribute the Program or its derivative works. These actions are
prohibited by law if you do not accept this License. Therefore, by
modifying or distributing the Program (or any work based on the
Program), you indicate your acceptance of this License to do so, and
all its terms and conditions for copying, distributing or modifying
the Program or works based on it.
6. Each time you redistribute the Program (or any work based on the
Program), the recipient automatically receives a license from the
original licensor to copy, distribute or modify the Program subject to
these terms and conditions. You may not impose any further
restrictions on the recipients' exercise of the rights granted herein.
You are not responsible for enforcing compliance by third parties to
this License.
7. If, as a consequence of a court judgment or allegation of patent
infringement or for any other reason (not limited to patent issues),
conditions are imposed on you (whether by court order, agreement or
otherwise) that contradict the conditions of this License, they do not
excuse you from the conditions of this License. If you cannot
distribute so as to satisfy simultaneously your obligations under this
License and any other pertinent obligations, then as a consequence you
may not distribute the Program at all. For example, if a patent
license would not permit royalty-free redistribution of the Program by
all those who receive copies directly or indirectly through you, then
the only way you could satisfy both it and this License would be to
refrain entirely from distribution of the Program.
If any portion of this section is held invalid or unenforceable under
any particular circumstance, the balance of the section is intended to
apply and the section as a whole is intended to apply in other
circumstances.
It is not the purpose of this section to induce you to infringe any
patents or other property right claims or to contest validity of any
such claims; this section has the sole purpose of protecting the
integrity of the free software distribution system, which is
implemented by public license practices. Many people have made
generous contributions to the wide range of software distributed
through that system in reliance on consistent application of that
system; it is up to the author/donor to decide if he or she is willing
to distribute software through any other system and a licensee cannot
impose that choice.
This section is intended to make thoroughly clear what is believed to
be a consequence of the rest of this License.
8. If the distribution and/or use of the Program is restricted in
certain countries either by patents or by copyrighted interfaces, the
original copyright holder who places the Program under this License
may add an explicit geographical distribution limitation excluding
those countries, so that distribution is permitted only in or among
countries not thus excluded. In such case, this License incorporates
the limitation as if written in the body of this License.
9. The Free Software Foundation may publish revised and/or new versions
of the General Public License from time to time. Such new versions will
be similar in spirit to the present version, but may differ in detail to
address new problems or concerns.
Each version is given a distinguishing version number. If the Program
specifies a version number of this License which applies to it and "any
later version", you have the option of following the terms and conditions
either of that version or of any later version published by the Free
Software Foundation. If the Program does not specify a version number of
this License, you may choose any version ever published by the Free Software
Foundation.
10. If you wish to incorporate parts of the Program into other free
programs whose distribution conditions are different, write to the author
to ask for permission. For software which is copyrighted by the Free
Software Foundation, write to the Free Software Foundation; we sometimes
make exceptions for this. Our decision will be guided by the two goals
of preserving the free status of all derivatives of our free software and
of promoting the sharing and reuse of software generally.
NO WARRANTY
11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN
OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS
TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE
PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
REPAIR OR CORRECTION.
12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
POSSIBILITY OF SUCH DAMAGES.
END OF TERMS AND CONDITIONS

6
slave/MANIFEST.in
View File

@@ -1,6 +0,0 @@
include MANIFEST.in README NEWS COPYING UPGRADING
include bin/buildslave
include docs/buildslave.1
include contrib/windows/* contrib/os-x/* contrib/init-scripts/*
include contrib/zsh/* contrib/bash/*

6
slave/Makefile
View File

@@ -1,6 +0,0 @@
# developer utilities
pylint:
pylint --rcfile=../common/pylintrc buildslave
flake8:
flake8 --config=../common/flake8rc buildslave

37
slave/README
View File

@@ -1,37 +0,0 @@
Buildbot: build/test automation
http://buildbot.net
Brian Warner <warner-buildbot @ lothar . com>
Dustin J. Mitchell <dustin@v.igoro.us>
Buildbot is a continuous integration system designed to automate the
build/test cycle. By automatically rebuilding and testing the tree each time
something has changed, build problems are pinpointed quickly, before other
developers are inconvenienced by the failure. Features
* Buildbot is easy to set up, but very extensible and customizable. It
supports arbitrary build processes, and is not limited to common build
processes for particular languages (e.g., autotools or ant)
* Buildbot supports building and testing on a variety of platforms.
Developers, who do not have the facilities to test their changes everywhere
before committing, will know shortly afterwards whether they have broken the
build or not.
* Buildbot has minimal requirements for slaves: using virtualenv, only a
Python installation is required.
* Slaves can be run behind a NAT firewall and communicate with the master
* Buildbot has a variety of status-reporting tools to get information about
builds in front of developers in a timely manner.
Buildslave:
This package contains only the buildslave implementation. The `buildbot`
package contains the buildmaster as well as a complete set of documentation.
See http://buildbot.net for more information and for an online version of the
Buildbot documentation.

2
slave/UPGRADING
View File

@@ -1,2 +0,0 @@
For information on ugprading a buildslave, see the section "Upgrading an
Existing Buildslave" in the buildbot documentation.

4
slave/bin/buildslave
View File

@@ -1,4 +0,0 @@
#!/usr/bin/env python
from buildslave.scripts import runner
runner.run()

67
slave/buildslave/__init__.py
View File

@@ -1,67 +0,0 @@
# This file is part of Buildbot. Buildbot is free software: you can
# redistribute it and/or modify it under the terms of the GNU General Public
# License as published by the Free Software Foundation, version 2.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 51
# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
# Copyright Buildbot Team Members
#
# Keep in sync with master/buildbot/__init__.py
#
# We can't put this method in utility modules, because they import dependancy packages
#
from __future__ import with_statement
import os
import re
from subprocess import PIPE
from subprocess import Popen
from subprocess import STDOUT
def getVersion(init_file):
"""
Return BUILDBOT_VERSION environment variable, content of VERSION file, git
tag or 'latest'
"""
try:
return os.environ['BUILDBOT_VERSION']
except KeyError:
pass
try:
cwd = os.path.dirname(os.path.abspath(init_file))
fn = os.path.join(cwd, 'VERSION')
with open(fn) as f:
return f.read().strip()
except IOError:
pass
# accept version to be coded with 2 or 3 parts (X.Y or X.Y.Z),
# no matter the number of digits for X, Y and Z
VERSION_MATCH = re.compile(r'(\d+\.\d+(\.\d+)?(\w|-)*)')
try:
p = Popen(['git', 'describe', '--tags', '--always'], stdout=PIPE, stderr=STDOUT, cwd=cwd)
out = p.communicate()[0]
if (not p.returncode) and out:
v = VERSION_MATCH.search(out)
if v:
return v.group(1)
except OSError:
pass
return "latest"
version = getVersion(__file__)

390
slave/buildslave/base.py
View File

@@ -1,390 +0,0 @@
# This file is part of Buildbot. Buildbot is free software: you can
# redistribute it and/or modify it under the terms of the GNU General Public
# License as published by the Free Software Foundation, version 2.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 51
# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
# Copyright Buildbot Team Members
import multiprocessing
import os.path
import socket
import sys
from twisted.application import service
from twisted.internet import defer
from twisted.internet import reactor
from twisted.python import log
from twisted.spread import pb
import buildslave
from buildslave import monkeypatches
from buildslave.commands import base
from buildslave.commands import registry
class UnknownCommand(pb.Error):
pass
class SlaveBuilderBase(service.Service):
"""This is the local representation of a single Builder: it handles a
single kind of build (like an all-warnings build). It has a name and a
home directory. The rest of its behavior is determined by the master.
"""
stopCommandOnShutdown = True
# remote is a ref to the Builder object on the master side, and is set
# when they attach. We use it to detect when the connection to the master
# is severed.
remote = None
# .command points to a SlaveCommand instance, and is set while the step
# is running. We use it to implement the stopBuild method.
command = None
# .remoteStep is a ref to the master-side BuildStep object, and is set
# when the step is started
remoteStep = None
bf = None
def __init__(self, name):
# service.Service.__init__(self) # Service has no __init__ method
self.setName(name)
def __repr__(self):
return "<SlaveBuilder '%s' at %d>" % (self.name, id(self))
def setServiceParent(self, parent):
service.Service.setServiceParent(self, parent)
self.bot = self.parent
# note that self.parent will go away when the buildmaster's config
# file changes and this Builder is removed (possibly because it has
# been changed, so the Builder will be re-added again in a moment).
# This may occur during a build, while a step is running.
def setBuilddir(self, builddir):
assert self.parent
self.builddir = builddir
self.basedir = os.path.join(self.bot.basedir, self.builddir)
if not os.path.isdir(self.basedir):
os.makedirs(self.basedir)
def stopService(self):
service.Service.stopService(self)
if self.stopCommandOnShutdown:
self.stopCommand()
def activity(self):
bot = self.parent
if bot:
bslave = bot.parent
if bslave and self.bf:
bf = bslave.bf
bf.activity()
def remote_setMaster(self, remote):
self.remote = remote
self.remote.notifyOnDisconnect(self.lostRemote)
def remote_print(self, message):
log.msg("SlaveBuilder.remote_print(%s): message from master: %s" %
(self.name, message))
def lostRemote(self, remote):
log.msg("lost remote")
self.remote = None
def lostRemoteStep(self, remotestep):
log.msg("lost remote step")
self.remoteStep = None
if self.stopCommandOnShutdown:
self.stopCommand()
# the following are Commands that can be invoked by the master-side
# Builder
def remote_startBuild(self):
"""This is invoked before the first step of any new build is run. It
doesn't do much, but masters call it so it's still here."""
pass
def remote_startCommand(self, stepref, stepId, command, args):
"""
This gets invoked by L{buildbot.process.step.RemoteCommand.start}, as
part of various master-side BuildSteps, to start various commands
that actually do the build. I return nothing. Eventually I will call
.commandComplete() to notify the master-side RemoteCommand that I'm
done.
"""
self.activity()
if self.command:
log.msg("leftover command, dropping it")
self.stopCommand()
try:
factory = registry.getFactory(command)
except KeyError:
raise UnknownCommand("unrecognized SlaveCommand '%s'" % command)
self.command = factory(self, stepId, args)
log.msg(" startCommand:%s [id %s]" % (command, stepId))
self.remoteStep = stepref
self.remoteStep.notifyOnDisconnect(self.lostRemoteStep)
d = self.command.doStart()
d.addCallback(lambda res: None)
d.addBoth(self.commandComplete)
return None
def remote_interruptCommand(self, stepId, why):
"""Halt the current step."""
log.msg("asked to interrupt current command: %s" % why)
self.activity()
if not self.command:
# TODO: just log it, a race could result in their interrupting a
# command that wasn't actually running
log.msg(" .. but none was running")
return
self.command.doInterrupt()
def stopCommand(self):
"""Make any currently-running command die, with no further status
output. This is used when the buildslave is shutting down or the
connection to the master has been lost. Interrupt the command,
silence it, and then forget about it."""
if not self.command:
return
log.msg("stopCommand: halting current command %s" % self.command)
self.command.doInterrupt() # shut up! and die!
self.command = None # forget you!
# sendUpdate is invoked by the Commands we spawn
def sendUpdate(self, data):
"""This sends the status update to the master-side
L{buildbot.process.step.RemoteCommand} object, giving it a sequence
number in the process. It adds the update to a queue, and asks the
master to acknowledge the update so it can be removed from that
queue."""
if not self.running:
# .running comes from service.Service, and says whether the
# service is running or not. If we aren't running, don't send any
# status messages.
return
# the update[1]=0 comes from the leftover 'updateNum', which the
# master still expects to receive. Provide it to avoid significant
# interoperability issues between new slaves and old masters.
if self.remoteStep:
update = [data, 0]
updates = [update]
d = self.remoteStep.callRemote("update", updates)
d.addCallback(self.ackUpdate)
d.addErrback(self._ackFailed, "SlaveBuilder.sendUpdate")
def ackUpdate(self, acknum):
self.activity() # update the "last activity" timer
def ackComplete(self, dummy):
self.activity() # update the "last activity" timer
def _ackFailed(self, why, where):
log.msg("SlaveBuilder._ackFailed:", where)
log.err(why) # we don't really care
# this is fired by the Deferred attached to each Command
def commandComplete(self, failure):
if failure:
log.msg("SlaveBuilder.commandFailed", self.command)
log.err(failure)
# failure, if present, is a failure.Failure. To send it across
# the wire, we must turn it into a pb.CopyableFailure.
failure = pb.CopyableFailure(failure)
failure.unsafeTracebacks = True
else:
# failure is None
log.msg("SlaveBuilder.commandComplete", self.command)
self.command = None
if not self.running:
log.msg(" but we weren't running, quitting silently")
return
if self.remoteStep:
self.remoteStep.dontNotifyOnDisconnect(self.lostRemoteStep)
d = self.remoteStep.callRemote("complete", failure)
d.addCallback(self.ackComplete)
d.addErrback(self._ackFailed, "sendComplete")
self.remoteStep = None
def remote_shutdown(self):
log.msg("slave shutting down on command from master")
log.msg(
"NOTE: master is using deprecated slavebuilder.shutdown method")
reactor.stop()
class BotBase(service.MultiService):
"""I represent the slave-side bot."""
usePTY = None
name = "bot"
SlaveBuilder = SlaveBuilderBase
def __init__(self, basedir, usePTY, unicode_encoding=None):
service.MultiService.__init__(self)
self.basedir = basedir
self.numcpus = None
self.usePTY = usePTY
self.unicode_encoding = unicode_encoding or sys.getfilesystemencoding(
) or 'ascii'
self.builders = {}
def startService(self):
assert os.path.isdir(self.basedir)
service.MultiService.startService(self)
def remote_getCommands(self):
commands = dict([
(n, base.command_version)
for n in registry.getAllCommandNames()
])
return commands
@defer.inlineCallbacks
def remote_setBuilderList(self, wanted):
retval = {}
wanted_names = set([name for (name, builddir) in wanted])
wanted_dirs = set([builddir for (name, builddir) in wanted])
wanted_dirs.add('info')
for (name, builddir) in wanted:
b = self.builders.get(name, None)
if b:
if b.builddir != builddir:
log.msg("changing builddir for builder %s from %s to %s"
% (name, b.builddir, builddir))
b.setBuilddir(builddir)
else:
b = self.SlaveBuilder(name)
b.usePTY = self.usePTY
b.unicode_encoding = self.unicode_encoding
b.setServiceParent(self)
b.setBuilddir(builddir)
self.builders[name] = b
retval[name] = b
# disown any builders no longer desired
to_remove = list(set(self.builders.keys()) - wanted_names)
if to_remove:
yield defer.gatherResults([
defer.maybeDeferred(self.builders[name].disownServiceParent)
for name in to_remove])
# and *then* remove them from the builder list
for name in to_remove:
del self.builders[name]
# finally warn about any leftover dirs
for dir in os.listdir(self.basedir):
if os.path.isdir(os.path.join(self.basedir, dir)):
if dir not in wanted_dirs:
log.msg("I have a leftover directory '%s' that is not "
"being used by the buildmaster: you can delete "
"it now" % dir)
defer.returnValue(retval)
def remote_print(self, message):
log.msg("message from master:", message)
def remote_getSlaveInfo(self):
"""This command retrieves data from the files in SLAVEDIR/info/* and
sends the contents to the buildmaster. These are used to describe
the slave and its configuration, and should be created and
maintained by the slave administrator. They will be retrieved each
time the master-slave connection is established.
"""
files = {}
basedir = os.path.join(self.basedir, "info")
if os.path.isdir(basedir):
for f in os.listdir(basedir):
filename = os.path.join(basedir, f)
if os.path.isfile(filename):
files[f] = open(filename, "r").read()
if not self.numcpus:
self.numcpus = multiprocessing.cpu_count()
files['environ'] = os.environ.copy()
files['system'] = os.name
files['basedir'] = self.basedir
files['numcpus'] = self.numcpus
files['version'] = self.remote_getVersion()
files['slave_commands'] = self.remote_getCommands()
return files
def remote_getVersion(self):
"""Send our version back to the Master"""
return buildslave.version
def remote_shutdown(self):
log.msg("slave shutting down on command from master")
# there's no good way to learn that the PB response has been delivered,
# so we'll just wait a bit, in hopes the master hears back. Masters are
# resilinet to slaves dropping their connections, so there is no harm
# if this timeout is too short.
reactor.callLater(0.2, reactor.stop)
class BuildSlaveBase(service.MultiService):
Bot = BotBase
def __init__(self, name, basedir,
usePTY, umask=None,
unicode_encoding=None):
service.MultiService.__init__(self)
self.name = name
bot = self.Bot(basedir, usePTY, unicode_encoding=unicode_encoding)
bot.setServiceParent(self)
self.bot = bot
self.umask = umask
self.basedir = basedir
def startService(self):
# first, apply all monkeypatches
monkeypatches.patch_all()
log.msg("Starting BuildSlave -- version: %s" % buildslave.version)
if self.umask is not None:
os.umask(self.umask)
self.recordHostname(self.basedir)
service.MultiService.startService(self)
def recordHostname(self, basedir):
"Record my hostname in twistd.hostname, for user convenience"
log.msg("recording hostname in twistd.hostname")
filename = os.path.join(basedir, "twistd.hostname")
try:
hostname = os.uname()[1] # only on unix
except AttributeError:
# this tends to fail on non-connected hosts, e.g., laptops
# on planes
hostname = socket.getfqdn()
try:
open(filename, "w").write("%s\n" % hostname)
except Exception:
log.msg("failed - ignoring")

19
slave/buildslave/bot.py
View File

@@ -1,19 +0,0 @@
# This file is part of Buildbot. Buildbot is free software: you can
# redistribute it and/or modify it under the terms of the GNU General Public
# License as published by the Free Software Foundation, version 2.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 51
# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
# Copyright Buildbot Team Members
from buildslave.null import LocalBuildSlave
from buildslave.pb import BuildSlave
__all__ = ['BuildSlave', 'LocalBuildSlave']

0
slave/buildslave/commands/__init__.py
View File

647
slave/buildslave/commands/base.py
View File

@@ -1,647 +0,0 @@
# This file is part of Buildbot. Buildbot is free software: you can
# redistribute it and/or modify it under the terms of the GNU General Public
# License as published by the Free Software Foundation, version 2.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 51
# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
# Copyright Buildbot Team Members
import os
import shutil
import sys
from base64 import b64encode
from twisted.internet import defer
from twisted.internet import reactor
from twisted.internet import threads
from twisted.python import failure
from twisted.python import log
from twisted.python import runtime
from zope.interface import implements
from buildslave import runprocess
from buildslave import util
from buildslave.commands import utils
from buildslave.exceptions import AbandonChain
from buildslave.interfaces import ISlaveCommand
# this used to be a CVS $-style "Revision" auto-updated keyword, but since I
# moved to Darcs as the primary repository, this is updated manually each
# time this file is changed. The last cvs_ver that was here was 1.51 .
command_version = "2.16"
# version history:
# >=1.17: commands are interruptable
# >=1.28: Arch understands 'revision', added Bazaar
# >=1.33: Source classes understand 'retry'
# >=1.39: Source classes correctly handle changes in branch (except Git)
# Darcs accepts 'revision' (now all do but Git) (well, and P4Sync)
# Arch/Baz should accept 'build-config'
# >=1.51: (release 0.7.3)
# >= 2.1: SlaveShellCommand now accepts 'initial_stdin', 'keep_stdin_open',
# and 'logfiles'. It now sends 'log' messages in addition to
# stdout/stdin/header/rc. It acquired writeStdin/closeStdin methods,
# but these are not remotely callable yet.
# (not externally visible: ShellCommandPP has writeStdin/closeStdin.
# ShellCommand accepts new arguments (logfiles=, initialStdin=,
# keepStdinOpen=) and no longer accepts stdin=)
# (release 0.7.4)
# >= 2.2: added monotone, uploadFile, and downloadFile (release 0.7.5)
# >= 2.3: added bzr (release 0.7.6)
# >= 2.4: Git understands 'revision' and branches
# >= 2.5: workaround added for remote 'hg clone --rev REV' when hg<0.9.2
# >= 2.6: added uploadDirectory
# >= 2.7: added usePTY option to SlaveShellCommand
# >= 2.8: added username and password args to SVN class
# >= 2.9: add depth arg to SVN class
# >= 2.10: CVS can handle 'extra_options' and 'export_options'
# >= 2.11: Arch, Bazaar, and Monotone removed
# >= 2.12: SlaveShellCommand no longer accepts 'keep_stdin_open'
# >= 2.13: SlaveFileUploadCommand supports option 'keepstamp'
# >= 2.14: RemoveDirectory can delete multiple directories
# >= 2.15: 'interruptSignal' option is added to SlaveShellCommand
# >= 2.16: 'sigtermTime' option is added to SlaveShellCommand
# >= 2.16: runprocess supports obfuscation via tuples (#1748)
# >= 2.16: listdir command added to read a directory
class Command(object):
implements(ISlaveCommand)
"""This class defines one command that can be invoked by the build master.
The command is executed on the slave side, and always sends back a
completion message when it finishes. It may also send intermediate status
as it runs (by calling builder.sendStatus). Some commands can be
interrupted (either by the build master or a local timeout), in which
case the step is expected to complete normally with a status message that
indicates an error occurred.
These commands are used by BuildSteps on the master side. Each kind of
BuildStep uses a single Command. The slave must implement all the
Commands required by the set of BuildSteps used for any given build:
this is checked at startup time.
All Commands are constructed with the same signature:
c = CommandClass(builder, stepid, args)
where 'builder' is the parent SlaveBuilder object, and 'args' is a
dict that is interpreted per-command.
The setup(args) method is available for setup, and is run from __init__.
Mandatory args can be declared by listing them in the requiredArgs property.
They will be checked before calling the setup(args) method.
The Command is started with start(). This method must be implemented in a
subclass, and it should return a Deferred. When your step is done, you
should fire the Deferred (the results are not used). If the command is
interrupted, it should fire the Deferred anyway.
While the command runs. it may send status messages back to the
buildmaster by calling self.sendStatus(statusdict). The statusdict is
interpreted by the master-side BuildStep however it likes.
A separate completion message is sent when the deferred fires, which
indicates that the Command has finished, but does not carry any status
data. If the Command needs to return an exit code of some sort, that
should be sent as a regular status message before the deferred is fired .
Once builder.commandComplete has been run, no more status messages may be
sent.
If interrupt() is called, the Command should attempt to shut down as
quickly as possible. Child processes should be killed, new ones should
not be started. The Command should send some kind of error status update,
then complete as usual by firing the Deferred.
.interrupted should be set by interrupt(), and can be tested to avoid
sending multiple error status messages.
If .running is False, the bot is shutting down (or has otherwise lost the
connection to the master), and should not send any status messages. This
is checked in Command.sendStatus .
"""
# builder methods:
# sendStatus(dict) (zero or more)
# commandComplete() or commandInterrupted() (one, at end)
requiredArgs = []
debug = False
interrupted = False
# set by Builder, cleared on shutdown or when the Deferred fires
running = False
_reactor = reactor
def __init__(self, builder, stepId, args):
self.builder = builder
self.stepId = stepId # just for logging
self.args = args
self.startTime = None
missingArgs = [arg for arg in self.requiredArgs if arg not in args]
if missingArgs:
raise ValueError("%s is missing args: %s" %
(self.__class__.__name__, ", ".join(missingArgs)))
self.setup(args)
def setup(self, args):
"""Override this in a subclass to extract items from the args dict."""
pass
def doStart(self):
self.running = True
self.startTime = util.now(self._reactor)
d = defer.maybeDeferred(self.start)
def commandComplete(res):
self.sendStatus(
{"elapsed": util.now(self._reactor) - self.startTime})
self.running = False
return res
d.addBoth(commandComplete)
return d
def start(self):
"""Start the command. This method should return a Deferred that will
fire when the command has completed. The Deferred's argument will be
ignored.
This method should be overridden by subclasses."""
raise NotImplementedError("You must implement this in a subclass")
def sendStatus(self, status):
"""Send a status update to the master."""
if self.debug:
log.msg("sendStatus", status)
if not self.running:
log.msg("would sendStatus but not .running")
return
self.builder.sendUpdate(status)
def doInterrupt(self):
self.running = False
self.interrupt()
def interrupt(self):
"""Override this in a subclass to allow commands to be interrupted.
May be called multiple times, test and set self.interrupted=True if
this matters."""
pass
# utility methods, mostly used by SlaveShellCommand and the like
def _abandonOnFailure(self, rc):
if not isinstance(rc, int):
log.msg("weird, _abandonOnFailure was given rc=%s (%s)" %
(rc, type(rc)))
assert isinstance(rc, int)
if rc != 0:
raise AbandonChain(rc)
return rc
def _sendRC(self, res):
self.sendStatus({'rc': 0})
def _checkAbandoned(self, why):
log.msg("_checkAbandoned", why)
why.trap(AbandonChain)
log.msg(" abandoning chain", why.value)
self.sendStatus({'rc': why.value.args[0]})
return None
class SourceBaseCommand(Command):
"""Abstract base class for Version Control System operations (checkout
and update). This class extracts the following arguments from the
dictionary received from the master:
- ['workdir']: (required) the subdirectory where the buildable sources
should be placed
- ['mode']: one of update/copy/clobber/export, defaults to 'update'
- ['revision']: (required) If not None, this is an int or string which indicates
which sources (along a time-like axis) should be used.
It is the thing you provide as the CVS -r or -D
argument.
- ['patch']: If not None, this is a tuple of (striplevel, patch)
which contains a patch that should be applied after the
checkout has occurred. Once applied, the tree is no
longer eligible for use with mode='update', and it only
makes sense to use this in conjunction with a
['revision'] argument. striplevel is an int, and patch
is a string in standard unified diff format. The patch
will be applied with 'patch -p%d <PATCH', with
STRIPLEVEL substituted as %d. The command will fail if
the patch process fails (rejected hunks).
- ['timeout']: seconds of silence tolerated before we kill off the
command
- ['maxTime']: seconds before we kill off the command
- ['retry']: If not None, this is a tuple of (delay, repeats)
which means that any failed VC updates should be
reattempted, up to REPEATS times, after a delay of
DELAY seconds. This is intended to deal with slaves
that experience transient network failures.
"""
sourcedata = ""
def setup(self, args):
# if we need to parse the output, use this environment. Otherwise
# command output will be in whatever the buildslave's native language
# has been set to.
self.env = os.environ.copy()
self.env['LC_MESSAGES'] = "C"
self.workdir = args['workdir']
self.mode = args.get('mode', "update")
self.revision = args.get('revision')
self.patch = args.get('patch')
self.timeout = args.get('timeout', 120)
self.maxTime = args.get('maxTime', None)
self.retry = args.get('retry')
self.logEnviron = args.get('logEnviron', True)
self._commandPaths = {}
# VC-specific subclasses should override this to extract more args.
# Make sure to upcall!
def getCommand(self, name):
"""Wrapper around utils.getCommand that will output a resonable
error message and raise AbandonChain if the command cannot be
found"""
if name not in self._commandPaths:
try:
self._commandPaths[name] = utils.getCommand(name)
except RuntimeError:
self.sendStatus({'stderr': "could not find '%s'\n" % name})
self.sendStatus(
{'stderr': "PATH is '%s'\n" % os.environ.get('PATH', '')})
raise AbandonChain(-1)
return self._commandPaths[name]
def start(self):
self.sendStatus({'header': "starting " + self.header + "\n"})
self.command = None
# self.srcdir is where the VC system should put the sources
if self.mode == "copy":
self.srcdir = "source" # hardwired directory name, sorry
else:
self.srcdir = self.workdir
self.sourcedatafile = os.path.join(self.builder.basedir,
".buildbot-sourcedata-" + b64encode(self.srcdir))
# upgrade older versions to the new sourcedata location
old_sd_path = os.path.join(
self.builder.basedir, self.srcdir, ".buildbot-sourcedata")
if os.path.exists(old_sd_path) and not os.path.exists(self.sourcedatafile):
os.rename(old_sd_path, self.sourcedatafile)
# also upgrade versions that didn't include the encoded version of the
# source directory
old_sd_path = os.path.join(
self.builder.basedir, ".buildbot-sourcedata")
if os.path.exists(old_sd_path) and not os.path.exists(self.sourcedatafile):
os.rename(old_sd_path, self.sourcedatafile)
d = defer.succeed(None)
self.maybeClobber(d)
if not (self.sourcedirIsUpdateable() and self.sourcedataMatches()):
# the directory cannot be updated, so we have to clobber it.
# Perhaps the master just changed modes from 'export' to
# 'update'.
d.addCallback(self.doClobber, self.srcdir)
d.addCallback(self.doVC)
if self.mode == "copy":
d.addCallback(self.doCopy)
if self.patch:
d.addCallback(self.doPatch)
d.addCallbacks(self._sendRC, self._checkAbandoned)
return d
def maybeClobber(self, d):
# do we need to clobber anything?
if self.mode in ("copy", "clobber", "export"):
d.addCallback(self.doClobber, self.workdir)
def interrupt(self):
self.interrupted = True
if self.command:
self.command.kill("command interrupted")
def doVC(self, res):
if self.interrupted:
raise AbandonChain(1)
if self.sourcedirIsUpdateable() and self.sourcedataMatches():
d = self.doVCUpdate()
d.addBoth(self.maybeDoVCFallback)
else:
d = self.doVCFull()
d.addBoth(self.maybeDoVCRetry)
d.addCallback(self._abandonOnFailure)
d.addCallback(self._handleGotRevision)
d.addCallback(self.writeSourcedata)
return d
def sourcedataMatches(self):
try:
olddata = self.readSourcedata()
if olddata != self.sourcedata:
return False
except IOError:
return False
return True
def sourcedirIsPatched(self):
return os.path.exists(os.path.join(self.builder.basedir,
self.workdir,
".buildbot-patched"))
def _handleGotRevision(self, res):
d = defer.maybeDeferred(self.parseGotRevision)
d.addCallback(lambda got_revision:
self.sendStatus({'got_revision': got_revision}))
return d
def parseGotRevision(self):
"""Override this in a subclass. It should return a string that
represents which revision was actually checked out, or a Deferred
that will fire with such a string. If, in a future build, you were to
pass this 'got_revision' string in as the 'revision' component of a
SourceStamp, you should wind up with the same source code as this
checkout just obtained.
It is probably most useful to scan self.command.stdout for a string
of some sort. Be sure to set keepStdout=True on the VC command that
you run, so that you'll have something available to look at.
If this information is unavailable, just return None."""
return None
def readSourcedata(self):
"""
Read the sourcedata file and return its contents
@returns: source data
@raises: IOError if the file does not exist
"""
return open(self.sourcedatafile, "r").read()
def writeSourcedata(self, res):
open(self.sourcedatafile, "w").write(self.sourcedata)
return res
def sourcedirIsUpdateable(self):
"""Returns True if the tree can be updated."""
raise NotImplementedError("this must be implemented in a subclass")
def doVCUpdate(self):
"""Returns a deferred with the steps to update a checkout."""
raise NotImplementedError("this must be implemented in a subclass")
def doVCFull(self):
"""Returns a deferred with the steps to do a fresh checkout."""
raise NotImplementedError("this must be implemented in a subclass")
def maybeDoVCFallback(self, rc):
if isinstance(rc, int) and rc == 0:
return rc
if self.interrupted:
raise AbandonChain(1)
# allow AssertionErrors to fall through, for benefit of the tests; for
# all other errors, carry on to try the fallback
if isinstance(rc, failure.Failure) and rc.check(AssertionError):
return rc
# Let VCS subclasses have an opportunity to handle
# unrecoverable errors without having to clobber the repo
self.maybeNotDoVCFallback(rc)
msg = "update failed, clobbering and trying again"
self.sendStatus({'header': msg + "\n"})
log.msg(msg)
d = self.doClobber(None, self.srcdir)
d.addCallback(self.doVCFallback2)
return d
def doVCFallback2(self, res):
msg = "now retrying VC operation"
self.sendStatus({'header': msg + "\n"})
log.msg(msg)
d = self.doVCFull()
d.addBoth(self.maybeDoVCRetry)
d.addCallback(self._abandonOnFailure)
return d
def maybeNotDoVCFallback(self, rc):
"""Override this in a subclass if you want to detect unrecoverable
checkout errors where clobbering the repo wouldn't help, and stop
the current VC chain before it clobbers the repo for future builds.
Use 'raise AbandonChain' to pass up a halt if you do detect such."""
pass
def maybeDoVCRetry(self, res):
"""We get here somewhere after a VC chain has finished. res could
be::
- 0: the operation was successful
- nonzero: the operation failed. retry if possible
- AbandonChain: the operation failed, someone else noticed. retry.
- Failure: some other exception, re-raise
"""
if isinstance(res, failure.Failure):
if self.interrupted:
return res # don't re-try interrupted builds
res.trap(AbandonChain)
else:
if isinstance(res, int) and res == 0:
return res
if self.interrupted:
raise AbandonChain(1)
# if we get here, we should retry, if possible
if self.retry:
delay, repeats = self.retry
if repeats >= 0:
self.retry = (delay, repeats - 1)
msg = ("update failed, trying %d more times after %d seconds"
% (repeats, delay))
self.sendStatus({'header': msg + "\n"})
log.msg(msg)
d = defer.Deferred()
# we are going to do a full checkout, so a clobber is
# required first
self.doClobber(d, self.workdir)
if self.srcdir:
self.doClobber(d, self.srcdir)
d.addCallback(lambda res: self.doVCFull())
d.addBoth(self.maybeDoVCRetry)
self._reactor.callLater(delay, d.callback, None)
return d
return res
def doClobber(self, dummy, dirname, chmodDone=False):
d = os.path.join(self.builder.basedir, dirname)
if runtime.platformType != "posix":
d = threads.deferToThread(utils.rmdirRecursive, d)
def cb(_):
return 0 # rc=0
def eb(f):
self.sendStatus(
{'header': 'exception from rmdirRecursive\n' + f.getTraceback()})
return -1 # rc=-1
d.addCallbacks(cb, eb)
return d
command = ["rm", "-rf", d]
c = runprocess.RunProcess(self.builder, command, self.builder.basedir,
sendRC=0, timeout=self.timeout, maxTime=self.maxTime,
logEnviron=self.logEnviron, usePTY=False)
self.command = c
# sendRC=0 means the rm command will send stdout/stderr to the
# master, but not the rc=0 when it finishes. That job is left to
# _sendRC
d = c.start()
# The rm -rf may fail if there is a left-over subdir with chmod 000
# permissions. So if we get a failure, we attempt to chmod suitable
# permissions and re-try the rm -rf.
if chmodDone:
d.addCallback(self._abandonOnFailure)
else:
d.addCallback(lambda rc: self.doClobberTryChmodIfFail(rc, dirname))
return d
def doClobberTryChmodIfFail(self, rc, dirname):
assert isinstance(rc, int)
if rc == 0:
return defer.succeed(0)
# Attempt a recursive chmod and re-try the rm -rf after.
command = ["chmod", "-Rf", "u+rwx",
os.path.join(self.builder.basedir, dirname)]
if sys.platform.startswith('freebsd'):
# Work around a broken 'chmod -R' on FreeBSD (it tries to recurse into a
# directory for which it doesn't have permission, before changing that
# permission) by running 'find' instead
command = ["find", os.path.join(self.builder.basedir, dirname),
'-exec', 'chmod', 'u+rwx', '{}', ';']
c = runprocess.RunProcess(self.builder, command, self.builder.basedir,
sendRC=0, timeout=self.timeout, maxTime=self.maxTime,
logEnviron=self.logEnviron, usePTY=False)
self.command = c
d = c.start()
d.addCallback(self._abandonOnFailure)
d.addCallback(lambda dummy: self.doClobber(dummy, dirname, True))
return d
def doCopy(self, res):
# now copy tree to workdir
fromdir = os.path.join(self.builder.basedir, self.srcdir)
todir = os.path.join(self.builder.basedir, self.workdir)
if runtime.platformType != "posix":
d = threads.deferToThread(shutil.copytree, fromdir, todir)
def cb(_):
return 0 # rc=0
def eb(f):
self.sendStatus(
{'header': 'exception from copytree\n' + f.getTraceback()})
return -1 # rc=-1
d.addCallbacks(cb, eb)
return d
if not os.path.exists(os.path.dirname(todir)):
os.makedirs(os.path.dirname(todir))
if os.path.exists(todir):
# I don't think this happens, but just in case..
log.msg(
"cp target '%s' already exists -- cp will not do what you think!" % todir)
command = ['cp', '-R', '-P', '-p', fromdir, todir]
c = runprocess.RunProcess(self.builder, command, self.builder.basedir,
sendRC=False, timeout=self.timeout, maxTime=self.maxTime,
logEnviron=self.logEnviron, usePTY=False)
self.command = c
d = c.start()
d.addCallback(self._abandonOnFailure)
return d
def doPatch(self, res):
patchlevel = self.patch[0]
diff = self.patch[1]
root = None
if len(self.patch) >= 3:
root = self.patch[2]
command = [
utils.getCommand("patch"),
'-p%d' % patchlevel,
'--remove-empty-files',
'--force',
'--forward',
'-i', '.buildbot-diff',
]
dir = os.path.join(self.builder.basedir, self.workdir)
# Mark the directory so we don't try to update it later, or at least try
# to revert first.
open(os.path.join(dir, ".buildbot-patched"), "w").write("patched\n")
# write the diff to a file, for reading later
open(os.path.join(dir, ".buildbot-diff"), "w").write(diff)
# Update 'dir' with the 'root' option. Make sure it is a subdirectory
# of dir.
if (root and
os.path.abspath(os.path.join(dir, root)
).startswith(os.path.abspath(dir))):
dir = os.path.join(dir, root)
# now apply the patch
c = runprocess.RunProcess(self.builder, command, dir,
sendRC=False, timeout=self.timeout,
maxTime=self.maxTime, logEnviron=self.logEnviron,
usePTY=False)
self.command = c
d = c.start()
# clean up the temp file
def cleanup(x):
try:
os.unlink(os.path.join(dir, ".buildbot-diff"))
except OSError:
pass
return x
d.addBoth(cleanup)
d.addCallback(self._abandonOnFailure)
return d
def setFileContents(self, filename, contents):
"""Put the given C{contents} in C{filename}; this is a bit more
succinct than opening, writing, and closing, and has the advantage of
being patchable in tests. Note that the enclosing directory is
not automatically created, nor is this an "atomic" overwrite."""
f = open(filename, 'w')
f.write(contents)
f.close()

260
slave/buildslave/commands/fs.py
View File

@@ -1,260 +0,0 @@
# This file is part of Buildbot. Buildbot is free software: you can
# redistribute it and/or modify it under the terms of the GNU General Public
# License as published by the Free Software Foundation, version 2.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 51
# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
# Copyright Buildbot Team Members
import glob
import os
import shutil
import sys
from twisted.internet import defer
from twisted.internet import threads
from twisted.python import log
from twisted.python import runtime
from buildslave import runprocess
from buildslave.commands import base
from buildslave.commands import utils
class MakeDirectory(base.Command):
header = "mkdir"
# args['dir'] is relative to Builder directory, and is required.
requiredArgs = ['dir']
def start(self):
dirname = os.path.join(self.builder.basedir, self.args['dir'])
try:
if not os.path.isdir(dirname):
os.makedirs(dirname)
self.sendStatus({'rc': 0})
except OSError as e:
log.msg("MakeDirectory %s failed" % dirname, e)
self.sendStatus(
{'header': '%s: %s: %s' % (self.header, e.strerror, dirname)})
self.sendStatus({'rc': e.errno})
class RemoveDirectory(base.Command):
header = "rmdir"
# args['dir'] is relative to Builder directory, and is required.
requiredArgs = ['dir']
def setup(self, args):
self.logEnviron = args.get('logEnviron', True)
@defer.inlineCallbacks
def start(self):
args = self.args
dirnames = args['dir']
self.timeout = args.get('timeout', 120)
self.maxTime = args.get('maxTime', None)
self.rc = 0
if isinstance(dirnames, list):
assert len(dirnames) != 0
for dirname in dirnames:
res = yield self.removeSingleDir(dirname)
# Even if single removal of single file/dir consider it as
# failure of whole command, but continue removing other files
# Send 'rc' to master to handle failure cases
if res != 0:
self.rc = res
else:
self.rc = yield self.removeSingleDir(dirnames)
self.sendStatus({'rc': self.rc})
def removeSingleDir(self, dirname):
self.dir = os.path.join(self.builder.basedir, dirname)
if runtime.platformType != "posix":
d = threads.deferToThread(utils.rmdirRecursive, self.dir)
def cb(_):
return 0 # rc=0
def eb(f):
self.sendStatus(
{'header': 'exception from rmdirRecursive\n' + f.getTraceback()})
return -1 # rc=-1
d.addCallbacks(cb, eb)
else:
d = self._clobber(None)
return d
def _clobber(self, dummy, chmodDone=False):
command = ["rm", "-rf", self.dir]
c = runprocess.RunProcess(self.builder, command, self.builder.basedir,
sendRC=0, timeout=self.timeout, maxTime=self.maxTime,
logEnviron=self.logEnviron, usePTY=False)
self.command = c
# sendRC=0 means the rm command will send stdout/stderr to the
# master, but not the rc=0 when it finishes. That job is left to
# _sendRC
d = c.start()
# The rm -rf may fail if there is a left-over subdir with chmod 000
# permissions. So if we get a failure, we attempt to chmod suitable
# permissions and re-try the rm -rf.
if not chmodDone:
d.addCallback(self._tryChmod)
return d
def _tryChmod(self, rc):
assert isinstance(rc, int)
if rc == 0:
return defer.succeed(0)
# Attempt a recursive chmod and re-try the rm -rf after.
command = ["chmod", "-Rf", "u+rwx",
os.path.join(self.builder.basedir, self.dir)]
if sys.platform.startswith('freebsd'):
# Work around a broken 'chmod -R' on FreeBSD (it tries to recurse into a
# directory for which it doesn't have permission, before changing that
# permission) by running 'find' instead
command = ["find", os.path.join(self.builder.basedir, self.dir),
'-exec', 'chmod', 'u+rwx', '{}', ';']
c = runprocess.RunProcess(self.builder, command, self.builder.basedir,
sendRC=0, timeout=self.timeout, maxTime=self.maxTime,
logEnviron=self.logEnviron, usePTY=False)
self.command = c
d = c.start()
d.addCallback(lambda dummy: self._clobber(dummy, True))
return d
class CopyDirectory(base.Command):
header = "cpdir"
# args['todir'] and args['fromdir'] are relative to Builder directory, and
# are required.
requiredArgs = ['todir', 'fromdir']
def setup(self, args):
self.logEnviron = args.get('logEnviron', True)
def start(self):
args = self.args
fromdir = os.path.join(self.builder.basedir, self.args['fromdir'])
todir = os.path.join(self.builder.basedir, self.args['todir'])
self.timeout = args.get('timeout', 120)
self.maxTime = args.get('maxTime', None)
if runtime.platformType != "posix":
d = threads.deferToThread(shutil.copytree, fromdir, todir)
def cb(_):
return 0 # rc=0
def eb(f):
self.sendStatus(
{'header': 'exception from copytree\n' + f.getTraceback()})
return -1 # rc=-1
d.addCallbacks(cb, eb)
@d.addCallback
def send_rc(rc):
self.sendStatus({'rc': rc})
else:
if not os.path.exists(os.path.dirname(todir)):
os.makedirs(os.path.dirname(todir))
if os.path.exists(todir):
# I don't think this happens, but just in case..
log.msg(
"cp target '%s' already exists -- cp will not do what you think!" % todir)
command = ['cp', '-R', '-P', '-p', '-v', fromdir, todir]
c = runprocess.RunProcess(self.builder, command, self.builder.basedir,
sendRC=False, timeout=self.timeout, maxTime=self.maxTime,
logEnviron=self.logEnviron, usePTY=False)
self.command = c
d = c.start()
d.addCallback(self._abandonOnFailure)
d.addCallbacks(self._sendRC, self._checkAbandoned)
return d
class StatFile(base.Command):
header = "stat"
# args['file'] is relative to Builder directory, and is required.
requireArgs = ['file']
def start(self):
filename = os.path.join(
self.builder.basedir, self.args.get('workdir', ''), self.args['file'])
try:
stat = os.stat(filename)
self.sendStatus({'stat': tuple(stat)})
self.sendStatus({'rc': 0})
except OSError as e:
log.msg("StatFile %s failed" % filename, e)
self.sendStatus(
{'header': '%s: %s: %s' % (self.header, e.strerror, filename)})
self.sendStatus({'rc': e.errno})
class GlobPath(base.Command):
header = "glob"
# args['path'] is relative to Builder directory, and is required.
requiredArgs = ['path']
def start(self):
pathname = os.path.join(self.builder.basedir, self.args['path'])
try:
files = glob.glob(pathname)
self.sendStatus({'files': files})
self.sendStatus({'rc': 0})
except OSError as e:
log.msg("GlobPath %s failed" % pathname, e)
self.sendStatus(
{'header': '%s: %s: %s' % (self.header, e.strerror, pathname)})
self.sendStatus({'rc': e.errno})
class ListDir(base.Command):
header = "listdir"
# args['dir'] is relative to Builder directory, and is required.
requireArgs = ['dir']
def start(self):
dirname = os.path.join(self.builder.basedir, self.args['dir'])
try:
files = os.listdir(dirname)
self.sendStatus({'files': files})
self.sendStatus({'rc': 0})
except OSError as e:
log.msg("ListDir %s failed" % dirname, e)
self.sendStatus(
{'header': '%s: %s: %s' % (self.header, e.strerror, dirname)})
self.sendStatus({'rc': e.errno})

52
slave/buildslave/commands/registry.py
View File

@@ -1,52 +0,0 @@
# This file is part of Buildbot. Buildbot is free software: you can
# redistribute it and/or modify it under the terms of the GNU General Public
# License as published by the Free Software Foundation, version 2.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 51
# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
# Copyright Buildbot Team Members
from twisted.python import reflect
commandRegistry = {
# command name : fully qualified factory name (callable)
"shell": "buildslave.commands.shell.SlaveShellCommand",
"uploadFile": "buildslave.commands.transfer.SlaveFileUploadCommand",
"uploadDirectory": "buildslave.commands.transfer.SlaveDirectoryUploadCommand",
"downloadFile": "buildslave.commands.transfer.SlaveFileDownloadCommand",
"repo": "buildslave.commands.repo.Repo",
"mkdir": "buildslave.commands.fs.MakeDirectory",
"rmdir": "buildslave.commands.fs.RemoveDirectory",
"cpdir": "buildslave.commands.fs.CopyDirectory",
"stat": "buildslave.commands.fs.StatFile",
"glob": "buildslave.commands.fs.GlobPath",
"listdir": "buildslave.commands.fs.ListDir",
# Commands that are no longer supported
"svn": "buildslave.commands.removed.Svn",
"bk": "buildslave.commands.removed.Bk",
"cvs": "buildslave.commands.removed.Cvs",
"darcs": "buildslave.commands.removed.Darcs",
"git": "buildslave.commands.removed.Git",
"bzr": "buildslave.commands.removed.Bzr",
"hg": "buildslave.commands.removed.Hg",
"p4": "buildslave.commands.removed.P4",
"mtn": "buildslave.commands.removed.Mtn",
}
def getFactory(command):
factory_name = commandRegistry[command]
factory = reflect.namedObject(factory_name)
return factory
def getAllCommandNames():
return list(commandRegistry)

72
slave/buildslave/commands/removed.py
View File

@@ -1,72 +0,0 @@
# This file is part of Buildbot. Buildbot is free software: you can
# redistribute it and/or modify it under the terms of the GNU General Public
# License as published by the Free Software Foundation, version 2.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 51
# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
# Copyright Buildbot Team Members
import buildslave
from buildslave.commands import base
class RemovedSourceCommand(base.SourceBaseCommand):
def start(self):
self.sendStatus(
{"header":
"slave-side source checkout for '{0}' is no longer supported by "
"build slave of version {1}\n"
"\n"
"Since BuildBot 0.9 old source checkout method with logic on slave-side\n"
"buildbot.steps.source.{0} was removed (deprecated since BuildBot 0.8)\n"
"\n"
"Instead please use new method which has its logic on master-side and has unified params list.\n"
"Using the plugin infrastructure it's available as buildbot.plugins.{0}\n"
"\n"
.format(self.name, buildslave.version)})
self.sendStatus({"rc": 1})
class Svn(RemovedSourceCommand):
name = "SVN"
class Bk(RemovedSourceCommand):
name = "Bk"
class Cvs(RemovedSourceCommand):
name = "Cvs"
class Darcs(RemovedSourceCommand):
name = "darcs"
class Git(RemovedSourceCommand):
name = "git"
class Bzr(RemovedSourceCommand):
name = "bzr"
class Hg(RemovedSourceCommand):
name = "hg"
class P4(RemovedSourceCommand):
name = "p4"
class Mtn(RemovedSourceCommand):
name = "mtn"

239
slave/buildslave/commands/repo.py
View File

@@ -1,239 +0,0 @@
# This file is part of Buildbot. Buildbot is free software: you can
# redistribute it and/or modify it under the terms of the GNU General Public
# License as published by the Free Software Foundation, version 2.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 51
# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
# Copyright Buildbot Team Members
from __future__ import print_function
import os
import re
import textwrap
from twisted.internet import defer
from buildslave import runprocess
from buildslave.commands.base import AbandonChain
from buildslave.commands.base import SourceBaseCommand
class Repo(SourceBaseCommand):
"""Repo specific VC operation. In addition to the arguments
handled by SourceBaseCommand, this command reads the following keys:
['manifest_url'] (required): The manifests repo repository.
['manifest_branch'] (optional): Which manifest repo version (i.e. branch or tag)
to retrieve. Default: "master".
['manifest_file'] (optional): Which manifest file to use. Default: "default.xml".
['manifest_override_url'] (optional): Which manifest file to use as an overide. Default: None.
This is usually set by forced build to build over a known working base
['tarball'] (optional): The tarball base to accelerate the fetch.
['repo_downloads'] (optional): Repo downloads to do. Computer from GerritChangeSource
and forced build properties.
['jobs'] (optional): number of connections to run in parallel
repo tool will use while syncing
"""
header = "repo operation"
requiredArgs = ['manifest_url']
def setup(self, args):
SourceBaseCommand.setup(self, args)
self.manifest_url = args.get('manifest_url')
self.manifest_branch = args.get('manifest_branch')
self.manifest_file = args.get('manifest_file')
self.manifest_override_url = args.get('manifest_override_url')
self.tarball = args.get('tarball')
self.repo_downloads = args.get('repo_downloads')
# we're using string instead of an array here, because it will be transferred back
# to the master as string anyway and using eval() could have security
# implications.
self.repo_downloaded = ""
self.jobs = args.get('jobs')
self.sourcedata = "%s %s" % (self.manifest_url, self.manifest_file)
self.re_change = re.compile(
r".* refs/changes/\d\d/(\d+)/(\d+) -> FETCH_HEAD$")
self.re_head = re.compile("^HEAD is now at ([0-9a-f]+)...")
def _fullSrcdir(self):
return os.path.join(self.builder.basedir, self.srcdir)
def sourcedirIsUpdateable(self):
print(os.path.join(self._fullSrcdir(), ".repo"))
print(os.path.isdir(os.path.join(self._fullSrcdir(), ".repo")))
return os.path.isdir(os.path.join(self._fullSrcdir(), ".repo"))
def _repoCmd(self, command, cb=None, abandonOnFailure=True, **kwargs):
repo = self.getCommand("repo")
c = runprocess.RunProcess(self.builder, [repo] + command, self._fullSrcdir(),
sendRC=False, timeout=self.timeout,
maxTime=self.maxTime, usePTY=False,
logEnviron=self.logEnviron, **kwargs)
self.command = c
d = c.start()
if cb:
if abandonOnFailure:
d.addCallback(self._abandonOnFailure)
d.addCallback(cb)
return d
def _Cmd(self, cmds, callback, abandonOnFailure=True):
c = runprocess.RunProcess(self.builder, cmds, self._fullSrcdir(),
sendRC=False, timeout=self.timeout,
maxTime=self.maxTime, usePTY=False,
logEnviron=self.logEnviron)
self.command = c
d = c.start()
if abandonOnFailure:
d.addCallback(self._abandonOnFailure)
d.addCallback(callback)
return d
def sourcedataMatches(self):
try:
olddata = self.readSourcedata()
return olddata == self.sourcedata
except IOError:
return False
def doVCFull(self):
os.makedirs(self._fullSrcdir())
if self.tarball and os.path.exists(self.tarball):
return self._Cmd(['tar', '-xvzf', self.tarball], self._doPreInitCleanUp)
else:
return self._doInit(None)
def _doInit(self, res):
# on fresh init, this file may confuse repo.
if os.path.exists(os.path.join(self._fullSrcdir(), ".repo/project.list")):
os.unlink(os.path.join(self._fullSrcdir(), ".repo/project.list"))
return self._repoCmd(['init', '-u', self.manifest_url, '-b', self.manifest_branch, '-m', self.manifest_file], self._didInit)
def _didInit(self, res):
return self.doVCUpdate()
def doVCUpdate(self):
if self.repo_downloads:
self.sendStatus({'header': "will download:\n" + "repo download " +
"\nrepo download ".join(self.repo_downloads) + "\n"})
return self._doPreSyncCleanUp(None)
# a simple shell script to gather all cleanup tweaks...
# doing them one by one just complicate the stuff
# and messup the stdio log
def _cleanupCommand(self):
command = textwrap.dedent("""\
set -v
if [ -d .repo/manifests ]
then
# repo just refuse to run if manifest is messed up
# so ensure we are in a known state
cd .repo/manifests
git fetch origin
git reset --hard remotes/origin/%(manifest_branch)s
git config branch.default.merge %(manifest_branch)s
cd ..
ln -sf manifests/%(manifest_file)s manifest.xml
cd ..
fi
repo forall -c rm -f .git/index.lock
repo forall -c git clean -f -d -x 2>/dev/null
repo forall -c git reset --hard HEAD 2>/dev/null
""") % self.__dict__
return "\n".join([s.strip() for s in command.splitlines()])
def _doPreInitCleanUp(self, dummy):
command = self._cleanupCommand()
return self._Cmd(["bash", "-c", command], self._doInit, abandonOnFailure=False)
def _doPreSyncCleanUp(self, dummy):
command = self._cleanupCommand()
return self._Cmd(["bash", "-c", command], self._doManifestOveride, abandonOnFailure=False)
def _doManifestOveride(self, dummy):
if self.manifest_override_url:
self.sendStatus(
{"header": "overriding manifest with %s\n" % (self.manifest_override_url)})
if os.path.exists(os.path.join(self._fullSrcdir(), self.manifest_override_url)):
os.system("cd %s; cp -f %s manifest_override.xml" %
(self._fullSrcdir(), self.manifest_override_url))
else:
command = [
"wget", self.manifest_override_url, '-O', 'manifest_override.xml']
return self._Cmd(command, self._doSync)
return self._doSync(None)
def _doSync(self, dummy):
if self.manifest_override_url:
os.system(
"cd %s/.repo; ln -sf ../manifest_override.xml manifest.xml" % (self._fullSrcdir()))
command = ['sync']
if self.jobs:
command.append('-j' + str(self.jobs))
self.sendStatus({"header": "synching manifest %s from branch %s from %s\n"
% (self.manifest_file, self.manifest_branch, self.manifest_url)})
return self._repoCmd(command, self._didSync)
def _didSync(self, dummy):
if self.tarball and not os.path.exists(self.tarball):
return self._Cmd(['tar', '-cvzf', self.tarball, ".repo"], self._doManifest)
else:
return self._doManifest(None)
def _doManifest(self, dummy):
command = ['manifest', '-r', '-o', 'manifest-original.xml']
return self._repoCmd(command, self._doDownload, abandonOnFailure=False)
def _doDownload(self, dummy):
if hasattr(self.command, 'stderr') and self.command.stderr:
if "Automatic cherry-pick failed" in self.command.stderr or "Automatic revert failed" in self.command.stderr:
command = ['forall', '-c', 'git', 'diff', 'HEAD']
self.cherry_pick_failed = True
# call again
return self._repoCmd(command, self._DownloadAbandon, abandonOnFailure=False, keepStderr=True)
lines = self.command.stderr.split('\n')
if len(lines) > 2:
match1 = self.re_change.match(lines[1])
match2 = self.re_head.match(lines[-2])
if match1 and match2:
self.repo_downloaded += "%s/%s %s " % (
match1.group(1), match1.group(2), match2.group(1))
if self.repo_downloads:
# download each changeset while the self.download variable is not
# empty
download = self.repo_downloads.pop(0)
command = ['download'] + download.split(' ')
self.sendStatus({"header": "downloading changeset %s\n"
% (download)})
# call again
return self._repoCmd(command, self._doDownload, abandonOnFailure=False, keepStderr=True)
if self.repo_downloaded:
self.sendStatus({"repo_downloaded": self.repo_downloaded[:-1]})
return defer.succeed(0)
def maybeNotDoVCFallback(self, res):
# If we were unable to find the branch/SHA on the remote,
# clobbering the repo won't help any, so just abort the chain
if hasattr(self.command, 'stderr'):
if "Couldn't find remote ref" in self.command.stderr:
raise AbandonChain(-1)
if hasattr(self, 'cherry_pick_failed') or "Automatic cherry-pick failed" in self.command.stderr:
raise AbandonChain(-1)
def _DownloadAbandon(self, dummy):
self.sendStatus({"header": "abandonned due to merge failure\n"})
raise AbandonChain(-1)

61
slave/buildslave/commands/shell.py
View File

@@ -1,61 +0,0 @@
# This file is part of Buildbot. Buildbot is free software: you can
# redistribute it and/or modify it under the terms of the GNU General Public
# License as published by the Free Software Foundation, version 2.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 51
# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
# Copyright Buildbot Team Members
import os
from buildslave import runprocess
from buildslave.commands import base
class SlaveShellCommand(base.Command):
requiredArgs = ['workdir', 'command']
def start(self):
args = self.args
workdir = os.path.join(self.builder.basedir, args['workdir'])
c = runprocess.RunProcess(
self.builder,
args['command'],
workdir,
environ=args.get('env'),
timeout=args.get('timeout', None),
maxTime=args.get('maxTime', None),
sigtermTime=args.get('sigtermTime', None),
sendStdout=args.get('want_stdout', True),
sendStderr=args.get('want_stderr', True),
sendRC=True,
initialStdin=args.get('initial_stdin'),
logfiles=args.get('logfiles', {}),
usePTY=args.get('usePTY', "slave-config"),
logEnviron=args.get('logEnviron', True),
)
if args.get('interruptSignal'):
c.interruptSignal = args['interruptSignal']
c._reactor = self._reactor
self.command = c
d = self.command.start()
return d
def interrupt(self):
self.interrupted = True
self.command.kill("command interrupted")
def writeStdin(self, data):
self.command.writeStdin(data)
def closeStdin(self):
self.command.closeStdin()

375
slave/buildslave/commands/transfer.py
View File

@@ -1,375 +0,0 @@
# This file is part of Buildbot. Buildbot is free software: you can
# redistribute it and/or modify it under the terms of the GNU General Public
# License as published by the Free Software Foundation, version 2.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 51
# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
# Copyright Buildbot Team Members
import os
import tarfile
import tempfile
from twisted.internet import defer
from twisted.python import log
from buildslave.commands.base import Command
class TransferCommand(Command):
def finished(self, res):
if self.debug:
log.msg('finished: stderr=%r, rc=%r' % (self.stderr, self.rc))
# don't use self.sendStatus here, since we may no longer be running
# if we have been interrupted
upd = {'rc': self.rc}
if self.stderr:
upd['stderr'] = self.stderr
self.builder.sendUpdate(upd)
return res
def interrupt(self):
if self.debug:
log.msg('interrupted')
if self.interrupted:
return
self.rc = 1
self.interrupted = True
# now we wait for the next trip around the loop. It abandon the file
# when it sees self.interrupted set.
class SlaveFileUploadCommand(TransferCommand):
"""
Upload a file from slave to build master
Arguments:
- ['workdir']: base directory to use
- ['slavesrc']: name of the slave-side file to read from
- ['writer']: RemoteReference to a buildslave.protocols.base.FileWriterProxy object
- ['maxsize']: max size (in bytes) of file to write
- ['blocksize']: max size for each data block
- ['keepstamp']: whether to preserve file modified and accessed times
"""
debug = False
requiredArgs = ['workdir', 'slavesrc', 'writer', 'blocksize']
def setup(self, args):
self.workdir = args['workdir']
self.filename = args['slavesrc']
self.writer = args['writer']
self.remaining = args['maxsize']
self.blocksize = args['blocksize']
self.keepstamp = args.get('keepstamp', False)
self.stderr = None
self.rc = 0
def start(self):
if self.debug:
log.msg('SlaveFileUploadCommand started')
# Open file
self.path = os.path.join(self.builder.basedir,
self.workdir,
os.path.expanduser(self.filename))
accessed_modified = None
try:
if self.keepstamp:
accessed_modified = (os.path.getatime(self.path),
os.path.getmtime(self.path))
self.fp = open(self.path, 'rb')
if self.debug:
log.msg("Opened '%s' for upload" % self.path)
except Exception:
self.fp = None
self.stderr = "Cannot open file '%s' for upload" % self.path
self.rc = 1
if self.debug:
log.msg("Cannot open file '%s' for upload" % self.path)
self.sendStatus({'header': "sending %s" % self.path})
d = defer.Deferred()
self._reactor.callLater(0, self._loop, d)
def _close_ok(res):
self.fp = None
d1 = self.writer.callRemote("close")
def _utime_ok(res):
return self.writer.callRemote("utime", accessed_modified)
if self.keepstamp:
d1.addCallback(_utime_ok)
return d1
def _close_err(f):
self.rc = 1
self.fp = None
# call remote's close(), but keep the existing failure
d1 = self.writer.callRemote("close")
def eb(f2):
log.msg("ignoring error from remote close():")
log.err(f2)
d1.addErrback(eb)
d1.addBoth(lambda _: f) # always return _loop failure
return d1
d.addCallbacks(_close_ok, _close_err)
d.addBoth(self.finished)
return d
def _loop(self, fire_when_done):
d = defer.maybeDeferred(self._writeBlock)
def _done(finished):
if finished:
fire_when_done.callback(None)
else:
self._loop(fire_when_done)
def _err(why):
fire_when_done.errback(why)
d.addCallbacks(_done, _err)
return None
def _writeBlock(self):
"""Write a block of data to the remote writer"""
if self.interrupted or self.fp is None:
if self.debug:
log.msg('SlaveFileUploadCommand._writeBlock(): end')
return True
length = self.blocksize
if self.remaining is not None and length > self.remaining:
length = self.remaining
if length <= 0:
if self.stderr is None:
self.stderr = 'Maximum filesize reached, truncating file \'%s\'' \
% self.path
self.rc = 1
data = ''
else:
data = self.fp.read(length)
if self.debug:
log.msg('SlaveFileUploadCommand._writeBlock(): ' +
'allowed=%d readlen=%d' % (length, len(data)))
if len(data) == 0:
log.msg("EOF: callRemote(close)")
return True
if self.remaining is not None:
self.remaining = self.remaining - len(data)
assert self.remaining >= 0
d = self.writer.callRemote('write', data)
d.addCallback(lambda res: False)
return d
class SlaveDirectoryUploadCommand(SlaveFileUploadCommand):
debug = False
requiredArgs = ['workdir', 'slavesrc', 'writer', 'blocksize']
def setup(self, args):
self.workdir = args['workdir']
self.dirname = args['slavesrc']
self.writer = args['writer']
self.remaining = args['maxsize']
self.blocksize = args['blocksize']
self.compress = args['compress']
self.stderr = None
self.rc = 0
def start(self):
if self.debug:
log.msg('SlaveDirectoryUploadCommand started')
self.path = os.path.join(self.builder.basedir,
self.workdir,
os.path.expanduser(self.dirname))
if self.debug:
log.msg("path: %r" % self.path)
# Create temporary archive
fd, self.tarname = tempfile.mkstemp()
fileobj = os.fdopen(fd, 'w')
if self.compress == 'bz2':
mode = 'w|bz2'
elif self.compress == 'gz':
mode = 'w|gz'
else:
mode = 'w'
archive = tarfile.open(name=self.tarname, mode=mode, fileobj=fileobj)
archive.add(self.path, '')
archive.close()
fileobj.close()
# Transfer it
self.fp = open(self.tarname, 'rb')
self.sendStatus({'header': "sending %s" % self.path})
d = defer.Deferred()
self._reactor.callLater(0, self._loop, d)
def unpack(res):
d1 = self.writer.callRemote("unpack")
def unpack_err(f):
self.rc = 1
return f
d1.addErrback(unpack_err)
d1.addCallback(lambda ignored: res)
return d1
d.addCallback(unpack)
d.addBoth(self.finished)
return d
def finished(self, res):
self.fp.close()
os.remove(self.tarname)
return TransferCommand.finished(self, res)
class SlaveFileDownloadCommand(TransferCommand):
"""
Download a file from master to slave
Arguments:
- ['workdir']: base directory to use
- ['slavedest']: name of the slave-side file to be created
- ['reader']: RemoteReference to a buildslave.protocols.base.FileReaderProxy object
- ['maxsize']: max size (in bytes) of file to write
- ['blocksize']: max size for each data block
- ['mode']: access mode for the new file
"""
debug = False
requiredArgs = ['workdir', 'slavedest', 'reader', 'blocksize']
def setup(self, args):
self.workdir = args['workdir']
self.filename = args['slavedest']
self.reader = args['reader']
self.bytes_remaining = args['maxsize']
self.blocksize = args['blocksize']
self.mode = args['mode']
self.stderr = None
self.rc = 0
def start(self):
if self.debug:
log.msg('SlaveFileDownloadCommand starting')
# Open file
self.path = os.path.join(self.builder.basedir,
self.workdir,
os.path.expanduser(self.filename))
dirname = os.path.dirname(self.path)
if not os.path.exists(dirname):
os.makedirs(dirname)
try:
self.fp = open(self.path, 'wb')
if self.debug:
log.msg("Opened '%s' for download" % self.path)
if self.mode is not None:
# note: there is a brief window during which the new file
# will have the buildslave's default (umask) mode before we
# set the new one. Don't use this mode= feature to keep files
# private: use the buildslave's umask for that instead. (it
# is possible to call os.umask() before and after the open()
# call, but cleaning up from exceptions properly is more of a
# nuisance that way).
os.chmod(self.path, self.mode)
except IOError:
# TODO: this still needs cleanup
self.fp = None
self.stderr = "Cannot open file '%s' for download" % self.path
self.rc = 1
if self.debug:
log.msg("Cannot open file '%s' for download" % self.path)
d = defer.Deferred()
self._reactor.callLater(0, self._loop, d)
def _close(res):
# close the file, but pass through any errors from _loop
d1 = self.reader.callRemote('close')
d1.addErrback(log.err, 'while trying to close reader')
d1.addCallback(lambda ignored: res)
return d1
d.addBoth(_close)
d.addBoth(self.finished)
return d
def _loop(self, fire_when_done):
d = defer.maybeDeferred(self._readBlock)
def _done(finished):
if finished:
fire_when_done.callback(None)
else:
self._loop(fire_when_done)
def _err(why):
fire_when_done.errback(why)
d.addCallbacks(_done, _err)
return None
def _readBlock(self):
"""Read a block of data from the remote reader."""
if self.interrupted or self.fp is None:
if self.debug:
log.msg('SlaveFileDownloadCommand._readBlock(): end')
return True
length = self.blocksize
if self.bytes_remaining is not None and length > self.bytes_remaining:
length = self.bytes_remaining
if length <= 0:
if self.stderr is None:
self.stderr = "Maximum filesize reached, truncating file '%s'" \
% self.path
self.rc = 1
return True
else:
d = self.reader.callRemote('read', length)
d.addCallback(self._writeData)
return d
def _writeData(self, data):
if self.debug:
log.msg('SlaveFileDownloadCommand._readBlock(): readlen=%d' %
len(data))
if len(data) == 0:
return True
if self.bytes_remaining is not None:
self.bytes_remaining = self.bytes_remaining - len(data)
assert self.bytes_remaining >= 0
self.fp.write(data)
return False
def finished(self, res):
if self.fp is not None:
self.fp.close()
return TransferCommand.finished(self, res)

101
slave/buildslave/commands/utils.py
View File

@@ -1,101 +0,0 @@
# This file is part of Buildbot. Buildbot is free software: you can
# redistribute it and/or modify it under the terms of the GNU General Public
# License as published by the Free Software Foundation, version 2.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 51
# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
# Copyright Buildbot Team Members
import os
from twisted.python import log
from twisted.python import runtime
from twisted.python.procutils import which
def getCommand(name):
possibles = which(name)
if not possibles:
raise RuntimeError("Couldn't find executable for '%s'" % name)
#
# Under windows, if there is more than one executable "thing"
# that matches (e.g. *.bat, *.cmd and *.exe), we not just use
# the first in alphabet (*.bat/*.cmd) if there is a *.exe.
# e.g. under MSysGit/Windows, there is both a git.cmd and a
# git.exe on path, but we want the git.exe, since the git.cmd
# does not seem to work properly with regard to errors raised
# and catched in buildbot slave command (vcs.py)
#
if runtime.platformType == 'win32' and len(possibles) > 1:
possibles_exe = which(name + ".exe")
if possibles_exe:
return possibles_exe[0]
return possibles[0]
# this just keeps pyflakes happy on non-Windows systems
if runtime.platformType != 'win32':
WindowsError = RuntimeError
if runtime.platformType == 'win32': # pragma: no cover
def rmdirRecursive(dir):
"""This is a replacement for shutil.rmtree that works better under
windows. Thanks to Bear at the OSAF for the code."""
if not os.path.exists(dir):
return
if os.path.islink(dir) or os.path.isfile(dir):
os.remove(dir)
return
# Verify the directory is read/write/execute for the current user
os.chmod(dir, 0o700)
# os.listdir below only returns a list of unicode filenames if the parameter is unicode
# Thus, if a non-unicode-named dir contains a unicode filename, that filename will get garbled.
# So force dir to be unicode.
if not isinstance(dir, unicode):
try:
dir = unicode(dir, "utf-8")
except UnicodeDecodeError:
log.err("rmdirRecursive: decoding from UTF-8 failed (ignoring)")
try:
list = os.listdir(dir)
except WindowsError as e:
msg = ("rmdirRecursive: unable to listdir %s (%s). Trying to "
"remove like a dir" % (dir, e.strerror.decode('mbcs')))
log.msg(msg.encode('utf-8'))
os.rmdir(dir)
return
for name in list:
full_name = os.path.join(dir, name)
# on Windows, if we don't have write permission we can't remove
# the file/directory either, so turn that on
if os.name == 'nt':
if not os.access(full_name, os.W_OK):
# I think this is now redundant, but I don't have an NT
# machine to test on, so I'm going to leave it in place
# -warner
os.chmod(full_name, 0o600)
if os.path.islink(full_name):
os.remove(full_name) # as suggested in bug #792
elif os.path.isdir(full_name):
rmdirRecursive(full_name)
else:
if os.path.isfile(full_name):
os.chmod(full_name, 0o700)
os.remove(full_name)
os.rmdir(dir)
else:
# use rmtree on POSIX
import shutil
rmdirRecursive = shutil.rmtree

25
slave/buildslave/exceptions.py
View File

@@ -1,25 +0,0 @@
# This file is part of Buildbot. Buildbot is free software: you can
# redistribute it and/or modify it under the terms of the GNU General Public
# License as published by the Free Software Foundation, version 2.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 51
# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
# Copyright Buildbot Team Members
class AbandonChain(Exception):
"""A series of chained steps can raise this exception to indicate that
one of the intermediate RunProcesses has failed, such that there is no
point in running the remainder. 'rc' should be the non-zero exit code of
the failing ShellCommand."""
def __repr__(self):
return "<AbandonChain rc=%s>" % self.args[0]

76
slave/buildslave/interfaces.py
View File

@@ -1,76 +0,0 @@
# This file is part of Buildbot. Buildbot is free software: you can
# redistribute it and/or modify it under the terms of the GNU General Public
# License as published by the Free Software Foundation, version 2.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 51
# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
# Copyright Buildbot Team Members
# disable pylint warnings triggered by interface definitions
# pylint: disable=no-self-argument
# pylint: disable=no-method-argument
from zope.interface import Interface
class ISlaveCommand(Interface):
"""This interface is implemented by all of the buildslave's Command
subclasses. It specifies how the buildslave can start, interrupt, and
query the various Commands running on behalf of the buildmaster."""
def __init__(builder, stepId, args):
"""Create the Command. 'builder' is a reference to the parent
buildbot.bot.SlaveBuilder instance, which will be used to send status
updates (by calling builder.sendStatus). 'stepId' is a random string
which helps correlate slave logs with the master. 'args' is a dict of
arguments that comes from the master-side BuildStep, with contents
that are specific to the individual Command subclass.
This method is not intended to be subclassed."""
def setup(args):
"""This method is provided for subclasses to override, to extract
parameters from the 'args' dictionary. The default implemention does
nothing. It will be called from __init__"""
def start():
"""Begin the command, and return a Deferred.
While the command runs, it should send status updates to the
master-side BuildStep by calling self.sendStatus(status). The
'status' argument is typically a dict with keys like 'stdout',
'stderr', and 'rc'.
When the step completes, it should fire the Deferred (the results are
not used). If an exception occurs during execution, it may also
errback the deferred, however any reasonable errors should be trapped
and indicated with a non-zero 'rc' status rather than raising an
exception. Exceptions should indicate problems within the buildbot
itself, not problems in the project being tested.
"""
def interrupt():
"""This is called to tell the Command that the build is being stopped
and therefore the command should be terminated as quickly as
possible. The command may continue to send status updates, up to and
including an 'rc' end-of-command update (which should indicate an
error condition). The Command's deferred should still be fired when
the command has finally completed.
If the build is being stopped because the slave it shutting down or
because the connection to the buildmaster has been lost, the status
updates will simply be discarded. The Command does not need to be
aware of this.
Child shell processes should be killed. Simple ShellCommand classes
can just insert a header line indicating that the process will be
killed, then os.kill() the child."""

55
slave/buildslave/monkeypatches/__init__.py
View File

@@ -1,55 +0,0 @@
# This file is part of Buildbot. Buildbot is free software: you can
# redistribute it and/or modify it under the terms of the GNU General Public
# License as published by the Free Software Foundation, version 2.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 51
# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
# Copyright Buildbot Team Members
import twisted
from twisted.python import versions
def patch_bug4881():
# this patch doesn't apply (or even import!) on Windows
import sys
if sys.platform == 'win32':
return
# this bug was only present in Twisted-10.2.0
if twisted.version == versions.Version('twisted', 10, 2, 0):
from buildslave.monkeypatches import bug4881
bug4881.patch()
def patch_bug5079():
# this bug will hopefully be patched in Twisted-12.0.0
if twisted.version < versions.Version('twisted', 12, 0, 0):
from buildslave.monkeypatches import bug5079
bug5079.patch()
def patch_testcase_assert_raises_regexp():
# pythons before 2.7 does not have TestCase.assertRaisesRegexp() method
# add our local implementation if needed
import sys
if sys.version_info[:2] < (2, 7):
from buildslave.monkeypatches import testcase_assert
testcase_assert.patch()
def patch_all(for_tests=False):
if for_tests:
patch_testcase_assert_raises_regexp()
patch_bug4881()
patch_bug5079()

212
slave/buildslave/monkeypatches/bug4881.py
View File

@@ -1,212 +0,0 @@
# coding=utf-8
# This file is part of Buildbot. Buildbot is free software: you can
# redistribute it and/or modify it under the terms of the GNU General Public
# License as published by the Free Software Foundation, version 2.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 51
# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
# Copyright Buildbot Team Members
import os
from builtins import range
from twisted.internet import process
from twisted.python import log
def patch():
log.msg("Applying patch for http://twistedmatrix.com/trac/ticket/4881")
process._listOpenFDs = _listOpenFDs
#
# Everything below this line was taken verbatim from Twisted, except as
# annotated.
#
# r31474:trunk/LICENSE
# Copyright (c) 2001-2010
# Allen Short
# Andy Gayton
# Andrew Bennetts
# Antoine Pitrou
# Apple Computer, Inc.
# Benjamin Bruheim
# Bob Ippolito
# Canonical Limited
# Christopher Armstrong
# David Reid
# Donovan Preston
# Eric Mangold
# Eyal Lotem
# Itamar Shtull-Trauring
# James Knight
# Jason A. Mobarak
# Jean-Paul Calderone
# Jessica McKellar
# Jonathan Jacobs
# Jonathan Lange
# Jonathan D. Simms
# Jürgen Hermann
# Kevin Horn
# Kevin Turner
# Mary Gardiner
# Matthew Lefkowitz
# Massachusetts Institute of Technology
# Moshe Zadka
# Paul Swartz
# Pavel Pergamenshchik
# Ralph Meijer
# Sean Riley
# Software Freedom Conservancy
# Travis B. Hartwell
# Thijs Triemstra
# Thomas Herve
# Timothy Allen
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
#
# r31474:trunk/twisted/internet/process.py
# Copyright (c) Twisted Matrix Laboratories.
# See LICENSE for details.
class _FDDetector(object):
"""
This class contains the logic necessary to decide which of the available
system techniques should be used to detect the open file descriptors for
the current process. The chosen technique gets monkey-patched into the
_listOpenFDs method of this class so that the detection only needs to occur
once.
@ivars listdir: The implementation of listdir to use. This gets overwritten
by the test cases.
@ivars getpid: The implementation of getpid to use, returns the PID of the
running process.
@ivars openfile: The implementation of open() to use, by default the Python
builtin.
"""
# So that we can unit test this
listdir = os.listdir
getpid = os.getpid
openfile = open
def _listOpenFDs(self):
"""
Figure out which implementation to use, then run it.
"""
self._listOpenFDs = self._getImplementation()
return self._listOpenFDs()
def _getImplementation(self):
"""
Check if /dev/fd works, if so, use that. Otherwise, check if
/proc/%d/fd exists, if so use that.
Otherwise, ask resource.getrlimit, if that throws an exception, then
fallback to _fallbackFDImplementation.
"""
try:
self.listdir("/dev/fd")
if self._checkDevFDSanity(): # FreeBSD support :-)
return self._devFDImplementation
else:
return self._fallbackFDImplementation
except Exception: # changed in Buildbot to avoid bare 'except'
try:
self.listdir("/proc/%d/fd" % (self.getpid(),))
return self._procFDImplementation
except Exception: # changed in Buildbot to avoid bare 'except'
try:
self._resourceFDImplementation() # Imports resource
return self._resourceFDImplementation
except Exception: # changed in Buildbot to avoid bare 'except'
return self._fallbackFDImplementation
def _checkDevFDSanity(self):
"""
Returns true iff opening a file modifies the fds visible
in /dev/fd, as it should on a sane platform.
"""
start = self.listdir("/dev/fd")
self.openfile("/dev/null", "r") # changed in Buildbot to hush pyflakes
end = self.listdir("/dev/fd")
return start != end
def _devFDImplementation(self):
"""
Simple implementation for systems where /dev/fd actually works.
See: http://www.freebsd.org/cgi/man.cgi?fdescfs
"""
dname = "/dev/fd"
result = [int(fd) for fd in os.listdir(dname)]
return result
def _procFDImplementation(self):
"""
Simple implementation for systems where /proc/pid/fd exists (we assume
it works).
"""
dname = "/proc/%d/fd" % (os.getpid(),)
return [int(fd) for fd in os.listdir(dname)]
def _resourceFDImplementation(self):
"""
Fallback implementation where the resource module can inform us about
how many FDs we can expect.
Note that on OS-X we expect to be using the /dev/fd implementation.
"""
import resource
maxfds = resource.getrlimit(resource.RLIMIT_NOFILE)[1] + 1
# OS-X reports 9223372036854775808. That's a lot of fds
# to close
if maxfds > 1024:
maxfds = 1024
return range(maxfds)
def _fallbackFDImplementation(self):
"""
Fallback-fallback implementation where we just assume that we need to
close 256 FDs.
"""
maxfds = 256
return range(maxfds)
detector = _FDDetector()
def _listOpenFDs():
"""
Use the global detector object to figure out which FD implementation to
use.
"""
return detector._listOpenFDs()

58
slave/buildslave/monkeypatches/bug5079.py
View File

@@ -1,58 +0,0 @@
# This file is part of Buildbot. Buildbot is free software: you can
# redistribute it and/or modify it under the terms of the GNU General Public
# License as published by the Free Software Foundation, version 2.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 51
# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
# Copyright Buildbot Team Members
from twisted import version
from twisted.python import log
from twisted.python import versions
from twisted.spread import pb
from twisted.spread.interfaces import IJellyable
def patch():
if version < versions.Version('twisted', 8, 2, 0):
return # too old
log.msg("Applying patch for http://twistedmatrix.com/trac/ticket/5079")
if not hasattr(pb, '_JellyableAvatarMixin'):
log.msg("..patch not applicable; please file a bug at buildbot.net")
else:
pb._JellyableAvatarMixin._cbLogin = _fixed_cbLogin
def _fixed_cbLogin(self, xxx_todo_changeme):
"""
Ensure that the avatar to be returned to the client is jellyable and
set up disconnection notification to call the realm's logout object.
"""
(interface, avatar, logout) = xxx_todo_changeme
if not IJellyable.providedBy(avatar):
avatar = pb.AsReferenceable(avatar, "perspective")
puid = avatar.processUniqueID()
# only call logout once, whether the connection is dropped (disconnect)
# or a logout occurs (cleanup), and be careful to drop the reference to
# it in either case
logout = [logout]
def maybeLogout():
if not logout:
return
fn = logout[0]
del logout[0]
fn()
self.broker._localCleanup[puid] = maybeLogout
self.broker.notifyOnDisconnect(maybeLogout)
return avatar

48
slave/buildslave/monkeypatches/testcase_assert.py
View File

@@ -1,48 +0,0 @@
# This file is part of Buildbot. Buildbot is free software: you can
# redistribute it and/or modify it under the terms of the GNU General Public
# License as published by the Free Software Foundation, version 2.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 51
# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
# Copyright Buildbot Team Members
import re
import unittest
def _assertRaisesRegexp(self, expected_exception, expected_regexp,
callable_obj, *args, **kwds):
"""
Asserts that the message in a raised exception matches a regexp.
This is a simple clone of unittest.TestCase.assertRaisesRegexp() method
introduced in python 2.7. The goal for this function is to behave exactly
as assertRaisesRegexp() in standard library.
"""
exception = None
try:
callable_obj(*args, **kwds)
except expected_exception as ex: # let unexpected exceptions pass through
exception = ex
if exception is None:
self.fail("%s not raised" % str(expected_exception.__name__))
if isinstance(expected_regexp, basestring):
expected_regexp = re.compile(expected_regexp)
if not expected_regexp.search(str(exception)):
self.fail('"%s" does not match "%s"' %
(expected_regexp.pattern, str(exception)))
def patch():
unittest.TestCase.assertRaisesRegexp = _assertRaisesRegexp

60
slave/buildslave/null.py
View File

@@ -1,60 +0,0 @@
# This file is part of Buildbot. Buildbot is free software: you can
# redistribute it and/or modify it under the terms of the GNU General Public
# License as published by the Free Software Foundation, version 2.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 51
# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
# Copyright Buildbot Team Members
from twisted.internet import defer
from buildslave.base import BuildSlaveBase
class LocalBuildSlave(BuildSlaveBase):
@defer.inlineCallbacks
def startService(self):
# importing here to avoid dependency on buildbot master package
# requires buildot version >= 0.9.0b5
from buildbot.worker.protocols.null import Connection
yield BuildSlaveBase.startService(self)
# TODO: This is a workaround for using worker with "slave"-api with
# updated master. Later buildbot-slave package will be replaced with
# buildbot-worker package which will be "slave"-free, and this patch
# will not be needed.
self._workername = self.name
conn = Connection(self.parent, self)
# I don't have a master property, but my parent has.
master = self.parent.master
# TODO: This is a workaround for using worker with "slave"-api with
# updated master. Later buildbot-slave package will be replaced with
# buildbot-worker package which will be "slave"-free, and this patch
# will not be needed.
res = yield master.workers.newConnection(conn, self.name)
if res:
yield self.parent.attached(conn)
# TODO: This is a workaround for using worker with "slave"-api with
# updated master. Later buildbot-slave package will be replaced with
# buildbot-worker package which will be "slave"-free, and this patch
# will not be needed.
@property
def workername(self):
return self._workername
# TODO: This is a workaround for using worker with "slave"-api with
# updated master. Later buildbot-slave package will be replaced with
# buildbot-worker package which will be "slave"-free, and this patch
# will not be needed.
@property
def slavename(self):
return self._workername

241
slave/buildslave/pb.py
View File

@@ -1,241 +0,0 @@
# This file is part of Buildbot. Buildbot is free software: you can
# redistribute it and/or modify it under the terms of the GNU General Public
# License as published by the Free Software Foundation, version 2.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 51
# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
# Copyright Buildbot Team Members
import os.path
import signal
from twisted.application import internet
from twisted.application import service
from twisted.cred import credentials
from twisted.internet import error
from twisted.internet import reactor
from twisted.internet import task
from twisted.python import log
from twisted.spread import pb
from buildslave.base import BotBase
from buildslave.base import BuildSlaveBase
from buildslave.base import SlaveBuilderBase
from buildslave.pbutil import ReconnectingPBClientFactory
class UnknownCommand(pb.Error):
pass
class SlaveBuilderPb(SlaveBuilderBase, pb.Referenceable):
pass
class BotPb(BotBase, pb.Referenceable):
SlaveBuilder = SlaveBuilderPb
class BotFactory(ReconnectingPBClientFactory):
# 'keepaliveInterval' serves two purposes. The first is to keep the
# connection alive: it guarantees that there will be at least some
# traffic once every 'keepaliveInterval' seconds, which may help keep an
# interposed NAT gateway from dropping the address mapping because it
# thinks the connection has been abandoned. This also gives the operating
# system a chance to notice that the master has gone away, and inform us
# of such (although this could take several minutes).
keepaliveInterval = None # None = do not use keepalives
# 'maxDelay' determines the maximum amount of time the slave will wait
# between connection retries
maxDelay = 300
keepaliveTimer = None
unsafeTracebacks = 1
perspective = None
# for tests
_reactor = reactor
def __init__(self, buildmaster_host, port, keepaliveInterval, maxDelay):
ReconnectingPBClientFactory.__init__(self)
self.maxDelay = maxDelay
self.keepaliveInterval = keepaliveInterval
# NOTE: this class does not actually make the TCP connections - this information is
# only here to print useful error messages
self.buildmaster_host = buildmaster_host
self.port = port
def startedConnecting(self, connector):
log.msg("Connecting to %s:%s" % (self.buildmaster_host, self.port))
ReconnectingPBClientFactory.startedConnecting(self, connector)
self.connector = connector
def gotPerspective(self, perspective):
log.msg("Connected to %s:%s; slave is ready" %
(self.buildmaster_host, self.port))
ReconnectingPBClientFactory.gotPerspective(self, perspective)
self.perspective = perspective
try:
perspective.broker.transport.setTcpKeepAlive(1)
except Exception:
log.msg("unable to set SO_KEEPALIVE")
if not self.keepaliveInterval:
self.keepaliveInterval = 10 * 60
self.activity()
if self.keepaliveInterval:
log.msg("sending application-level keepalives every %d seconds"
% self.keepaliveInterval)
self.startTimers()
def clientConnectionFailed(self, connector, reason):
self.connector = None
why = reason
if reason.check(error.ConnectionRefusedError):
why = "Connection Refused"
log.msg("Connection to %s:%s failed: %s" %
(self.buildmaster_host, self.port, why))
ReconnectingPBClientFactory.clientConnectionFailed(self,
connector, reason)
def clientConnectionLost(self, connector, reason):
log.msg("Lost connection to %s:%s" %
(self.buildmaster_host, self.port))
self.connector = None
self.stopTimers()
self.perspective = None
ReconnectingPBClientFactory.clientConnectionLost(self,
connector, reason)
def startTimers(self):
assert self.keepaliveInterval
assert not self.keepaliveTimer
def doKeepalive():
self.keepaliveTimer = None
self.startTimers()
# Send the keepalive request. If an error occurs
# was already dropped, so just log and ignore.
log.msg("sending app-level keepalive")
d = self.perspective.callRemote("keepalive")
d.addErrback(log.err, "error sending keepalive")
self.keepaliveTimer = self._reactor.callLater(self.keepaliveInterval,
doKeepalive)
def stopTimers(self):
if self.keepaliveTimer:
self.keepaliveTimer.cancel()
self.keepaliveTimer = None
def activity(self, res=None):
"""Subclass or monkey-patch this method to be alerted whenever there is
active communication between the master and slave."""
pass
def stopFactory(self):
ReconnectingPBClientFactory.stopFactory(self)
self.stopTimers()
class BuildSlave(BuildSlaveBase, service.MultiService):
Bot = BotPb
def __init__(self, buildmaster_host, port, name, passwd, basedir,
keepalive, usePTY, keepaliveTimeout=None, umask=None,
maxdelay=300, numcpus=None, unicode_encoding=None,
allow_shutdown=None):
# note: keepaliveTimeout is ignored, but preserved here for
# backward-compatibility
service.MultiService.__init__(self)
BuildSlaveBase.__init__(
self, name, basedir, usePTY, umask=umask, unicode_encoding=unicode_encoding)
if keepalive == 0:
keepalive = None
self.numcpus = numcpus
self.shutdown_loop = None
if allow_shutdown == 'signal':
if not hasattr(signal, 'SIGHUP'):
raise ValueError("Can't install signal handler")
elif allow_shutdown == 'file':
self.shutdown_file = os.path.join(basedir, 'shutdown.stamp')
self.shutdown_mtime = 0
self.allow_shutdown = allow_shutdown
bf = self.bf = BotFactory(buildmaster_host, port, keepalive, maxdelay)
bf.startLogin(
credentials.UsernamePassword(name, passwd), client=self.bot)
self.connection = c = internet.TCPClient(buildmaster_host, port, bf)
c.setServiceParent(self)
def startService(self):
BuildSlaveBase.startService(self)
if self.allow_shutdown == 'signal':
log.msg("Setting up SIGHUP handler to initiate shutdown")
signal.signal(signal.SIGHUP, self._handleSIGHUP)
elif self.allow_shutdown == 'file':
log.msg("Watching %s's mtime to initiate shutdown" %
self.shutdown_file)
if os.path.exists(self.shutdown_file):
self.shutdown_mtime = os.path.getmtime(self.shutdown_file)
self.shutdown_loop = l = task.LoopingCall(self._checkShutdownFile)
l.start(interval=10)
def stopService(self):
self.bf.continueTrying = 0
self.bf.stopTrying()
if self.shutdown_loop:
self.shutdown_loop.stop()
self.shutdown_loop = None
return service.MultiService.stopService(self)
def _handleSIGHUP(self, *args):
log.msg("Initiating shutdown because we got SIGHUP")
return self.gracefulShutdown()
def _checkShutdownFile(self):
if os.path.exists(self.shutdown_file) and \
os.path.getmtime(self.shutdown_file) > self.shutdown_mtime:
log.msg("Initiating shutdown because %s was touched" %
self.shutdown_file)
self.gracefulShutdown()
# In case the shutdown fails, update our mtime so we don't keep
# trying to shutdown over and over again.
# We do want to be able to try again later if the master is
# restarted, so we'll keep monitoring the mtime.
self.shutdown_mtime = os.path.getmtime(self.shutdown_file)
def gracefulShutdown(self):
"""Start shutting down"""
if not self.bf.perspective:
log.msg("No active connection, shutting down NOW")
reactor.stop()
return
log.msg(
"Telling the master we want to shutdown after any running builds are finished")
d = self.bf.perspective.callRemote("shutdown")
def _shutdownfailed(err):
if err.check(AttributeError):
log.msg(
"Master does not support slave initiated shutdown. Upgrade master to 0.8.3 or later to use this feature.")
else:
log.msg('callRemote("shutdown") failed')
log.err(err)
d.addErrback(_shutdownfailed)
return d

149
slave/buildslave/pbutil.py
View File

@@ -1,149 +0,0 @@
# This file is part of Buildbot. Buildbot is free software: you can
# redistribute it and/or modify it under the terms of the GNU General Public
# License as published by the Free Software Foundation, version 2.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 51
# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
# Copyright Buildbot Team Members
"""Base classes handy for use with PB clients.
"""
from twisted.cred import error
from twisted.internet import protocol
from twisted.internet import reactor
from twisted.python import log
from twisted.spread import pb
from twisted.spread.pb import PBClientFactory
class ReconnectingPBClientFactory(PBClientFactory,
protocol.ReconnectingClientFactory):
"""Reconnecting client factory for PB brokers.
Like PBClientFactory, but if the connection fails or is lost, the factory
will attempt to reconnect.
Instead of using f.getRootObject (which gives a Deferred that can only
be fired once), override the gotRootObject method.
Instead of using the newcred f.login (which is also one-shot), call
f.startLogin() with the credentials and client, and override the
gotPerspective method.
gotRootObject and gotPerspective will be called each time the object is
received (once per successful connection attempt). You will probably want
to use obj.notifyOnDisconnect to find out when the connection is lost.
If an authorization error occurs, failedToGetPerspective() will be
invoked.
To use me, subclass, then hand an instance to a connector (like
TCPClient).
"""
# hung connections wait for a relatively long time, since a busy master may
# take a while to get back to us.
hungConnectionTimer = None
HUNG_CONNECTION_TIMEOUT = 120
def clientConnectionFailed(self, connector, reason):
PBClientFactory.clientConnectionFailed(self, connector, reason)
if self.continueTrying:
self.connector = connector
self.retry()
def clientConnectionLost(self, connector, reason):
PBClientFactory.clientConnectionLost(self, connector, reason,
reconnecting=True)
RCF = protocol.ReconnectingClientFactory
RCF.clientConnectionLost(self, connector, reason)
def startedConnecting(self, connector):
self.startHungConnectionTimer(connector)
def clientConnectionMade(self, broker):
self.resetDelay()
PBClientFactory.clientConnectionMade(self, broker)
self.doLogin(self._root, broker)
self.gotRootObject(self._root)
# newcred methods
def login(self, *args):
raise RuntimeError("login is one-shot: use startLogin instead")
def startLogin(self, credentials, client=None):
self._credentials = credentials
self._client = client
def doLogin(self, root, broker):
# newcred login()
d = self._cbSendUsername(root, self._credentials.username,
self._credentials.password, self._client)
d.addCallbacks(self.gotPerspective, self.failedToGetPerspective,
errbackArgs=(broker,))
# timer for hung connections
def startHungConnectionTimer(self, connector):
self.stopHungConnectionTimer()
def hungConnection():
log.msg(
"connection attempt timed out (is the port number correct?)")
self.hungConnectionTimer = None
connector.disconnect()
# (this will trigger the retry)
self.hungConnectionTimer = reactor.callLater(
self.HUNG_CONNECTION_TIMEOUT, hungConnection)
def stopHungConnectionTimer(self):
if self.hungConnectionTimer:
self.hungConnectionTimer.cancel()
self.hungConnectionTimer = None
# methods to override
def gotPerspective(self, perspective):
"""The remote avatar or perspective (obtained each time this factory
connects) is now available."""
self.stopHungConnectionTimer()
def gotRootObject(self, root):
"""The remote root object (obtained each time this factory connects)
is now available. This method will be called each time the connection
is established and the object reference is retrieved."""
self.stopHungConnectionTimer()
def failedToGetPerspective(self, why, broker):
"""The login process failed, most likely because of an authorization
failure (bad password), but it is also possible that we lost the new
connection before we managed to send our credentials.
"""
log.msg("ReconnectingPBClientFactory.failedToGetPerspective")
self.stopHungConnectionTimer()
# put something useful in the logs
if why.check(pb.PBConnectionLost):
log.msg("we lost the brand-new connection")
# fall through
elif why.check(error.UnauthorizedLogin):
log.msg("unauthorized login; check slave name and password")
# fall through
else:
log.err(why, 'While trying to connect:')
self.stopTrying()
reactor.stop()
return
# lose the current connection, which will trigger a retry
broker.transport.loseConnection()

928
slave/buildslave/runprocess.py
View File

@@ -1,928 +0,0 @@
# This file is part of Buildbot. Buildbot is free software: you can
# redistribute it and/or modify it under the terms of the GNU General Public
# License as published by the Free Software Foundation, version 2.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 51
# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
# Copyright Buildbot Team Members
"""
Support for running 'shell commands'
"""
import os
import pprint
import re
import signal
import stat
import subprocess
import sys
import traceback
from collections import deque
from tempfile import NamedTemporaryFile
from future.utils import iteritems
from twisted.internet import defer
from twisted.internet import error
from twisted.internet import protocol
from twisted.internet import reactor
from twisted.internet import task
from twisted.python import failure
from twisted.python import log
from twisted.python import runtime
from twisted.python.win32 import quoteArguments
from buildslave import util
from buildslave.exceptions import AbandonChain
if runtime.platformType == 'posix':
from twisted.internet.process import Process
def win32_batch_quote(cmd_list):
# Quote cmd_list to a string that is suitable for inclusion in a
# Windows batch file. This is not quite the same as quoting it for the
# shell, as cmd.exe doesn't support the %% escape in interactive mode.
def escape_arg(arg):
arg = quoteArguments([arg])
# escape shell special characters
arg = re.sub(r'[@()^"<>&|]', r'^\g<0>', arg)
# prevent variable expansion
return arg.replace('%', '%%')
return ' '.join(map(escape_arg, cmd_list))
def shell_quote(cmd_list):
# attempt to quote cmd_list such that a shell will properly re-interpret
# it. The pipes module is only available on UNIX; also, the quote
# function is undocumented (although it looks like it will be documented
# soon: http://bugs.python.org/issue9723). Finally, it has a nasty bug
# in some versions where an empty string is not quoted.
#
# So:
# - use pipes.quote on UNIX, handling '' as a special case
# - use our own custom function on Windows
if runtime.platformType == 'win32':
return win32_batch_quote(cmd_list)
else:
import pipes
def quote(e):
if not e:
return '""'
return pipes.quote(e)
return " ".join([quote(e) for e in cmd_list])
class LogFileWatcher(object):
POLL_INTERVAL = 2
def __init__(self, command, name, logfile, follow=False):
self.command = command
self.name = name
self.logfile = logfile
log.msg("LogFileWatcher created to watch %s" % logfile)
# we are created before the ShellCommand starts. If the logfile we're
# supposed to be watching already exists, record its size and
# ctime/mtime so we can tell when it starts to change.
self.old_logfile_stats = self.statFile()
self.started = False
# follow the file, only sending back lines
# added since we started watching
self.follow = follow
# every 2 seconds we check on the file again
self.poller = task.LoopingCall(self.poll)
def start(self):
self.poller.start(self.POLL_INTERVAL).addErrback(self._cleanupPoll)
def _cleanupPoll(self, err):
log.err(err, msg="Polling error")
self.poller = None
def stop(self):
self.poll()
if self.poller is not None:
self.poller.stop()
if self.started:
self.f.close()
def statFile(self):
if os.path.exists(self.logfile):
s = os.stat(self.logfile)
return (s[stat.ST_CTIME], s[stat.ST_MTIME], s[stat.ST_SIZE])
return None
def poll(self):
if not self.started:
s = self.statFile()
if s == self.old_logfile_stats:
return # not started yet
if not s:
# the file was there, but now it's deleted. Forget about the
# initial state, clearly the process has deleted the logfile
# in preparation for creating a new one.
self.old_logfile_stats = None
return # no file to work with
self.f = open(self.logfile, "rb")
# if we only want new lines, seek to
# where we stat'd so we only find new
# lines
if self.follow:
self.f.seek(s[2], 0)
self.started = True
self.f.seek(self.f.tell(), 0)
while True:
data = self.f.read(10000)
if not data:
return
self.command.addLogfile(self.name, data)
if runtime.platformType == 'posix':
class ProcGroupProcess(Process):
"""Simple subclass of Process to also make the spawned process a process
group leader, so we can kill all members of the process group."""
def _setupChild(self, *args, **kwargs):
Process._setupChild(self, *args, **kwargs)
# this will cause the child to be the leader of its own process group;
# it's also spelled setpgrp() on BSD, but this spelling seems to work
# everywhere
os.setpgid(0, 0)
class RunProcessPP(protocol.ProcessProtocol):
debug = False
def __init__(self, command):
self.command = command
self.pending_stdin = ""
self.stdin_finished = False
self.killed = False
def setStdin(self, data):
assert not self.connected
self.pending_stdin = data
def connectionMade(self):
if self.debug:
log.msg("RunProcessPP.connectionMade")
if self.command.useProcGroup:
if self.debug:
log.msg(" recording pid %d as subprocess pgid"
% (self.transport.pid,))
self.transport.pgid = self.transport.pid
if self.pending_stdin:
if self.debug:
log.msg(" writing to stdin")
self.transport.write(self.pending_stdin)
if self.debug:
log.msg(" closing stdin")
self.transport.closeStdin()
def outReceived(self, data):
if self.debug:
log.msg("RunProcessPP.outReceived")
self.command.addStdout(data)
def errReceived(self, data):
if self.debug:
log.msg("RunProcessPP.errReceived")
self.command.addStderr(data)
def processEnded(self, status_object):
if self.debug:
log.msg("RunProcessPP.processEnded", status_object)
# status_object is a Failure wrapped around an
# error.ProcessTerminated or and error.ProcessDone.
# requires twisted >= 1.0.4 to overcome a bug in process.py
sig = status_object.value.signal
rc = status_object.value.exitCode
# sometimes, even when we kill a process, GetExitCodeProcess will still return
# a zero exit status. So we force it. See
# http://stackoverflow.com/questions/2061735/42-passed-to-terminateprocess-sometimes-getexitcodeprocess-returns-0
if self.killed and rc == 0:
log.msg(
"process was killed, but exited with status 0; faking a failure")
# windows returns '1' even for signalled failures, while POSIX
# returns -1
if runtime.platformType == 'win32':
rc = 1
else:
rc = -1
self.command.finished(sig, rc)
class RunProcess(object):
"""
This is a helper class, used by slave commands to run programs in a child
shell.
"""
notreally = False
BACKUP_TIMEOUT = 5
interruptSignal = "KILL"
CHUNK_LIMIT = 128 * 1024
# Don't send any data until at least BUFFER_SIZE bytes have been collected
# or BUFFER_TIMEOUT elapsed
BUFFER_SIZE = 64 * 1024
BUFFER_TIMEOUT = 5
# For sending elapsed time:
startTime = None
elapsedTime = None
# For scheduling future events
_reactor = reactor
# I wish we had easy access to CLOCK_MONOTONIC in Python:
# http://www.opengroup.org/onlinepubs/000095399/functions/clock_getres.html
# Then changes to the system clock during a run wouldn't effect the "elapsed
# time" results.
def __init__(self, builder, command,
workdir, environ=None,
sendStdout=True, sendStderr=True, sendRC=True,
timeout=None, maxTime=None, sigtermTime=None,
initialStdin=None, keepStdout=False, keepStderr=False,
logEnviron=True, logfiles={}, usePTY="slave-config",
useProcGroup=True):
"""
@param keepStdout: if True, we keep a copy of all the stdout text
that we've seen. This copy is available in
self.stdout, which can be read after the command
has finished.
@param keepStderr: same, for stderr
@param usePTY: "slave-config" -> use the SlaveBuilder's usePTY;
otherwise, true to use a PTY, false to not use a PTY.
@param useProcGroup: (default True) use a process group for non-PTY
process invocations
"""
self.builder = builder
if isinstance(command, list):
def obfus(w):
if (isinstance(w, tuple) and len(w) == 3
and w[0] == 'obfuscated'):
return util.Obfuscated(w[1], w[2])
return w
command = [obfus(w) for w in command]
# We need to take unicode commands and arguments and encode them using
# the appropriate encoding for the slave. This is mostly platform
# specific, but can be overridden in the slave's buildbot.tac file.
#
# Encoding the command line here ensures that the called executables
# receive arguments as bytestrings encoded with an appropriate
# platform-specific encoding. It also plays nicely with twisted's
# spawnProcess which checks that arguments are regular strings or
# unicode strings that can be encoded as ascii (which generates a
# warning).
def to_str(cmd):
if isinstance(cmd, (tuple, list)):
for i, a in enumerate(cmd):
if isinstance(a, unicode):
cmd[i] = a.encode(self.builder.unicode_encoding)
elif isinstance(cmd, unicode):
cmd = cmd.encode(self.builder.unicode_encoding)
return cmd
self.command = to_str(util.Obfuscated.get_real(command))
self.fake_command = to_str(util.Obfuscated.get_fake(command))
self.sendStdout = sendStdout
self.sendStderr = sendStderr
self.sendRC = sendRC
self.logfiles = logfiles
self.workdir = workdir
self.process = None
if not os.path.exists(workdir):
os.makedirs(workdir)
if environ:
for key, v in iteritems(environ):
if isinstance(v, list):
# Need to do os.pathsep translation. We could either do that
# by replacing all incoming ':'s with os.pathsep, or by
# accepting lists. I like lists better.
# If it's not a string, treat it as a sequence to be
# turned in to a string.
environ[key] = os.pathsep.join(environ[key])
if "PYTHONPATH" in environ:
environ['PYTHONPATH'] += os.pathsep + "${PYTHONPATH}"
# do substitution on variable values matching pattern: ${name}
p = re.compile(r'\${([0-9a-zA-Z_]*)}')
def subst(match):
return os.environ.get(match.group(1), "")
newenv = {}
for key in os.environ:
# setting a key to None will delete it from the slave
# environment
if key not in environ or environ[key] is not None:
newenv[key] = os.environ[key]
for key, v in iteritems(environ):
if v is not None:
if not isinstance(v, basestring):
raise RuntimeError("'env' values must be strings or "
"lists; key '%s' is incorrect" % (key,))
newenv[key] = p.sub(subst, v)
self.environ = newenv
else: # not environ
self.environ = os.environ.copy()
self.initialStdin = to_str(initialStdin)
self.logEnviron = logEnviron
self.timeout = timeout
self.ioTimeoutTimer = None
self.sigtermTime = sigtermTime
self.maxTime = maxTime
self.maxTimeoutTimer = None
self.killTimer = None
self.keepStdout = keepStdout
self.keepStderr = keepStderr
self.buffered = deque()
self.buflen = 0
self.sendBuffersTimer = None
if usePTY == "slave-config":
self.usePTY = self.builder.usePTY
else:
self.usePTY = usePTY
# usePTY=True is a convenience for cleaning up all children and
# grandchildren of a hung command. Fall back to usePTY=False on systems
# and in situations where ptys cause problems. PTYs are posix-only,
# and for .closeStdin to matter, we must use a pipe, not a PTY
if runtime.platformType != "posix" or initialStdin is not None:
if self.usePTY and usePTY != "slave-config":
self.sendStatus(
{'header': "WARNING: disabling usePTY for this command"})
self.usePTY = False
# use an explicit process group on POSIX, noting that usePTY always implies
# a process group.
if runtime.platformType != 'posix':
useProcGroup = False
elif self.usePTY:
useProcGroup = True
self.useProcGroup = useProcGroup
self.logFileWatchers = []
for name, filevalue in self.logfiles.items():
filename = filevalue
follow = False
# check for a dictionary of options
# filename is required, others are optional
if isinstance(filevalue, dict):
filename = filevalue['filename']
follow = filevalue.get('follow', False)
w = LogFileWatcher(self, name,
os.path.join(self.workdir, filename),
follow=follow)
self.logFileWatchers.append(w)
def __repr__(self):
return "<%s '%s'>" % (self.__class__.__name__, self.fake_command)
def sendStatus(self, status):
self.builder.sendUpdate(status)
def start(self):
# return a Deferred which fires (with the exit code) when the command
# completes
if self.keepStdout:
self.stdout = ""
if self.keepStderr:
self.stderr = ""
self.deferred = defer.Deferred()
try:
self._startCommand()
except Exception:
log.err(failure.Failure(), "error in RunProcess._startCommand")
self._addToBuffers('stderr', "error in RunProcess._startCommand\n")
self._addToBuffers('stderr', traceback.format_exc())
self._sendBuffers()
# pretend it was a shell error
self.deferred.errback(AbandonChain(-1))
return self.deferred
def _startCommand(self):
# ensure workdir exists
if not os.path.isdir(self.workdir):
os.makedirs(self.workdir)
log.msg("RunProcess._startCommand")
if self.notreally:
self._addToBuffers('header', "command '%s' in dir %s" %
(self.fake_command, self.workdir))
self._addToBuffers('header', "(not really)\n")
self.finished(None, 0)
return
self.pp = RunProcessPP(self)
self.using_comspec = False
if isinstance(self.command, basestring):
if runtime.platformType == 'win32':
# allow %COMSPEC% to have args
argv = os.environ['COMSPEC'].split()
if '/c' not in argv:
argv += ['/c']
argv += [self.command]
self.using_comspec = True
else:
# for posix, use /bin/sh. for other non-posix, well, doesn't
# hurt to try
argv = ['/bin/sh', '-c', self.command]
display = self.fake_command
else:
# On windows, CreateProcess requires an absolute path to the executable.
# When we call spawnProcess below, we pass argv[0] as the executable.
# So, for .exe's that we have absolute paths to, we can call directly
# Otherwise, we should run under COMSPEC (usually cmd.exe) to
# handle path searching, etc.
if runtime.platformType == 'win32' and not \
(self.command[0].lower().endswith(".exe") and os.path.isabs(self.command[0])):
# allow %COMSPEC% to have args
argv = os.environ['COMSPEC'].split()
if '/c' not in argv:
argv += ['/c']
argv += list(self.command)
self.using_comspec = True
else:
argv = self.command
# Attempt to format this for use by a shell, although the process
# isn't perfect
display = shell_quote(self.fake_command)
# $PWD usually indicates the current directory; spawnProcess may not
# update this value, though, so we set it explicitly here. This causes
# weird problems (bug #456) on msys, though..
if not self.environ.get('MACHTYPE', None) == 'i686-pc-msys':
self.environ['PWD'] = os.path.abspath(self.workdir)
# self.stdin is handled in RunProcessPP.connectionMade
log.msg(" " + display)
self._addToBuffers('header', display + "\n")
# then comes the secondary information
msg = " in dir %s" % (self.workdir,)
if self.timeout:
if self.timeout == 1:
unit = "sec"
else:
unit = "secs"
msg += " (timeout %d %s)" % (self.timeout, unit)
if self.maxTime:
if self.maxTime == 1:
unit = "sec"
else:
unit = "secs"
msg += " (maxTime %d %s)" % (self.maxTime, unit)
log.msg(" " + msg)
self._addToBuffers('header', msg + "\n")
msg = " watching logfiles %s" % (self.logfiles,)
log.msg(" " + msg)
self._addToBuffers('header', msg + "\n")
# then the obfuscated command array for resolving unambiguity
msg = " argv: %s" % (self.fake_command,)
log.msg(" " + msg)
self._addToBuffers('header', msg + "\n")
# then the environment, since it sometimes causes problems
if self.logEnviron:
msg = " environment:\n"
env_names = sorted(self.environ.keys())
for name in env_names:
msg += " %s=%s\n" % (name, self.environ[name])
log.msg(" environment:\n%s" % (pprint.pformat(self.environ),))
self._addToBuffers('header', msg)
if self.initialStdin:
msg = " writing %d bytes to stdin" % len(self.initialStdin)
log.msg(" " + msg)
self._addToBuffers('header', msg + "\n")
msg = " using PTY: %s" % bool(self.usePTY)
log.msg(" " + msg)
self._addToBuffers('header', msg + "\n")
# put data into stdin and close it, if necessary. This will be
# buffered until connectionMade is called
if self.initialStdin:
self.pp.setStdin(self.initialStdin)
self.startTime = util.now(self._reactor)
# start the process
self.process = self._spawnProcess(
self.pp, argv[0], argv,
self.environ,
self.workdir,
usePTY=self.usePTY)
# set up timeouts
if self.timeout:
self.ioTimeoutTimer = self._reactor.callLater(
self.timeout, self.doTimeout)
if self.maxTime:
self.maxTimeoutTimer = self._reactor.callLater(
self.maxTime, self.doMaxTimeout)
for w in self.logFileWatchers:
w.start()
def _spawnProcess(self, processProtocol, executable, args=(), env={},
path=None, uid=None, gid=None, usePTY=False, childFDs=None):
"""private implementation of reactor.spawnProcess, to allow use of
L{ProcGroupProcess}"""
# use the ProcGroupProcess class, if available
if runtime.platformType == 'posix':
if self.useProcGroup and not usePTY:
return ProcGroupProcess(reactor, executable, args, env, path,
processProtocol, uid, gid, childFDs)
# fall back
if self.using_comspec:
return self._spawnAsBatch(processProtocol, executable, args, env,
path, usePTY=usePTY)
else:
return reactor.spawnProcess(processProtocol, executable, args, env,
path, usePTY=usePTY)
def _spawnAsBatch(self, processProtocol, executable, args, env,
path, usePTY):
"""A cheat that routes around the impedance mismatch between
twisted and cmd.exe with respect to escaping quotes"""
tf = NamedTemporaryFile(dir='.', suffix=".bat", delete=False)
# echo off hides this cheat from the log files.
tf.write("@echo off\n")
if isinstance(self.command, basestring):
tf.write(self.command)
else:
tf.write(win32_batch_quote(self.command))
tf.close()
argv = os.environ['COMSPEC'].split() # allow %COMSPEC% to have args
if '/c' not in argv:
argv += ['/c']
argv += [tf.name]
def unlink_temp(result):
os.unlink(tf.name)
return result
self.deferred.addBoth(unlink_temp)
return reactor.spawnProcess(processProtocol, executable, argv, env,
path, usePTY=usePTY)
def _chunkForSend(self, data):
"""
limit the chunks that we send over PB to 128k, since it has a hardwired
string-size limit of 640k.
"""
LIMIT = self.CHUNK_LIMIT
for i in range(0, len(data), LIMIT):
yield data[i:i + LIMIT]
def _collapseMsg(self, msg):
"""
Take msg, which is a dictionary of lists of output chunks, and
concatenate all the chunks into a single string
"""
retval = {}
for logname in msg:
data = "".join(msg[logname])
if isinstance(logname, tuple) and logname[0] == 'log':
retval['log'] = (logname[1], data)
else:
retval[logname] = data
return retval
def _sendMessage(self, msg):
"""
Collapse and send msg to the master
"""
if not msg:
return
msg = self._collapseMsg(msg)
self.sendStatus(msg)
def _bufferTimeout(self):
self.sendBuffersTimer = None
self._sendBuffers()
def _sendBuffers(self):
"""
Send all the content in our buffers.
"""
msg = {}
msg_size = 0
lastlog = None
logdata = []
while self.buffered:
# Grab the next bits from the buffer
logname, data = self.buffered.popleft()
# If this log is different than the last one, then we have to send
# out the message so far. This is because the message is
# transferred as a dictionary, which makes the ordering of keys
# unspecified, and makes it impossible to interleave data from
# different logs. A future enhancement could be to change the
# master to support a list of (logname, data) tuples instead of a
# dictionary.
# On our first pass through this loop lastlog is None
if lastlog is None:
lastlog = logname
elif logname != lastlog:
self._sendMessage(msg)
msg = {}
msg_size = 0
lastlog = logname
logdata = msg.setdefault(logname, [])
# Chunkify the log data to make sure we're not sending more than
# CHUNK_LIMIT at a time
for chunk in self._chunkForSend(data):
if len(chunk) == 0:
continue
logdata.append(chunk)
msg_size += len(chunk)
if msg_size >= self.CHUNK_LIMIT:
# We've gone beyond the chunk limit, so send out our
# message. At worst this results in a message slightly
# larger than (2*CHUNK_LIMIT)-1
self._sendMessage(msg)
msg = {}
logdata = msg.setdefault(logname, [])
msg_size = 0
self.buflen = 0
if logdata:
self._sendMessage(msg)
if self.sendBuffersTimer:
if self.sendBuffersTimer.active():
self.sendBuffersTimer.cancel()
self.sendBuffersTimer = None
def _addToBuffers(self, logname, data):
"""
Add data to the buffer for logname
Start a timer to send the buffers if BUFFER_TIMEOUT elapses.
If adding data causes the buffer size to grow beyond BUFFER_SIZE, then
the buffers will be sent.
"""
n = len(data)
self.buflen += n
self.buffered.append((logname, data))
if self.buflen > self.BUFFER_SIZE:
self._sendBuffers()
elif not self.sendBuffersTimer:
self.sendBuffersTimer = self._reactor.callLater(
self.BUFFER_TIMEOUT, self._bufferTimeout)
def addStdout(self, data):
if self.sendStdout:
self._addToBuffers('stdout', data)
if self.keepStdout:
self.stdout += data
if self.ioTimeoutTimer:
self.ioTimeoutTimer.reset(self.timeout)
def addStderr(self, data):
if self.sendStderr:
self._addToBuffers('stderr', data)
if self.keepStderr:
self.stderr += data
if self.ioTimeoutTimer:
self.ioTimeoutTimer.reset(self.timeout)
def addLogfile(self, name, data):
self._addToBuffers(('log', name), data)
if self.ioTimeoutTimer:
self.ioTimeoutTimer.reset(self.timeout)
def finished(self, sig, rc):
self.elapsedTime = util.now(self._reactor) - self.startTime
log.msg("command finished with signal %s, exit code %s, elapsedTime: %0.6f" % (
sig, rc, self.elapsedTime))
for w in self.logFileWatchers:
# this will send the final updates
w.stop()
self._sendBuffers()
if sig is not None:
rc = -1
if self.sendRC:
if sig is not None:
self.sendStatus(
{'header': "process killed by signal %d\n" % sig})
self.sendStatus({'rc': rc})
self.sendStatus({'header': "elapsedTime=%0.6f\n" % self.elapsedTime})
self._cancelTimers()
d = self.deferred
self.deferred = None
if d:
d.callback(rc)
else:
log.msg("Hey, command %s finished twice" % self)
def failed(self, why):
self._sendBuffers()
log.msg("RunProcess.failed: command failed: %s" % (why,))
self._cancelTimers()
d = self.deferred
self.deferred = None
if d:
d.errback(why)
else:
log.msg("Hey, command %s finished twice" % self)
def doTimeout(self):
self.ioTimeoutTimer = None
msg = "command timed out: %d seconds without output running %s" % (
self.timeout, self.fake_command)
self.kill(msg)
def doMaxTimeout(self):
self.maxTimeoutTimer = None
msg = "command timed out: %d seconds elapsed running %s" % (
self.maxTime, self.fake_command)
self.kill(msg)
def isDead(self):
if self.process.pid is None:
return True
pid = int(self.process.pid)
try:
os.kill(pid, 0)
except OSError:
return True # dead
return False # alive
def checkProcess(self):
self.sigtermTimer = None
if not self.isDead():
hit = self.sendSig(self.interruptSignal)
else:
hit = 1
self.cleanUp(hit)
def cleanUp(self, hit):
if not hit:
log.msg("signalProcess/os.kill failed both times")
if runtime.platformType == "posix":
# we only do this under posix because the win32eventreactor
# blocks here until the process has terminated, while closing
# stderr. This is weird.
self.pp.transport.loseConnection()
if self.deferred:
# finished ought to be called momentarily. Just in case it doesn't,
# set a timer which will abandon the command.
self.killTimer = self._reactor.callLater(self.BACKUP_TIMEOUT,
self.doBackupTimeout)
def sendSig(self, interruptSignal):
hit = 0
# try signalling the process group
if not hit and self.useProcGroup and runtime.platformType == "posix":
sig = getattr(signal, "SIG" + interruptSignal, None)
if sig is None:
log.msg("signal module is missing SIG%s" % interruptSignal)
elif not hasattr(os, "kill"):
log.msg("os module is missing the 'kill' function")
elif self.process.pgid is None:
log.msg("self.process has no pgid")
else:
log.msg("trying to kill process group %d" %
(self.process.pgid,))
try:
os.kill(-self.process.pgid, sig)
log.msg(" signal %s sent successfully" % sig)
self.process.pgid = None
hit = 1
except OSError:
log.msg('failed to kill process group (ignored): %s' %
(sys.exc_info()[1],))
# probably no-such-process, maybe because there is no process
# group
pass
elif runtime.platformType == "win32":
if interruptSignal is None:
log.msg("interruptSignal==None, only pretending to kill child")
elif self.process.pid is not None:
if interruptSignal == "TERM":
log.msg("using TASKKILL PID /T to kill pid %s" %
self.process.pid)
subprocess.check_call(
"TASKKILL /PID %s /T" % self.process.pid)
log.msg("taskkill'd pid %s" % self.process.pid)
hit = 1
elif interruptSignal == "KILL":
log.msg("using TASKKILL PID /F /T to kill pid %s" %
self.process.pid)
subprocess.check_call(
"TASKKILL /F /PID %s /T" % self.process.pid)
log.msg("taskkill'd pid %s" % self.process.pid)
hit = 1
# try signalling the process itself (works on Windows too, sorta)
if not hit:
try:
log.msg("trying process.signalProcess('%s')" %
(interruptSignal,))
self.process.signalProcess(interruptSignal)
log.msg(" signal %s sent successfully" % (interruptSignal,))
hit = 1
except OSError:
log.err("from process.signalProcess:")
# could be no-such-process, because they finished very recently
pass
except error.ProcessExitedAlready:
log.msg("Process exited already - can't kill")
# the process has already exited, and likely finished() has
# been called already or will be called shortly
pass
return hit
def kill(self, msg):
# This may be called by the timeout, or when the user has decided to
# abort this build.
self._sendBuffers()
self._cancelTimers()
msg += ", attempting to kill"
log.msg(msg)
self.sendStatus({'header': "\n" + msg + "\n"})
# let the PP know that we are killing it, so that it can ensure that
# the exit status comes out right
self.pp.killed = True
sendSigterm = self.sigtermTime is not None
if sendSigterm:
self.sendSig("TERM")
self.sigtermTimer = self._reactor.callLater(
self.sigtermTime, self.checkProcess)
else:
hit = self.sendSig(self.interruptSignal)
self.cleanUp(hit)
def doBackupTimeout(self):
log.msg("we tried to kill the process, and it wouldn't die.."
" finish anyway")
self.killTimer = None
signalName = "SIG" + self.interruptSignal
self.sendStatus({'header': signalName + " failed to kill process\n"})
if self.sendRC:
self.sendStatus({'header': "using fake rc=-1\n"})
self.sendStatus({'rc': -1})
self.failed(RuntimeError(signalName + " failed to kill process"))
def _cancelTimers(self):
for timerName in ('ioTimeoutTimer', 'killTimer', 'maxTimeoutTimer', 'sendBuffersTimer', 'sigtermTimer'):
timer = getattr(self, timerName, None)
if timer:
timer.cancel()
setattr(self, timerName, None)

0
slave/buildslave/scripts/__init__.py
View File

37
slave/buildslave/scripts/base.py
View File

@@ -1,37 +0,0 @@
# This file is part of Buildbot. Buildbot is free software: you can
# redistribute it and/or modify it under the terms of the GNU General Public
# License as published by the Free Software Foundation, version 2.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 51
# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
# Copyright Buildbot Team Members
from __future__ import print_function
import os
def isBuildslaveDir(dir):
def print_error(error_message):
print("%s\ninvalid buildslave directory '%s'" % (error_message, dir))
buildbot_tac = os.path.join(dir, "buildbot.tac")
try:
contents = open(buildbot_tac).read()
except IOError as exception:
print_error("error reading '%s': %s" %
(buildbot_tac, exception.strerror))
return False
if "Application('buildslave')" not in contents:
print_error("unexpected content in '%s'" % buildbot_tac)
return False
return True

230
slave/buildslave/scripts/create_slave.py
View File

@@ -1,230 +0,0 @@
# This file is part of Buildbot. Buildbot is free software: you can
# redistribute it and/or modify it under the terms of the GNU General Public
# License as published by the Free Software Foundation, version 2.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 51
# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
# Copyright Buildbot Team Members
from __future__ import print_function
import os
slaveTACTemplate = ["""
import os
from buildslave.bot import BuildSlave
from twisted.application import service
basedir = %(basedir)r
rotateLength = %(log-size)d
maxRotatedFiles = %(log-count)s
# if this is a relocatable tac file, get the directory containing the TAC
if basedir == '.':
import os.path
basedir = os.path.abspath(os.path.dirname(__file__))
# note: this line is matched against to check that this is a buildslave
# directory; do not edit it.
application = service.Application('buildslave')
""",
"""
try:
from twisted.python.logfile import LogFile
from twisted.python.log import ILogObserver, FileLogObserver
logfile = LogFile.fromFullPath(os.path.join(basedir, "twistd.log"), rotateLength=rotateLength,
maxRotatedFiles=maxRotatedFiles)
application.setComponent(ILogObserver, FileLogObserver(logfile).emit)
except ImportError:
# probably not yet twisted 8.2.0 and beyond, can't set log yet
pass
""",
"""
buildmaster_host = %(host)r
port = %(port)d
slavename = %(name)r
passwd = %(passwd)r
if "SLAVEPASS" in os.environ:
del os.environ['SLAVEPASS']
keepalive = %(keepalive)d
usepty = %(usepty)d
umask = %(umask)s
maxdelay = %(maxdelay)d
numcpus = %(numcpus)s
allow_shutdown = %(allow-shutdown)s
s = BuildSlave(buildmaster_host, port, slavename, passwd, basedir,
keepalive, usepty, umask=umask, maxdelay=maxdelay,
numcpus=numcpus, allow_shutdown=allow_shutdown)
s.setServiceParent(application)
"""]
class CreateSlaveError(Exception):
"""
Raised on errors while setting up buildslave directory.
"""
def _makeBaseDir(basedir, quiet):
"""
Make buildslave base directory if needed.
@param basedir: buildslave base directory relative path
@param quiet: if True, don't print info messages
@raise CreateSlaveError: on error making base directory
"""
if os.path.exists(basedir):
if not quiet:
print("updating existing installation")
return
if not quiet:
print("mkdir", basedir)
try:
os.mkdir(basedir)
except OSError as exception:
raise CreateSlaveError("error creating directory %s: %s" %
(basedir, exception.strerror))
def _makeBuildbotTac(basedir, tac_file_contents, quiet):
"""
Create buildbot.tac file. If buildbot.tac file already exists with
different contents, create buildbot.tac.new instead.
@param basedir: buildslave base directory relative path
@param tac_file_contents: contents of buildbot.tac file to write
@param quiet: if True, don't print info messages
@raise CreateSlaveError: on error reading or writing tac file
"""
tacfile = os.path.join(basedir, "buildbot.tac")
if os.path.exists(tacfile):
try:
oldcontents = open(tacfile, "rt").read()
except IOError as exception:
raise CreateSlaveError("error reading %s: %s" %
(tacfile, exception.strerror))
if oldcontents == tac_file_contents:
if not quiet:
print("buildbot.tac already exists and is correct")
return
if not quiet:
print("not touching existing buildbot.tac")
print("creating buildbot.tac.new instead")
tacfile = os.path.join(basedir, "buildbot.tac.new")
try:
f = open(tacfile, "wt")
f.write(tac_file_contents)
f.close()
os.chmod(tacfile, 0o600)
except IOError as exception:
raise CreateSlaveError("could not write %s: %s" %
(tacfile, exception.strerror))
def _makeInfoFiles(basedir, quiet):
"""
Create info/* files inside basedir.
@param basedir: buildslave base directory relative path
@param quiet: if True, don't print info messages
@raise CreateSlaveError: on error making info directory or
writing info files
"""
def createFile(path, file, contents):
filepath = os.path.join(path, file)
if os.path.exists(filepath):
return False
if not quiet:
print("Creating %s, you need to edit it appropriately." %
os.path.join("info", file))
try:
open(filepath, "wt").write(contents)
except IOError as exception:
raise CreateSlaveError("could not write %s: %s" %
(filepath, exception.strerror))
return True
path = os.path.join(basedir, "info")
if not os.path.exists(path):
if not quiet:
print("mkdir", path)
try:
os.mkdir(path)
except OSError as exception:
raise CreateSlaveError("error creating directory %s: %s" %
(path, exception.strerror))
# create 'info/admin' file
created = createFile(path, "admin",
"Your Name Here <admin@youraddress.invalid>\n")
# create 'info/host' file
created = createFile(path, "host",
"Please put a description of this build host here\n")
access_uri = os.path.join(path, "access_uri")
if not os.path.exists(access_uri):
if not quiet:
print("Not creating %s - add it if you wish" %
os.path.join("info", "access_uri"))
if created and not quiet:
print("Please edit the files in %s appropriately." % path)
def createSlave(config):
basedir = config['basedir']
quiet = config['quiet']
if config['relocatable']:
config['basedir'] = '.'
asd = config['allow-shutdown']
if asd:
config['allow-shutdown'] = repr(asd)
if config['no-logrotate']:
slaveTAC = "".join([slaveTACTemplate[0]] + slaveTACTemplate[2:])
else:
slaveTAC = "".join(slaveTACTemplate)
contents = slaveTAC % config
try:
_makeBaseDir(basedir, quiet)
_makeBuildbotTac(basedir, contents, quiet)
_makeInfoFiles(basedir, quiet)
except CreateSlaveError as exception:
print("%s\nfailed to configure buildslave in %s" %
(exception, config['basedir']))
return 1
if not quiet:
print("buildslave configured in %s" % basedir)
return 0

137
slave/buildslave/scripts/logwatcher.py
View File

@@ -1,137 +0,0 @@
# This file is part of Buildbot. Buildbot is free software: you can
# redistribute it and/or modify it under the terms of the GNU General Public
# License as published by the Free Software Foundation, version 2.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 51
# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
# Copyright Buildbot Team Members
from __future__ import print_function
import os
import platform
from twisted.internet import defer
from twisted.internet import error
from twisted.internet import protocol
from twisted.internet import reactor
from twisted.protocols.basic import LineOnlyReceiver
from twisted.python.failure import Failure
class FakeTransport(object):
disconnecting = False
class BuildmasterTimeoutError(Exception):
pass
class BuildslaveTimeoutError(Exception):
pass
class ReconfigError(Exception):
pass
class BuildSlaveDetectedError(Exception):
pass
class TailProcess(protocol.ProcessProtocol):
def outReceived(self, data):
self.lw.dataReceived(data)
def errReceived(self, data):
print("ERR: '%s'" % (data,))
class LogWatcher(LineOnlyReceiver):
POLL_INTERVAL = 0.1
TIMEOUT_DELAY = 10.0
delimiter = os.linesep
def __init__(self, logfile):
self.logfile = logfile
self.in_reconfig = False
self.transport = FakeTransport()
self.pp = TailProcess()
self.pp.lw = self
self.processtype = "buildmaster"
self.timer = None
def start(self):
# If the log file doesn't exist, create it now.
if not os.path.exists(self.logfile):
open(self.logfile, 'a').close()
# return a Deferred that fires when the reconfig process has
# finished. It errbacks with TimeoutError if the finish line has not
# been seen within 10 seconds, and with ReconfigError if the error
# line was seen. If the logfile could not be opened, it errbacks with
# an IOError.
if platform.system().lower() == 'sunos' and os.path.exists('/usr/xpg4/bin/tail'):
tailBin = "/usr/xpg4/bin/tail"
else:
tailBin = "/usr/bin/tail"
self.p = reactor.spawnProcess(self.pp, tailBin,
("tail", "-f", "-n", "0", self.logfile),
env=os.environ,
)
self.running = True
d = defer.maybeDeferred(self._start)
return d
def _start(self):
self.d = defer.Deferred()
self.timer = reactor.callLater(self.TIMEOUT_DELAY, self.timeout)
return self.d
def timeout(self):
self.timer = None
if self.processtype == "buildmaster":
e = BuildmasterTimeoutError()
else:
e = BuildslaveTimeoutError()
self.finished(Failure(e))
def finished(self, results):
try:
self.p.signalProcess("KILL")
except error.ProcessExitedAlready:
pass
if self.timer:
self.timer.cancel()
self.timer = None
self.running = False
self.in_reconfig = False
self.d.callback(results)
def lineReceived(self, line):
if not self.running:
return
if "Log opened." in line:
self.in_reconfig = True
if "loading configuration from" in line:
self.in_reconfig = True
if "Creating BuildSlave" in line:
self.processtype = "buildslave"
if self.in_reconfig:
print(line)
if "message from master: attached" in line:
return self.finished("buildslave")
if "I will keep using the previous config file" in line:
return self.finished(Failure(ReconfigError()))
if "configuration update complete" in line:
return self.finished("buildmaster")

38
slave/buildslave/scripts/restart.py
View File

@@ -1,38 +0,0 @@
# This file is part of Buildbot. Buildbot is free software: you can
# redistribute it and/or modify it under the terms of the GNU General Public
# License as published by the Free Software Foundation, version 2.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 51
# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
# Copyright Buildbot Team Members
from __future__ import print_function
from buildslave.scripts import base
from buildslave.scripts import start
from buildslave.scripts import stop
def restart(config):
quiet = config['quiet']
basedir = config['basedir']
if not base.isBuildslaveDir(basedir):
return 1
try:
stop.stopSlave(basedir, quiet)
except stop.SlaveNotRunning:
if not quiet:
print("no old buildslave process found to stop")
if not quiet:
print("now restarting buildslave process..")
return start.startSlave(basedir, quiet, config['nodaemon'])

273
slave/buildslave/scripts/runner.py
View File

@@ -1,273 +0,0 @@
# This file is part of Buildbot. Buildbot is free software: you can
# redistribute it and/or modify it under the terms of the GNU General Public
# License as published by the Free Software Foundation, version 2.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 51
# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
# Copyright Buildbot Team Members
# N.B.: don't import anything that might pull in a reactor yet. Some of our
# subcommands want to load modules that need the gtk reactor.
from __future__ import print_function
import os
import re
import sys
from twisted.python import reflect
from twisted.python import usage
# the create/start/stop commands should all be run as the same user,
# preferably a separate 'buildbot' account.
# Note that the terms 'options' and 'config' are used interchangeably here - in
# fact, they are interchanged several times. Caveat legator.
class MakerBase(usage.Options):
optFlags = [
['help', 'h', "Display this message"],
["quiet", "q", "Do not emit the commands being run"],
]
longdesc = """
Operates upon the specified <basedir> (or the current directory, if not
specified).
"""
# on tab completion, suggest directories as first argument
if hasattr(usage, 'Completions'):
# only set completion suggestion if running with
# twisted version (>=11.1.0) that supports it
compData = usage.Completions(
extraActions=[usage.CompleteDirs(descr="slave base directory")])
opt_h = usage.Options.opt_help
def parseArgs(self, *args):
if len(args) > 0:
self['basedir'] = args[0]
else:
# Use the current directory if no basedir was specified.
self['basedir'] = os.getcwd()
if len(args) > 1:
raise usage.UsageError("I wasn't expecting so many arguments")
def postOptions(self):
self['basedir'] = os.path.abspath(self['basedir'])
class StartOptions(MakerBase):
subcommandFunction = "buildslave.scripts.start.startCommand"
optFlags = [
['quiet', 'q', "Don't display startup log messages"],
['nodaemon', None, "Don't daemonize (stay in foreground)"],
]
def getSynopsis(self):
return "Usage: buildslave start [<basedir>]"
class StopOptions(MakerBase):
subcommandFunction = "buildslave.scripts.stop.stop"
def getSynopsis(self):
return "Usage: buildslave stop [<basedir>]"
class RestartOptions(MakerBase):
subcommandFunction = "buildslave.scripts.restart.restart"
optFlags = [
['quiet', 'q', "Don't display startup log messages"],
['nodaemon', None, "Don't daemonize (stay in foreground)"],
]
def getSynopsis(self):
return "Usage: buildslave restart [<basedir>]"
class UpgradeSlaveOptions(MakerBase):
subcommandFunction = "buildslave.scripts.upgrade_slave.upgradeSlave"
optFlags = [
]
optParameters = [
]
def getSynopsis(self):
return "Usage: buildslave upgrade-slave [<basedir>]"
longdesc = """
This command takes an existing buildslave working directory and
upgrades it to the current version.
"""
class CreateSlaveOptions(MakerBase):
subcommandFunction = "buildslave.scripts.create_slave.createSlave"
optFlags = [
["force", "f", "Re-use an existing directory"],
["relocatable", "r",
"Create a relocatable buildbot.tac"],
["no-logrotate", "n",
"Do not permit buildmaster rotate logs by itself"]
]
optParameters = [
["keepalive", "k", 600,
"Interval at which keepalives should be sent (in seconds)"],
["usepty", None, 0,
"(1 or 0) child processes should be run in a pty (default 0)"],
["umask", None, "None",
"controls permissions of generated files. "
"Use --umask=022 to be world-readable"],
["maxdelay", None, 300,
"Maximum time between connection attempts"],
["numcpus", None, "None",
"Number of available cpus to use on a build. "],
["log-size", "s", "10000000",
"size at which to rotate twisted log files"],
["log-count", "l", "10",
"limit the number of kept old twisted log files "
"(None for unlimited)"],
["allow-shutdown", "a", None,
"Allows the buildslave to initiate a graceful shutdown. One of "
"'signal' or 'file'"]
]
longdesc = """
This command creates a buildslave working directory and buildbot.tac
file. The bot will use the <name> and <passwd> arguments to authenticate
itself when connecting to the master. All commands are run in a
build-specific subdirectory of <basedir>. <master> is a string of the
form 'hostname[:port]', and specifies where the buildmaster can be reached.
port defaults to 9989
The appropriate values for <name>, <passwd>, and <master> should be
provided to you by the buildmaster administrator. You must choose <basedir>
yourself.
"""
def validateMasterArgument(self, master_arg):
"""
Parse the <master> argument.
@param master_arg: the <master> argument to parse
@return: tuple of master's host and port
@raise UsageError: on errors parsing the argument
"""
if master_arg[:5] == "http:":
raise usage.UsageError("<master> is not a URL - do not use URL")
if ":" not in master_arg:
master = master_arg
port = 9989
else:
master, port = master_arg.split(":")
if len(master) < 1:
raise usage.UsageError("invalid <master> argument '%s'" %
master_arg)
try:
port = int(port)
except ValueError:
raise usage.UsageError("invalid master port '%s', "
"needs to be an number" % port)
return master, port
def getSynopsis(self):
return "Usage: buildslave create-slave " \
"[options] <basedir> <master> <name> <passwd>"
def parseArgs(self, *args):
if len(args) != 4:
raise usage.UsageError("incorrect number of arguments")
basedir, master, name, passwd = args
self['basedir'] = basedir
self['host'], self['port'] = self.validateMasterArgument(master)
self['name'] = name
self['passwd'] = passwd
def postOptions(self):
MakerBase.postOptions(self)
# check and convert numeric parameters
for argument in ["usepty", "keepalive", "maxdelay", "log-size"]:
try:
self[argument] = int(self[argument])
except ValueError:
raise usage.UsageError("%s parameter needs to be an number"
% argument)
if not re.match(r'^\d+$', self['log-count']) and \
self['log-count'] != 'None':
raise usage.UsageError("log-count parameter needs to be an number"
" or None")
if not re.match(r'^\d+$', self['umask']) and \
self['umask'] != 'None':
raise usage.UsageError("umask parameter needs to be an number"
" or None")
if not re.match(r'^\d+$', self['numcpus']) and \
self['numcpus'] != 'None':
raise usage.UsageError("numcpus parameter needs to be an number"
" or None")
if self['allow-shutdown'] not in [None, 'signal', 'file']:
raise usage.UsageError("allow-shutdown needs to be one of"
" 'signal' or 'file'")
class Options(usage.Options):
synopsis = "Usage: buildslave <command> [command options]"
subCommands = [
# the following are all admin commands
['create-slave', None, CreateSlaveOptions,
"Create and populate a directory for a new buildslave"],
['upgrade-slave', None, UpgradeSlaveOptions,
"Upgrade an existing buildslave directory for the current version"],
['start', None, StartOptions, "Start a buildslave"],
['stop', None, StopOptions, "Stop a buildslave"],
['restart', None, RestartOptions,
"Restart a buildslave"],
]
def opt_version(self):
import buildslave
print("Buildslave version: %s" % buildslave.version)
usage.Options.opt_version(self)
def opt_verbose(self):
from twisted.python import log
log.startLogging(sys.stderr)
def postOptions(self):
if not hasattr(self, 'subOptions'):
raise usage.UsageError("must specify a command")
def run():
config = Options()
try:
config.parseOptions()
except usage.error as e:
print("%s: %s" % (sys.argv[0], e))
print()
c = getattr(config, 'subOptions', config)
print(str(c))
sys.exit(1)
subconfig = config.subOptions
subcommandFunction = reflect.namedObject(subconfig.subcommandFunction)
sys.exit(subcommandFunction(subconfig))

161
slave/buildslave/scripts/start.py
View File

@@ -1,161 +0,0 @@
# This file is part of Buildbot. Buildbot is free software: you can
# redistribute it and/or modify it under the terms of the GNU General Public
# License as published by the Free Software Foundation, version 2.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 51
# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
# Copyright Buildbot Team Members
import os
import sys
import time
from buildslave.scripts import base
class Follower(object):
def follow(self):
from twisted.internet import reactor
from buildslave.scripts.logwatcher import LogWatcher
self.rc = 0
print("Following twistd.log until startup finished..")
lw = LogWatcher("twistd.log")
d = lw.start()
d.addCallbacks(self._success, self._failure)
reactor.run()
return self.rc
def _success(self, processtype):
from twisted.internet import reactor
print("The %s appears to have (re)started correctly." % processtype)
self.rc = 0
reactor.stop()
def _failure(self, why):
from twisted.internet import reactor
from buildslave.scripts.logwatcher import BuildmasterTimeoutError, \
ReconfigError, BuildslaveTimeoutError, BuildSlaveDetectedError
if why.check(BuildmasterTimeoutError):
print("""
The buildslave took more than 10 seconds to start, so we were unable to
confirm that it started correctly. Please 'tail twistd.log' and look for a
line that says 'configuration update complete' to verify correct startup.
""")
elif why.check(BuildslaveTimeoutError):
print("""
The buildslave took more than 10 seconds to start and/or connect to the
buildslave, so we were unable to confirm that it started and connected
correctly. Please 'tail twistd.log' and look for a line that says 'message
from master: attached' to verify correct startup. If you see a bunch of
messages like 'will retry in 6 seconds', your buildslave might not have the
correct hostname or portnumber for the buildslave, or the buildslave might
not be running. If you see messages like
'Failure: twisted.cred.error.UnauthorizedLogin'
then your buildslave might be using the wrong botname or password. Please
correct these problems and then restart the buildslave.
""")
elif why.check(ReconfigError):
print("""
The buildslave appears to have encountered an error in the master.cfg config
file during startup. It is probably running with an empty configuration right
now. Please inspect and fix master.cfg, then restart the buildslave.
""")
elif why.check(BuildSlaveDetectedError):
print("""
Buildslave is starting up, not following logfile.
""")
else:
print("""
Unable to confirm that the buildslave started correctly. You may need to
stop it, fix the config file, and restart.
""")
print(why)
self.rc = 1
reactor.stop()
def startCommand(config):
basedir = config['basedir']
if not base.isBuildslaveDir(basedir):
return 1
return startSlave(basedir, config['quiet'], config['nodaemon'])
def startSlave(basedir, quiet, nodaemon):
"""
Start slave process.
Fork and start twisted application described in basedir buildbot.tac file.
Print it's log messages to stdout for a while and try to figure out if
start was successful.
If quiet or nodaemon parameters are True, or we are running on a win32
system, will not fork and log will not be printed to stdout.
@param basedir: buildslave's basedir path
@param quiet: don't display startup log messages
@param nodaemon: don't daemonize (stay in foreground)
@return: 0 if slave was successfully started,
1 if we are not sure that slave started successfully
"""
os.chdir(basedir)
if quiet or nodaemon:
return launch(nodaemon)
# we probably can't do this os.fork under windows
from twisted.python.runtime import platformType
if platformType == "win32":
return launch(nodaemon)
# fork a child to launch the daemon, while the parent process tails the
# logfile
if os.fork():
# this is the parent
rc = Follower().follow()
return rc
# this is the child: give the logfile-watching parent a chance to start
# watching it before we start the daemon
time.sleep(0.2)
launch(nodaemon)
def launch(nodaemon):
sys.path.insert(0, os.path.abspath(os.getcwd()))
# see if we can launch the application without actually having to
# spawn twistd, since spawning processes correctly is a real hassle
# on windows.
from twisted.python.runtime import platformType
argv = ["twistd",
"--no_save",
"--logfile=twistd.log", # windows doesn't use the same default
"--python=buildbot.tac"]
if nodaemon:
argv.extend(['--nodaemon'])
sys.argv = argv
# this is copied from bin/twistd. twisted-2.0.0 through 2.4.0 use
# _twistw.run . Twisted-2.5.0 and later use twistd.run, even for
# windows.
from twisted import __version__
major, minor, ignored = __version__.split(".", 2)
major = int(major)
minor = int(minor)
if (platformType == "win32" and (major == 2 and minor < 5)):
from twisted.scripts import _twistw
run = _twistw.run
else:
from twisted.scripts import twistd
run = twistd.run
run()

89
slave/buildslave/scripts/stop.py
View File

@@ -1,89 +0,0 @@
# This file is part of Buildbot. Buildbot is free software: you can
# redistribute it and/or modify it under the terms of the GNU General Public
# License as published by the Free Software Foundation, version 2.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 51
# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
# Copyright Buildbot Team Members
from __future__ import print_function
import os
import time
from buildslave.scripts import base
class SlaveNotRunning(Exception):
"""
raised when trying to stop slave process that is not running
"""
def stopSlave(basedir, quiet, signame="TERM"):
"""
Stop slave process by sending it a signal.
Using the specified basedir path, read slave process's pid file and
try to terminate that process with specified signal.
@param basedir: buildslave's basedir path
@param quite: if False, don't print any messages to stdout
@param signame: signal to send to the slave process
@raise SlaveNotRunning: if slave pid file is not found
"""
import signal
os.chdir(basedir)
try:
f = open("twistd.pid", "rt")
except IOError:
raise SlaveNotRunning()
pid = int(f.read().strip())
signum = getattr(signal, "SIG" + signame)
timer = 0
try:
os.kill(pid, signum)
except OSError as e:
if e.errno != 3:
raise
time.sleep(0.1)
while timer < 10:
# poll once per second until twistd.pid goes away, up to 10 seconds
try:
os.kill(pid, 0)
except OSError:
if not quiet:
print("buildslave process %d is dead" % pid)
return
timer += 1
time.sleep(1)
if not quiet:
print("never saw process go away")
def stop(config, signame="TERM"):
quiet = config['quiet']
basedir = config['basedir']
if not base.isBuildslaveDir(basedir):
return 1
try:
stopSlave(basedir, quiet, signame)
except SlaveNotRunning:
if not quiet:
print("buildslave not running")
return 0

39
slave/buildslave/scripts/upgrade_slave.py
View File

@@ -1,39 +0,0 @@
# This file is part of Buildbot. Buildbot is free software: you can
# redistribute it and/or modify it under the terms of the GNU General Public
# License as published by the Free Software Foundation, version 2.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 51
# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
# Copyright Buildbot Team Members
from __future__ import print_function
import os
from buildslave.scripts import base
def upgradeSlave(config):
basedir = os.path.expanduser(config['basedir'])
if not base.isBuildslaveDir(basedir):
return 1
buildbot_tac = open(os.path.join(basedir, "buildbot.tac")).read()
new_buildbot_tac = buildbot_tac.replace(
"from buildbot.slave.bot import BuildSlave",
"from buildslave.bot import BuildSlave")
if new_buildbot_tac != buildbot_tac:
open(os.path.join(basedir, "buildbot.tac"), "w").write(new_buildbot_tac)
print("buildbot.tac updated")
else:
print("No changes made")
return 0

65
slave/buildslave/test/__init__.py
View File

@@ -1,65 +0,0 @@
# This file is part of Buildbot. Buildbot is free software: you can
# redistribute it and/or modify it under the terms of the GNU General Public
# License as published by the Free Software Foundation, version 2.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 51
# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
# Copyright Buildbot Team Members
import sys
import twisted
from buildslave import monkeypatches
from twisted.trial import unittest
# apply the same patches the slave does when it starts
monkeypatches.patch_all(for_tests=True)
def add_debugging_monkeypatches():
"""
DO NOT CALL THIS DIRECTLY
This adds a few "harmless" monkeypatches which make it easier to debug
failing tests.
"""
from twisted.application.service import Service
old_startService = Service.startService
old_stopService = Service.stopService
def startService(self):
assert not self.running
return old_startService(self)
def stopService(self):
assert self.running
return old_stopService(self)
Service.startService = startService
Service.stopService = stopService
# versions of Twisted before 9.0.0 did not have a UnitTest.patch that worked
# on Python-2.7
if twisted.version.major <= 9 and sys.version_info[:2] == (2, 7):
def nopatch(self, *args):
raise unittest.SkipTest('unittest.TestCase.patch is not available')
unittest.TestCase.patch = nopatch
add_debugging_monkeypatches()
__all__ = []
# import mock so we bail out early if it's not installed
try:
import mock
mock = mock
except ImportError:
raise ImportError("Buildbot tests require the 'mock' module; "
"try 'pip install mock'")

0
slave/buildslave/test/fake/__init__.py
View File

37
slave/buildslave/test/fake/remote.py
View File

@@ -1,37 +0,0 @@
# This file is part of Buildbot. Buildbot is free software: you can
# redistribute it and/or modify it under the terms of the GNU General Public
# License as published by the Free Software Foundation, version 2.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 51
# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
# Copyright Buildbot Team Members
from twisted.internet import defer
class FakeRemote(object):
"""
Wrap a local object to make it look like it's remote
"""
def __init__(self, original, method_prefix="remote_"):
self.original = original
self.method_prefix = method_prefix
def callRemote(self, meth, *args, **kwargs):
fn = getattr(self.original, self.method_prefix + meth)
return defer.maybeDeferred(fn, *args, **kwargs)
def notifyOnDisconnect(self, what):
pass
def dontNotifyOnDisconnect(self, what):
pass

187
slave/buildslave/test/fake/runprocess.py
View File

@@ -1,187 +0,0 @@
# This file is part of Buildbot. Buildbot is free software: you can
# redistribute it and/or modify it under the terms of the GNU General Public
# License as published by the Free Software Foundation, version 2.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 51
# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
# Copyright Buildbot Team Members
from twisted.internet import defer
from twisted.python import failure
class Expect(object):
"""
An expected instantiation of RunProcess. Usually used within a RunProcess
expect invocation:
rp.expect(
Expect("echo", "bar", usePTY=False)
+ { 'stdout' : 'hello!!' }
+ { 'rc' : 13 }
+ 13 # for a callback with rc=13; or
+ Failure(..), # for a failure
Expect(..) + .. ,
...
)
Note that the default values are accepted for all keyword arguments if they
are not omitted.
"""
def __init__(self, command, workdir, **kwargs):
self.kwargs = dict(command=command, workdir=workdir)
self.kwargs.update(kwargs)
self.result = None
self.status_updates = []
def __add__(self, other):
if isinstance(other, dict):
self.status_updates.append(other)
elif isinstance(other, int):
self.result = ('c', other)
elif isinstance(other, failure.Failure):
self.result = ('e', other)
else:
raise ValueError("invalid expectation '%r'" % (other,))
return self
def __str__(self):
other_kwargs = self.kwargs.copy()
del other_kwargs['command']
del other_kwargs['workdir']
return "Command: %s\n workdir: %s\n kwargs: %s\n result: %s\n" % (
self.kwargs['command'], self.kwargs['workdir'],
other_kwargs, self.result)
class FakeRunProcess(object):
"""
A fake version of L{buildslave.runprocess.RunProcess} which will
simulate running external processes without actually running them (which is
very fragile in tests!)
This class is first programmed with the set of instances that are expected,
and with their expected results. It will raise an AssertionError if the
expected behavior is not seen.
Note that this handles sendStderr/sendStdout and keepStderr/keepStdout properly.
"""
@classmethod
def expect(cls, *expectations):
"""
Set the expectations for this test run
"""
cls._expectations = list(expectations)
# list the first expectation last, so we can pop it
cls._expectations.reverse()
@classmethod
def test_done(cls):
"""
Indicate that this test is finished; if any expected instantiations
have not taken place, this will raise the appropriate AssertionError.
"""
if cls._expectations:
raise AssertionError("%d expected instances not created" % len(cls._expectations))
del cls._expectations
def __init__(self, builder, command, workdir, **kwargs):
kwargs['command'] = command
kwargs['workdir'] = workdir
# the default values for the constructor kwargs; if we got a default
# value in **kwargs and didn't expect anything, well count that as OK
default_values = dict(environ=None,
sendStdout=True, sendStderr=True, sendRC=True,
timeout=None, maxTime=None, sigtermTime=None, initialStdin=None,
keepStdout=False, keepStderr=False,
logEnviron=True, logfiles={}, usePTY="slave-config")
if not self._expectations:
raise AssertionError("unexpected instantiation: %s" % (kwargs,))
exp = self._exp = self._expectations.pop()
if exp.kwargs != kwargs:
msg = []
for key in sorted(list(set(exp.kwargs.keys()) | set(kwargs.keys()))):
if key not in exp.kwargs:
if key in default_values:
if default_values[key] == kwargs[key]:
continue # default values are expected
msg.append('%s: expected default (%r),\n got %r' %
(key, default_values[key], kwargs[key]))
else:
msg.append('%s: unexpected arg, value = %r' % (key, kwargs[key]))
elif key not in kwargs:
msg.append('%s: did not get expected arg' % (key,))
elif exp.kwargs[key] != kwargs[key]:
msg.append('%s: expected %r,\n got %r' % (key, exp.kwargs[key], kwargs[key]))
if msg:
msg.insert(0, 'did not get expected __init__ arguments for\n '
+ " ".join(map(repr, kwargs.get('command', ['unknown command']))))
self._expectations[:] = [] # don't expect any more instances, since we're failing
raise AssertionError("\n".join(msg))
self._builder = builder
self.stdout = ''
self.stderr = ''
def start(self):
# figure out the stdio-related parameters
keepStdout = self._exp.kwargs.get('keepStdout', False)
keepStderr = self._exp.kwargs.get('keepStderr', False)
sendStdout = self._exp.kwargs.get('sendStdout', True)
sendStderr = self._exp.kwargs.get('sendStderr', True)
if keepStdout:
self.stdout = ''
if keepStderr:
self.stderr = ''
finish_immediately = True
# send the updates, accounting for the stdio parameters
for upd in self._exp.status_updates:
if 'stdout' in upd:
if keepStdout:
self.stdout += upd['stdout']
if not sendStdout:
del upd['stdout']
if 'stderr' in upd:
if keepStderr:
self.stderr += upd['stderr']
if not sendStderr:
del upd['stderr']
if 'wait' in upd:
finish_immediately = False
continue # don't send this update
if not upd:
continue
self._builder.sendUpdate(upd)
d = self.run_deferred = defer.Deferred()
if finish_immediately:
self._finished()
return d
def _finished(self):
if self._exp.result[0] == 'e':
self.run_deferred.errback(self._exp.result[1])
else:
self.run_deferred.callback(self._exp.result[1])
def kill(self, reason):
self._builder.sendUpdate({'hdr': 'killing'})
self._builder.sendUpdate({'rc': -1})
self.run_deferred.callback(-1)

42
slave/buildslave/test/fake/slavebuilder.py
View File

@@ -1,42 +0,0 @@
# This file is part of Buildbot. Buildbot is free software: you can
# redistribute it and/or modify it under the terms of the GNU General Public
# License as published by the Free Software Foundation, version 2.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 51
# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
# Copyright Buildbot Team Members
from __future__ import print_function
import pprint
class FakeSlaveBuilder(object):
"""
Simulates a SlaveBuilder, but just records the updates from sendUpdate
in its updates attribute. Call show() to get a pretty-printed string
showing the updates. Set debug to True to show updates as they happen.
"""
debug = False
def __init__(self, usePTY=False, basedir="/slavebuilder/basedir"):
self.updates = []
self.basedir = basedir
self.usePTY = usePTY
self.unicode_encoding = 'utf-8'
def sendUpdate(self, data):
if self.debug:
print("FakeSlaveBuilder.sendUpdate", data)
self.updates.append(data)
def show(self):
return pprint.pformat(self.updates)

23
slave/buildslave/test/test_extra_coverage.py
View File

@@ -1,23 +0,0 @@
# This file is part of Buildbot. Buildbot is free software: you can
# redistribute it and/or modify it under the terms of the GNU General Public
# License as published by the Free Software Foundation, version 2.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 51
# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
# Copyright Buildbot Team Members
# this file imports a number of source files that are not
# included in the coverage because none of the tests import
# them; this results in a more accurate total coverage percent.
from buildslave.scripts import logwatcher
modules = [] # for the benefit of pyflakes
modules.extend([logwatcher])

0
slave/buildslave/test/unit/__init__.py
View File

121
slave/buildslave/test/unit/runprocess-scripts.py
View File

@@ -1,121 +0,0 @@
# This file is part of Buildbot. Buildbot is free software: you can
# redistribute it and/or modify it under the terms of the GNU General Public
# License as published by the Free Software Foundation, version 2.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 51
# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
# Copyright Buildbot Team Members
# This file contains scripts run by the test_runprocess tests. Note that since
# this code runs in a different Python interpreter, it does not necessarily
# have access to any of the Buildbot source. Functions here should be kept
# very simple!
import os
import select
import signal
import sys
import time
# utils
def write_pidfile(pidfile):
pidfile_tmp = pidfile + "~"
f = open(pidfile_tmp, "w")
f.write(str(os.getpid()))
f.close()
os.rename(pidfile_tmp, pidfile)
def sleep_forever():
signal.alarm(110) # die after 110 seconds
while True:
time.sleep(10)
def wait_for_parent_death(orig_parent_pid):
while True:
ppid = os.getppid()
if ppid != orig_parent_pid:
return
# on some systems, getppid will keep returning
# a dead pid, so check it for liveness
try:
os.kill(ppid, 0)
except OSError: # Probably ENOSUCH
return
script_fns = {}
def script(fn):
script_fns[fn.__name__] = fn
return fn
# scripts
@script
def write_pidfile_and_sleep():
pidfile = sys.argv[2]
write_pidfile(pidfile)
sleep_forever()
@script
def spawn_child():
parent_pidfile, child_pidfile = sys.argv[2:]
if os.fork() == 0:
write_pidfile(child_pidfile)
else:
write_pidfile(parent_pidfile)
sleep_forever()
@script
def double_fork():
# when using a PTY, the child process will get SIGHUP when the
# parent process exits, so ignore that.
signal.signal(signal.SIGHUP, signal.SIG_IGN)
parent_pidfile, child_pidfile = sys.argv[2:]
parent_pid = os.getpid()
if os.fork() == 0:
wait_for_parent_death(parent_pid)
write_pidfile(child_pidfile)
sleep_forever()
else:
write_pidfile(parent_pidfile)
sys.exit(0)
@script
def assert_stdin_closed():
# EOF counts as readable data, so we should see stdin in the readable list,
# although it may not appear immediately, and select may return early
bail_at = time.time() + 10
while True:
r, w, x = select.select([0], [], [], 0.01)
if r == [0]:
return # succcess!
if time.time() > bail_at:
assert False # failure :(
# make sure this process dies if necessary
if not hasattr(signal, 'alarm'):
signal.alarm = lambda t: None
signal.alarm(110) # die after 110 seconds
# dispatcher
script_fns[sys.argv[1]]()

446
slave/buildslave/test/unit/test_bot.py
View File

@@ -1,446 +0,0 @@
# This file is part of Buildbot. Buildbot is free software: you can
# redistribute it and/or modify it under the terms of the GNU General Public
# License as published by the Free Software Foundation, version 2.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 51
# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
# Copyright Buildbot Team Members
import multiprocessing
import os
import shutil
from builtins import range
import mock
from twisted.internet import defer
from twisted.internet import reactor
from twisted.internet import task
from twisted.python import failure
from twisted.python import log
from twisted.trial import unittest
import buildslave
from buildslave import base
from buildslave import pb
from buildslave.test.fake.remote import FakeRemote
from buildslave.test.fake.runprocess import Expect
from buildslave.test.util import command
from buildslave.test.util import compat
class TestBot(unittest.TestCase):
def setUp(self):
self.basedir = os.path.abspath("basedir")
if os.path.exists(self.basedir):
shutil.rmtree(self.basedir)
os.makedirs(self.basedir)
self.real_bot = base.BotBase(self.basedir, False)
self.real_bot.startService()
self.bot = FakeRemote(self.real_bot)
def tearDown(self):
d = defer.succeed(None)
if self.real_bot and self.real_bot.running:
d.addCallback(lambda _: self.real_bot.stopService())
if os.path.exists(self.basedir):
shutil.rmtree(self.basedir)
return d
def test_getCommands(self):
d = self.bot.callRemote("getCommands")
def check(cmds):
# just check that 'shell' is present..
self.assertTrue('shell' in cmds)
d.addCallback(check)
return d
def test_getVersion(self):
d = self.bot.callRemote("getVersion")
def check(vers):
self.assertEqual(vers, buildslave.version)
d.addCallback(check)
return d
def test_getSlaveInfo(self):
infodir = os.path.join(self.basedir, "info")
os.makedirs(infodir)
open(os.path.join(infodir, "admin"), "w").write("testy!")
open(os.path.join(infodir, "foo"), "w").write("bar")
open(os.path.join(infodir, "environ"), "w").write("something else")
d = self.bot.callRemote("getSlaveInfo")
def check(info):
self.assertEqual(info, dict(
admin='testy!', foo='bar',
environ=os.environ, system=os.name, basedir=self.basedir,
slave_commands=self.real_bot.remote_getCommands(),
version=self.real_bot.remote_getVersion(),
numcpus=multiprocessing.cpu_count()))
d.addCallback(check)
return d
def test_getSlaveInfo_nodir(self):
d = self.bot.callRemote("getSlaveInfo")
def check(info):
self.assertEqual(set(info.keys()), set(
['environ', 'system', 'numcpus', 'basedir', 'slave_commands', 'version']))
d.addCallback(check)
return d
def test_setBuilderList_empty(self):
d = self.bot.callRemote("setBuilderList", [])
def check(builders):
self.assertEqual(builders, {})
d.addCallback(check)
return d
def test_setBuilderList_single(self):
d = self.bot.callRemote("setBuilderList", [('mybld', 'myblddir')])
def check(builders):
self.assertEqual(list(builders), ['mybld'])
self.assertTrue(
os.path.exists(os.path.join(self.basedir, 'myblddir')))
# note that we test the SlaveBuilder instance below
d.addCallback(check)
return d
def test_setBuilderList_updates(self):
d = defer.succeed(None)
slavebuilders = {}
def add_my(_):
d = self.bot.callRemote("setBuilderList", [
('mybld', 'myblddir')])
def check(builders):
self.assertEqual(list(builders), ['mybld'])
self.assertTrue(
os.path.exists(os.path.join(self.basedir, 'myblddir')))
slavebuilders['my'] = builders['mybld']
d.addCallback(check)
return d
d.addCallback(add_my)
def add_your(_):
d = self.bot.callRemote("setBuilderList", [
('mybld', 'myblddir'), ('yourbld', 'yourblddir')])
def check(builders):
self.assertEqual(
sorted(builders.keys()), sorted(['mybld', 'yourbld']))
self.assertTrue(
os.path.exists(os.path.join(self.basedir, 'myblddir')))
self.assertTrue(
os.path.exists(os.path.join(self.basedir, 'yourblddir')))
# 'my' should still be the same slavebuilder object
self.assertEqual(
id(slavebuilders['my']), id(builders['mybld']))
slavebuilders['your'] = builders['yourbld']
d.addCallback(check)
return d
d.addCallback(add_your)
def remove_my(_):
d = self.bot.callRemote("setBuilderList", [
('yourbld', 'yourblddir2')]) # note new builddir
def check(builders):
self.assertEqual(sorted(builders.keys()), sorted(['yourbld']))
# note that build dirs are not deleted..
self.assertTrue(
os.path.exists(os.path.join(self.basedir, 'myblddir')))
self.assertTrue(
os.path.exists(os.path.join(self.basedir, 'yourblddir')))
self.assertTrue(
os.path.exists(os.path.join(self.basedir, 'yourblddir2')))
# 'your' should still be the same slavebuilder object
self.assertEqual(
id(slavebuilders['your']), id(builders['yourbld']))
d.addCallback(check)
return d
d.addCallback(remove_my)
def add_and_remove(_):
d = self.bot.callRemote("setBuilderList", [
('theirbld', 'theirblddir')])
def check(builders):
self.assertEqual(sorted(builders.keys()), sorted(['theirbld']))
self.assertTrue(
os.path.exists(os.path.join(self.basedir, 'myblddir')))
self.assertTrue(
os.path.exists(os.path.join(self.basedir, 'yourblddir')))
self.assertTrue(
os.path.exists(os.path.join(self.basedir, 'theirblddir')))
d.addCallback(check)
return d
d.addCallback(add_and_remove)
return d
def test_shutdown(self):
d1 = defer.Deferred()
self.patch(reactor, "stop", lambda: d1.callback(None))
d2 = self.bot.callRemote("shutdown")
# don't return until both the shutdown method has returned, and
# reactor.stop has been called
return defer.gatherResults([d1, d2])
class FakeStep(object):
"A fake master-side BuildStep that records its activities."
def __init__(self):
self.finished_d = defer.Deferred()
self.actions = []
def wait_for_finish(self):
return self.finished_d
def remote_update(self, updates):
for update in updates:
if 'elapsed' in update[0]:
update[0]['elapsed'] = 1
self.actions.append(["update", updates])
def remote_complete(self, f):
self.actions.append(["complete", f])
self.finished_d.callback(None)
class TestSlaveBuilder(command.CommandTestMixin, unittest.TestCase):
@defer.inlineCallbacks
def setUp(self):
self.basedir = os.path.abspath("basedir")
if os.path.exists(self.basedir):
shutil.rmtree(self.basedir)
os.makedirs(self.basedir)
self.bot = base.BotBase(self.basedir, False)
self.bot.startService()
# get a SlaveBuilder object from the bot and wrap it as a fake remote
builders = yield self.bot.remote_setBuilderList([('sb', 'sb')])
self.sb = FakeRemote(builders['sb'])
self.setUpCommand()
def tearDown(self):
self.tearDownCommand()
d = defer.succeed(None)
if self.bot and self.bot.running:
d.addCallback(lambda _: self.bot.stopService())
if os.path.exists(self.basedir):
shutil.rmtree(self.basedir)
return d
def test_print(self):
return self.sb.callRemote("print", "Hello, SlaveBuilder.")
def test_setMaster(self):
# not much to check here - what the SlaveBuilder does with the
# master is not part of the interface (and, in fact, it does very
# little)
return self.sb.callRemote("setMaster", mock.Mock())
def test_shutdown(self):
# don't *actually* shut down the reactor - that would be silly
stop = mock.Mock()
self.patch(reactor, "stop", stop)
d = self.sb.callRemote("shutdown")
def check(_):
self.assertTrue(stop.called)
d.addCallback(check)
return d
def test_startBuild(self):
return self.sb.callRemote("startBuild")
def test_startCommand(self):
# set up a fake step to receive updates
st = FakeStep()
# patch runprocess to handle the 'echo', below
self.patch_runprocess(
Expect(['echo', 'hello'], os.path.join(
self.basedir, 'sb', 'workdir'))
+ {'hdr': 'headers'} + {'stdout': 'hello\n'} + {'rc': 0}
+ 0,
)
d = defer.succeed(None)
def do_start(_):
return self.sb.callRemote("startCommand", FakeRemote(st),
"13", "shell", dict(
command=['echo', 'hello'],
workdir='workdir'))
d.addCallback(do_start)
d.addCallback(lambda _: st.wait_for_finish())
def check(_):
self.assertEqual(st.actions, [
['update', [[{'hdr': 'headers'}, 0]]],
['update', [[{'stdout': 'hello\n'}, 0]]],
['update', [[{'rc': 0}, 0]]],
['update', [[{'elapsed': 1}, 0]]],
['complete', None],
])
d.addCallback(check)
return d
def test_startCommand_interruptCommand(self):
# set up a fake step to receive updates
st = FakeStep()
# patch runprocess to pretend to sleep (it will really just hang forever,
# except that we interrupt it)
self.patch_runprocess(
Expect(['sleep', '10'], os.path.join(
self.basedir, 'sb', 'workdir'))
+ {'hdr': 'headers'}
+ {'wait': True}
)
d = defer.succeed(None)
def do_start(_):
return self.sb.callRemote("startCommand", FakeRemote(st),
"13", "shell", dict(
command=['sleep', '10'],
workdir='workdir'))
d.addCallback(do_start)
# wait a jiffy..
def do_wait(_):
d = defer.Deferred()
reactor.callLater(0.01, d.callback, None)
return d
d.addCallback(do_wait)
# and then interrupt the step
def do_interrupt(_):
return self.sb.callRemote("interruptCommand", "13", "tl/dr")
d.addCallback(do_interrupt)
d.addCallback(lambda _: st.wait_for_finish())
def check(_):
self.assertEqual(st.actions, [
['update', [[{'hdr': 'headers'}, 0]]],
['update', [[{'hdr': 'killing'}, 0]]],
['update', [[{'rc': -1}, 0]]],
['complete', None],
])
d.addCallback(check)
return d
def test_startCommand_failure(self):
# set up a fake step to receive updates
st = FakeStep()
# patch runprocess to generate a failure
self.patch_runprocess(
Expect(['sleep', '10'], os.path.join(
self.basedir, 'sb', 'workdir'))
+ failure.Failure(Exception("Oops"))
)
# patch the log.err, otherwise trial will think something *actually*
# failed
self.patch(log, "err", lambda f: None)
d = defer.succeed(None)
def do_start(_):
return self.sb.callRemote("startCommand", FakeRemote(st),
"13", "shell", dict(
command=['sleep', '10'],
workdir='workdir'))
d.addCallback(do_start)
d.addCallback(lambda _: st.wait_for_finish())
def check(_):
self.assertEqual(st.actions[1][0], 'complete')
self.assertTrue(isinstance(st.actions[1][1], failure.Failure))
d.addCallback(check)
return d
def test_startCommand_missing_args(self):
# set up a fake step to receive updates
st = FakeStep()
d = defer.succeed(None)
def do_start(_):
return self.sb.callRemote("startCommand", FakeRemote(st),
"13", "shell", dict())
d.addCallback(do_start)
d.addCallback(lambda _: self.assertTrue(False))
d.addErrback(lambda _: True)
return d
class TestBotFactory(unittest.TestCase):
def setUp(self):
self.bf = pb.BotFactory('mstr', 9010, 35, 200)
# tests
def test_timers(self):
clock = self.bf._reactor = task.Clock()
calls = []
def callRemote(method):
calls.append(clock.seconds())
self.assertEqual(method, 'keepalive')
# simulate the response taking a few seconds
d = defer.Deferred()
clock.callLater(5, d.callback, None)
return d
self.bf.perspective = mock.Mock()
self.bf.perspective.callRemote = callRemote
self.bf.startTimers()
clock.callLater(100, self.bf.stopTimers)
clock.pump((1 for _ in range(150)))
self.assertEqual(calls, [35, 70])
@compat.usesFlushLoggedErrors
def test_timers_exception(self):
clock = self.bf._reactor = task.Clock()
self.bf.perspective = mock.Mock()
def callRemote(method):
return defer.fail(RuntimeError("oh noes"))
self.bf.perspective.callRemote = callRemote
self.bf.startTimers()
clock.advance(35)
self.assertEqual(len(self.flushLoggedErrors(RuntimeError)), 1)
# note that the BuildSlave class is tested in test_bot_BuildSlave

246
slave/buildslave/test/unit/test_bot_BuildSlave.py
View File

@@ -1,246 +0,0 @@
# This file is part of Buildbot. Buildbot is free software: you can
# redistribute it and/or modify it under the terms of the GNU General Public
# License as published by the Free Software Foundation, version 2.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 51
# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
# Copyright Buildbot Team Members
import os
import shutil
import socket
from mock import Mock
from twisted.cred import checkers
from twisted.cred import portal
from twisted.internet import defer
from twisted.internet import reactor
from twisted.spread import pb
from twisted.trial import unittest
from zope.interface import implements
from buildslave import bot
from buildslave.test.util import misc
# I don't see any simple way to test the PB equipment without actually setting
# up a TCP connection. This just tests that the PB code will connect and can
# execute a basic ping. The rest is done without TCP (or PB) in other
# test modules.
class MasterPerspective(pb.Avatar):
def __init__(self, on_keepalive=None):
self.on_keepalive = on_keepalive
def perspective_keepalive(self):
if self.on_keepalive:
on_keepalive, self.on_keepalive = self.on_keepalive, None
on_keepalive()
class MasterRealm(object):
def __init__(self, perspective, on_attachment):
self.perspective = perspective
self.on_attachment = on_attachment
implements(portal.IRealm)
def requestAvatar(self, avatarId, mind, *interfaces):
assert pb.IPerspective in interfaces
self.mind = mind
self.perspective.mind = mind
d = defer.succeed(None)
if self.on_attachment:
d.addCallback(lambda _: self.on_attachment(mind))
def returnAvatar(_):
return pb.IPerspective, self.perspective, lambda: None
d.addCallback(returnAvatar)
return d
def shutdown(self):
return self.mind.broker.transport.loseConnection()
class TestBuildSlave(misc.PatcherMixin, unittest.TestCase):
def setUp(self):
self.realm = None
self.buildslave = None
self.listeningport = None
self.basedir = os.path.abspath("basedir")
if os.path.exists(self.basedir):
shutil.rmtree(self.basedir)
os.makedirs(self.basedir)
def tearDown(self):
d = defer.succeed(None)
if self.realm:
d.addCallback(lambda _: self.realm.shutdown())
if self.buildslave and self.buildslave.running:
d.addCallback(lambda _: self.buildslave.stopService())
if self.listeningport:
d.addCallback(lambda _: self.listeningport.stopListening())
if os.path.exists(self.basedir):
shutil.rmtree(self.basedir)
return d
def start_master(self, perspective, on_attachment=None):
self.realm = MasterRealm(perspective, on_attachment)
p = portal.Portal(self.realm)
p.registerChecker(
checkers.InMemoryUsernamePasswordDatabaseDontUse(testy="westy"))
self.listeningport = reactor.listenTCP(
0, pb.PBServerFactory(p), interface='127.0.0.1')
# return the dynamically allocated port number
return self.listeningport.getHost().port
def test_constructor_minimal(self):
# only required arguments
bot.BuildSlave('mstr', 9010, 'me', 'pwd', '/s', 10, False)
def test_constructor_083_tac(self):
# invocation as made from default 083 tac files
bot.BuildSlave('mstr', 9010, 'me', 'pwd', '/s', 10, False,
umask=0o123, maxdelay=10)
def test_constructor_full(self):
# invocation with all args
bot.BuildSlave('mstr', 9010, 'me', 'pwd', '/s', 10, False,
umask=0o123, maxdelay=10, keepaliveTimeout=10,
unicode_encoding='utf8', allow_shutdown=True)
def test_buildslave_print(self):
d = defer.Deferred()
# set up to call print when we are attached, and chain the results onto
# the deferred for the whole test
def call_print(mind):
print_d = mind.callRemote("print", "Hi, slave.")
print_d.addCallbacks(d.callback, d.errback)
# start up the master and slave
persp = MasterPerspective()
port = self.start_master(persp, on_attachment=call_print)
self.buildslave = bot.BuildSlave("127.0.0.1", port,
"testy", "westy", self.basedir,
keepalive=0, usePTY=False, umask=0o22)
self.buildslave.startService()
# and wait for the result of the print
return d
def test_recordHostname_uname(self):
self.patch_os_uname(lambda: [0, 'test-hostname.domain.com'])
self.buildslave = bot.BuildSlave("127.0.0.1", 9999,
"testy", "westy", self.basedir,
keepalive=0, usePTY=False, umask=0o22)
self.buildslave.recordHostname(self.basedir)
self.assertEqual(open(os.path.join(self.basedir, "twistd.hostname")).read().strip(),
'test-hostname.domain.com')
def test_recordHostname_getfqdn(self):
def missing():
raise AttributeError
self.patch_os_uname(missing)
self.patch(socket, "getfqdn", lambda: 'test-hostname.domain.com')
self.buildslave = bot.BuildSlave("127.0.0.1", 9999,
"testy", "westy", self.basedir,
keepalive=0, usePTY=False, umask=0o22)
self.buildslave.recordHostname(self.basedir)
self.assertEqual(open(os.path.join(self.basedir, "twistd.hostname")).read().strip(),
'test-hostname.domain.com')
def test_buildslave_graceful_shutdown(self):
"""Test that running the build slave's gracefulShutdown method results
in a call to the master's shutdown method"""
d = defer.Deferred()
fakepersp = Mock()
called = []
def fakeCallRemote(*args):
called.append(args)
d1 = defer.succeed(None)
return d1
fakepersp.callRemote = fakeCallRemote
# set up to call shutdown when we are attached, and chain the results onto
# the deferred for the whole test
def call_shutdown(mind):
self.buildslave.bf.perspective = fakepersp
shutdown_d = self.buildslave.gracefulShutdown()
shutdown_d.addCallbacks(d.callback, d.errback)
persp = MasterPerspective()
port = self.start_master(persp, on_attachment=call_shutdown)
self.buildslave = bot.BuildSlave("127.0.0.1", port,
"testy", "westy", self.basedir,
keepalive=0, usePTY=False, umask=0o22)
self.buildslave.startService()
def check(ign):
self.assertEquals(called, [('shutdown',)])
d.addCallback(check)
return d
def test_buildslave_shutdown(self):
"""Test watching an existing shutdown_file results in gracefulShutdown
being called."""
buildslave = bot.BuildSlave("127.0.0.1", 1234,
"testy", "westy", self.basedir,
keepalive=0, usePTY=False, umask=0o22,
allow_shutdown='file')
# Mock out gracefulShutdown
buildslave.gracefulShutdown = Mock()
# Mock out os.path methods
exists = Mock()
mtime = Mock()
self.patch(os.path, 'exists', exists)
self.patch(os.path, 'getmtime', mtime)
# Pretend that the shutdown file doesn't exist
mtime.return_value = 0
exists.return_value = False
buildslave._checkShutdownFile()
# We shouldn't have called gracefulShutdown
self.assertEquals(buildslave.gracefulShutdown.call_count, 0)
# Pretend that the file exists now, with an mtime of 2
exists.return_value = True
mtime.return_value = 2
buildslave._checkShutdownFile()
# Now we should have changed gracefulShutdown
self.assertEquals(buildslave.gracefulShutdown.call_count, 1)
# Bump the mtime again, and make sure we call shutdown again
mtime.return_value = 3
buildslave._checkShutdownFile()
self.assertEquals(buildslave.gracefulShutdown.call_count, 2)
# Try again, we shouldn't call shutdown another time
buildslave._checkShutdownFile()
self.assertEquals(buildslave.gracefulShutdown.call_count, 2)

154
slave/buildslave/test/unit/test_commands_base.py
View File

@@ -1,154 +0,0 @@
# This file is part of Buildbot. Buildbot is free software: you can
# redistribute it and/or modify it under the terms of the GNU General Public
# License as published by the Free Software Foundation, version 2.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 51
# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
# Copyright Buildbot Team Members
from twisted.internet import defer
from twisted.trial import unittest
from buildslave.commands.base import Command
from buildslave.test.util.command import CommandTestMixin
# set up a fake Command subclass to test the handling in Command. Think of
# this as testing Command's subclassability.
class DummyCommand(Command):
def setup(self, args):
self.setup_done = True
self.interrupted = False
self.started = False
def start(self):
self.started = True
self.sendStatus(self.args)
self.cmd_deferred = defer.Deferred()
return self.cmd_deferred
def interrupt(self):
self.interrupted = True
self.finishCommand()
def finishCommand(self):
d = self.cmd_deferred
self.cmd_deferred = None
d.callback(None)
def failCommand(self):
d = self.cmd_deferred
self.cmd_deferred = None
d.errback(RuntimeError("forced failure"))
class DummyArgsCommand(DummyCommand):
requiredArgs = ['workdir']
class TestDummyCommand(CommandTestMixin, unittest.TestCase):
def setUp(self):
self.setUpCommand()
def tearDown(self):
self.tearDownCommand()
def assertState(self, setup_done, running, started, interrupted, msg=None):
self.assertEqual(
{
'setup_done': self.cmd.setup_done,
'running': self.cmd.running,
'started': self.cmd.started,
'interrupted': self.cmd.interrupted,
}, {
'setup_done': setup_done,
'running': running,
'started': started,
'interrupted': interrupted,
}, msg)
def test_run(self):
cmd = self.make_command(DummyCommand, {'stdout': 'yay'})
self.assertState(
True, False, False, False, "setup called by constructor")
# start the command
d = self.run_command()
self.assertState(
True, True, True, False, "started and running both set")
# allow the command to finish and check the result
cmd.finishCommand()
def check(_):
self.assertState(
True, False, True, False, "started and not running when done")
d.addCallback(check)
def checkresult(_):
self.assertUpdates([{'stdout': 'yay'}], "updates processed")
d.addCallback(checkresult)
return d
def test_run_failure(self):
cmd = self.make_command(DummyCommand, {})
self.assertState(
True, False, False, False, "setup called by constructor")
# start the command
d = self.run_command()
self.assertState(
True, True, True, False, "started and running both set")
# fail the command with an exception, and check the result
cmd.failCommand()
def check(_):
self.assertState(
True, False, True, False, "started and not running when done")
d.addErrback(check)
def checkresult(_):
self.assertUpdates([{}], "updates processed")
d.addCallback(checkresult)
return d
def test_run_interrupt(self):
cmd = self.make_command(DummyCommand, {})
self.assertState(
True, False, False, False, "setup called by constructor")
# start the command
d = self.run_command()
self.assertState(
True, True, True, False, "started and running both set")
# interrupt the command
cmd.doInterrupt()
self.assertTrue(cmd.interrupted)
def check(_):
self.assertState(
True, False, True, True, "finishes with interrupted set")
d.addCallback(check)
return d
def test_required_args(self):
self.make_command(DummyArgsCommand, {'workdir': '.'})
try:
self.make_command(DummyArgsCommand, {'stdout': 'boo'})
except ValueError:
return
self.fail("Command was supposed to raise ValueError when missing args")

363
slave/buildslave/test/unit/test_commands_fs.py
View File

@@ -1,363 +0,0 @@
# This file is part of Buildbot. Buildbot is free software: you can
# redistribute it and/or modify it under the terms of the GNU General Public
# License as published by the Free Software Foundation, version 2.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 51
# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
# Copyright Buildbot Team Members
import os
import shutil
from twisted.python import runtime
from twisted.trial import unittest
from buildslave.commands import fs
from buildslave.commands import utils
from buildslave.test.util.command import CommandTestMixin
# python-2.4 doesn't have os.errno
if hasattr(os, 'errno'):
errno = os.errno
else:
import errno
class TestRemoveDirectory(CommandTestMixin, unittest.TestCase):
def setUp(self):
self.setUpCommand()
def tearDown(self):
self.tearDownCommand()
def test_simple(self):
self.make_command(fs.RemoveDirectory, dict(
dir='workdir',
), True)
d = self.run_command()
def check(_):
self.assertFalse(
os.path.exists(os.path.abspath(os.path.join(self.basedir, 'workdir'))))
self.assertIn({'rc': 0},
self.get_updates(),
self.builder.show())
d.addCallback(check)
return d
def test_simple_exception(self):
if runtime.platformType == "posix":
return # we only use rmdirRecursive on windows
def fail(dir):
raise RuntimeError("oh noes")
self.patch(utils, 'rmdirRecursive', fail)
self.make_command(fs.RemoveDirectory, dict(
dir='workdir',
), True)
d = self.run_command()
def check(_):
self.assertIn({'rc': -1}, self.get_updates(),
self.builder.show())
d.addCallback(check)
return d
def test_multiple_dirs(self):
self.make_command(fs.RemoveDirectory, dict(
dir=['workdir', 'sourcedir'],
), True)
d = self.run_command()
def check(_):
for dirname in ['workdir', 'sourcedir']:
self.assertFalse(
os.path.exists(os.path.abspath(os.path.join(self.basedir, dirname))))
self.assertIn({'rc': 0},
self.get_updates(),
self.builder.show())
d.addCallback(check)
return d
class TestCopyDirectory(CommandTestMixin, unittest.TestCase):
def setUp(self):
self.setUpCommand()
def tearDown(self):
self.tearDownCommand()
def test_simple(self):
self.make_command(fs.CopyDirectory, dict(
fromdir='workdir',
todir='copy',
), True)
d = self.run_command()
def check(_):
self.assertTrue(
os.path.exists(os.path.abspath(os.path.join(self.basedir, 'copy'))))
self.assertIn({'rc': 0}, # this may ignore a 'header' : '..', which is OK
self.get_updates(),
self.builder.show())
d.addCallback(check)
return d
def test_simple_exception(self):
if runtime.platformType == "posix":
return # we only use rmdirRecursive on windows
def fail(src, dest):
raise RuntimeError("oh noes")
self.patch(shutil, 'copytree', fail)
self.make_command(fs.CopyDirectory, dict(
fromdir='workdir',
todir='copy',
), True)
d = self.run_command()
def check(_):
self.assertIn({'rc': -1},
self.get_updates(),
self.builder.show())
d.addCallback(check)
return d
class TestMakeDirectory(CommandTestMixin, unittest.TestCase):
def setUp(self):
self.setUpCommand()
def tearDown(self):
self.tearDownCommand()
def test_simple(self):
self.make_command(fs.MakeDirectory, dict(
dir='test-dir',
), True)
d = self.run_command()
def check(_):
self.assertTrue(
os.path.exists(os.path.abspath(os.path.join(self.basedir, 'test-dir'))))
self.assertUpdates(
[{'rc': 0}],
self.builder.show())
d.addCallback(check)
return d
def test_already_exists(self):
self.make_command(fs.MakeDirectory, dict(
dir='workdir',
), True)
d = self.run_command()
def check(_):
self.assertUpdates(
[{'rc': 0}],
self.builder.show())
d.addCallback(check)
return d
def test_existing_file(self):
self.make_command(fs.MakeDirectory, dict(
dir='test-file',
), True)
open(os.path.join(self.basedir, 'test-file'), "w")
d = self.run_command()
def check(_):
self.assertIn({'rc': errno.EEXIST},
self.get_updates(),
self.builder.show())
d.addCallback(check)
return d
class TestStatFile(CommandTestMixin, unittest.TestCase):
def setUp(self):
self.setUpCommand()
def tearDown(self):
self.tearDownCommand()
def test_non_existant(self):
self.make_command(fs.StatFile, dict(
file='no-such-file',
), True)
d = self.run_command()
def check(_):
self.assertIn({'rc': errno.ENOENT},
self.get_updates(),
self.builder.show())
d.addCallback(check)
return d
def test_directory(self):
self.make_command(fs.StatFile, dict(
file='workdir',
), True)
d = self.run_command()
def check(_):
import stat
self.assertTrue(
stat.S_ISDIR(self.get_updates()[0]['stat'][stat.ST_MODE]))
self.assertIn({'rc': 0},
self.get_updates(),
self.builder.show())
d.addCallback(check)
return d
def test_file(self):
self.make_command(fs.StatFile, dict(
file='test-file',
), True)
open(os.path.join(self.basedir, 'test-file'), "w")
d = self.run_command()
def check(_):
import stat
self.assertTrue(
stat.S_ISREG(self.get_updates()[0]['stat'][stat.ST_MODE]))
self.assertIn({'rc': 0},
self.get_updates(),
self.builder.show())
d.addCallback(check)
return d
def test_file_workdir(self):
self.make_command(fs.StatFile, dict(
file='test-file',
workdir='wd'
), True)
os.mkdir(os.path.join(self.basedir, 'wd'))
open(os.path.join(self.basedir, 'wd', 'test-file'), "w")
d = self.run_command()
def check(_):
import stat
self.assertTrue(
stat.S_ISREG(self.get_updates()[0]['stat'][stat.ST_MODE]))
self.assertIn({'rc': 0},
self.get_updates(),
self.builder.show())
d.addCallback(check)
return d
class TestGlobPath(CommandTestMixin, unittest.TestCase):
def setUp(self):
self.setUpCommand()
def tearDown(self):
self.tearDownCommand()
def test_non_existant(self):
self.make_command(fs.GlobPath, dict(
path='no-*-file',
), True)
d = self.run_command()
def check(_):
self.assertEqual(self.get_updates()[0]['files'], [])
self.assertIn({'rc': 0},
self.get_updates(),
self.builder.show())
d.addCallback(check)
return d
def test_directory(self):
self.make_command(fs.GlobPath, dict(
path='[wxyz]or?d*',
), True)
d = self.run_command()
def check(_):
self.assertEqual(
self.get_updates()[0]['files'], [os.path.join(self.basedir, 'workdir')])
self.assertIn({'rc': 0},
self.get_updates(),
self.builder.show())
d.addCallback(check)
return d
def test_file(self):
self.make_command(fs.GlobPath, dict(
path='t*-file',
), True)
open(os.path.join(self.basedir, 'test-file'), "w")
d = self.run_command()
def check(_):
self.assertEqual(
self.get_updates()[0]['files'], [os.path.join(self.basedir, 'test-file')])
self.assertIn({'rc': 0},
self.get_updates(),
self.builder.show())
d.addCallback(check)
return d
class TestListDir(CommandTestMixin, unittest.TestCase):
def setUp(self):
self.setUpCommand()
def tearDown(self):
self.tearDownCommand()
def test_non_existant(self):
self.make_command(fs.ListDir,
dict(dir='no-such-dir'),
True)
d = self.run_command()
def check(_):
self.assertIn({'rc': errno.ENOENT},
self.get_updates(),
self.builder.show())
d.addCallback(check)
return d
def test_dir(self):
self.make_command(fs.ListDir, dict(
dir='workdir',
), True)
workdir = os.path.join(self.basedir, 'workdir')
open(os.path.join(workdir, 'file1'), "w")
open(os.path.join(workdir, 'file2'), "w")
d = self.run_command()
def any(items): # not a builtin on python-2.4
for i in items:
if i:
return True
def check(_):
self.assertIn({'rc': 0},
self.get_updates(),
self.builder.show())
self.failUnless(any([
'files' in upd and sorted(upd['files']) == ['file1', 'file2']
for upd in self.get_updates()]),
self.builder.show())
d.addCallback(check)
return d

38
slave/buildslave/test/unit/test_commands_registry.py
View File

@@ -1,38 +0,0 @@
# This file is part of Buildbot. Buildbot is free software: you can
# redistribute it and/or modify it under the terms of the GNU General Public
# License as published by the Free Software Foundation, version 2.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 51
# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
# Copyright Buildbot Team Members
from twisted.trial import unittest
from buildslave.commands import registry
from buildslave.commands import shell
class Registry(unittest.TestCase):
def test_getFactory(self):
factory = registry.getFactory('shell')
self.assertEqual(factory, shell.SlaveShellCommand)
def test_getFactory_KeyError(self):
self.assertRaises(
KeyError, lambda: registry.getFactory('nosuchcommand'))
def test_getAllCommandNames(self):
self.failUnless('shell' in registry.getAllCommandNames())
def test_all_commands_exist(self):
# if this doesn't raise a KeyError, then we're good
for n in registry.getAllCommandNames():
registry.getFactory(n)

53
slave/buildslave/test/unit/test_commands_shell.py
View File

@@ -1,53 +0,0 @@
# This file is part of Buildbot. Buildbot is free software: you can
# redistribute it and/or modify it under the terms of the GNU General Public
# License as published by the Free Software Foundation, version 2.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 51
# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
# Copyright Buildbot Team Members
from twisted.trial import unittest
from buildslave.commands import shell
from buildslave.test.fake.runprocess import Expect
from buildslave.test.util.command import CommandTestMixin
class TestSlaveShellCommand(CommandTestMixin, unittest.TestCase):
def setUp(self):
self.setUpCommand()
def tearDown(self):
self.tearDownCommand()
def test_simple(self):
self.make_command(shell.SlaveShellCommand, dict(
command=['echo', 'hello'],
workdir='workdir',
))
self.patch_runprocess(
Expect(['echo', 'hello'], self.basedir_workdir)
+ {'hdr': 'headers'} + {'stdout': 'hello\n'} + {'rc': 0}
+ 0,
)
d = self.run_command()
# note that SlaveShellCommand does not add any extra updates of it own
def check(_):
self.assertUpdates(
[{'hdr': 'headers'}, {'stdout': 'hello\n'}, {'rc': 0}],
self.builder.show())
d.addCallback(check)
return d
# TODO: test all functionality that SlaveShellCommand adds atop RunProcess

539
slave/buildslave/test/unit/test_commands_transfer.py
View File

@@ -1,539 +0,0 @@
# This file is part of Buildbot. Buildbot is free software: you can
# redistribute it and/or modify it under the terms of the GNU General Public
# License as published by the Free Software Foundation, version 2.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 51
# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
# Copyright Buildbot Team Members
import io
import os
import shutil
import sys
import tarfile
from twisted.internet import defer
from twisted.internet import reactor
from twisted.python import failure
from twisted.python import runtime
from twisted.trial import unittest
from buildslave.commands import transfer
from buildslave.test.fake.remote import FakeRemote
from buildslave.test.util.command import CommandTestMixin
class FakeMasterMethods(object):
# a fake to represent any of:
# - FileWriter
# - FileDirectoryWriter
# - FileReader
def __init__(self, add_update):
self.add_update = add_update
self.delay_write = False
self.count_writes = False
self.keep_data = False
self.write_out_of_space_at = None
self.delay_read = False
self.count_reads = False
self.unpack_fail = False
self.written = False
self.read = False
self.data = ''
def remote_write(self, data):
if self.write_out_of_space_at is not None:
self.write_out_of_space_at -= len(data)
if self.write_out_of_space_at <= 0:
f = failure.Failure(RuntimeError("out of space"))
return defer.fail(f)
if self.count_writes:
self.add_update('write %d' % len(data))
elif not self.written:
self.add_update('write(s)')
self.written = True
if self.keep_data:
self.data += data
if self.delay_write:
d = defer.Deferred()
reactor.callLater(0.01, d.callback, None)
return d
def remote_read(self, length):
if self.count_reads:
self.add_update('read %d' % length)
elif not self.read:
self.add_update('read(s)')
self.read = True
if not self.data:
return ''
slice, self.data = self.data[:length], self.data[length:]
if self.delay_read:
d = defer.Deferred()
reactor.callLater(0.01, d.callback, slice)
return d
else:
return slice
def remote_unpack(self):
self.add_update('unpack')
if self.unpack_fail:
return defer.fail(failure.Failure(RuntimeError("out of space")))
def remote_utime(self, accessed_modified):
self.add_update('utime - %s' % accessed_modified[0])
def remote_close(self):
self.add_update('close')
class TestUploadFile(CommandTestMixin, unittest.TestCase):
def setUp(self):
self.setUpCommand()
self.fakemaster = FakeMasterMethods(self.add_update)
# write 180 bytes of data to upload
self.datadir = os.path.join(self.basedir, 'workdir')
if os.path.exists(self.datadir):
shutil.rmtree(self.datadir)
os.makedirs(self.datadir)
self.datafile = os.path.join(self.datadir, 'data')
# note: use of 'wb' here ensures newlines aren't translated on the
# upload
open(self.datafile, "wb").write("this is some data\n" * 10)
def tearDown(self):
self.tearDownCommand()
if os.path.exists(self.datadir):
shutil.rmtree(self.datadir)
def test_simple(self):
self.fakemaster.count_writes = True # get actual byte counts
self.make_command(transfer.SlaveFileUploadCommand, dict(
workdir='workdir',
slavesrc='data',
writer=FakeRemote(self.fakemaster),
maxsize=1000,
blocksize=64,
keepstamp=False,
))
d = self.run_command()
def check(_):
self.assertUpdates([
{'header': 'sending %s' % self.datafile},
'write 64', 'write 64', 'write 52', 'close',
{'rc': 0}
])
d.addCallback(check)
return d
def test_truncated(self):
self.fakemaster.count_writes = True # get actual byte counts
self.make_command(transfer.SlaveFileUploadCommand, dict(
workdir='workdir',
slavesrc='data',
writer=FakeRemote(self.fakemaster),
maxsize=100,
blocksize=64,
keepstamp=False,
))
d = self.run_command()
def check(_):
self.assertUpdates([
{'header': 'sending %s' % self.datafile},
'write 64', 'write 36', 'close',
{'rc': 1,
'stderr': "Maximum filesize reached, truncating file '%s'" % self.datafile}
])
d.addCallback(check)
return d
def test_missing(self):
self.make_command(transfer.SlaveFileUploadCommand, dict(
workdir='workdir',
slavesrc='data-nosuch',
writer=FakeRemote(self.fakemaster),
maxsize=100,
blocksize=64,
keepstamp=False,
))
d = self.run_command()
def check(_):
df = self.datafile + "-nosuch"
self.assertUpdates([
{'header': 'sending %s' % df},
'close',
{'rc': 1,
'stderr': "Cannot open file '%s' for upload" % df}
])
d.addCallback(check)
return d
def test_out_of_space(self):
self.fakemaster.write_out_of_space_at = 70
self.fakemaster.count_writes = True # get actual byte counts
self.make_command(transfer.SlaveFileUploadCommand, dict(
workdir='workdir',
slavesrc='data',
writer=FakeRemote(self.fakemaster),
maxsize=1000,
blocksize=64,
keepstamp=False,
))
d = self.run_command()
self.assertFailure(d, RuntimeError)
def check(_):
self.assertUpdates([
{'header': 'sending %s' % self.datafile},
'write 64', 'close',
{'rc': 1}
])
d.addCallback(check)
return d
def test_interrupted(self):
self.fakemaster.delay_write = True # write veery slowly
self.make_command(transfer.SlaveFileUploadCommand, dict(
workdir='workdir',
slavesrc='data',
writer=FakeRemote(self.fakemaster),
maxsize=100,
blocksize=2,
keepstamp=False,
))
d = self.run_command()
# wait a jiffy..
interrupt_d = defer.Deferred()
reactor.callLater(0.01, interrupt_d.callback, None)
# and then interrupt the step
def do_interrupt(_):
return self.cmd.interrupt()
interrupt_d.addCallback(do_interrupt)
dl = defer.DeferredList([d, interrupt_d])
def check(_):
self.assertUpdates([
{'header': 'sending %s' % self.datafile},
'write(s)', 'close', {'rc': 1}
])
dl.addCallback(check)
return dl
def test_timestamp(self):
self.fakemaster.count_writes = True # get actual byte counts
timestamp = (os.path.getatime(self.datafile),
os.path.getmtime(self.datafile))
self.make_command(transfer.SlaveFileUploadCommand, dict(
workdir='workdir',
slavesrc='data',
writer=FakeRemote(self.fakemaster),
maxsize=1000,
blocksize=64,
keepstamp=True,
))
d = self.run_command()
def check(_):
self.assertUpdates([
{'header': 'sending %s' % self.datafile},
'write 64', 'write 64', 'write 52',
'close', 'utime - %s' % timestamp[0],
{'rc': 0}
])
d.addCallback(check)
return d
class TestSlaveDirectoryUpload(CommandTestMixin, unittest.TestCase):
def setUp(self):
self.setUpCommand()
self.fakemaster = FakeMasterMethods(self.add_update)
# write a directory to upload
self.datadir = os.path.join(self.basedir, 'workdir', 'data')
if os.path.exists(self.datadir):
shutil.rmtree(self.datadir)
os.makedirs(self.datadir)
open(os.path.join(self.datadir, "aa"), "wb").write("lots of a" * 100)
open(os.path.join(self.datadir, "bb"), "wb").write(
"and a little b" * 17)
def tearDown(self):
self.tearDownCommand()
if os.path.exists(self.datadir):
shutil.rmtree(self.datadir)
def test_simple(self, compress=None):
self.fakemaster.keep_data = True
self.make_command(transfer.SlaveDirectoryUploadCommand, dict(
workdir='workdir',
slavesrc='data',
writer=FakeRemote(self.fakemaster),
maxsize=None,
blocksize=512,
compress=compress,
))
d = self.run_command()
def check(_):
self.assertUpdates([
{'header': 'sending %s' % self.datadir},
'write(s)', 'unpack', # note no 'close"
{'rc': 0}
])
d.addCallback(check)
def check_tarfile(_):
f = io.BytesIO(self.fakemaster.data)
a = tarfile.open(fileobj=f, name='check.tar')
exp_names = ['.', 'aa', 'bb']
got_names = [n.rstrip('/') for n in a.getnames()]
# py27 uses '' instead of '.'
got_names = sorted([n or '.' for n in got_names])
self.assertEqual(got_names, exp_names, "expected archive contents")
a.close()
f.close()
d.addCallback(check_tarfile)
return d
# try it again with bz2 and gzip
def test_simple_bz2(self):
return self.test_simple('bz2')
def test_simple_gz(self):
return self.test_simple('gz')
# except bz2 can't operate in stream mode on py24
if sys.version_info[:2] <= (2, 4):
test_simple_bz2.skip = "bz2 stream decompression not supported on Python-2.4"
def test_out_of_space_unpack(self):
self.fakemaster.keep_data = True
self.fakemaster.unpack_fail = True
self.make_command(transfer.SlaveDirectoryUploadCommand, dict(
workdir='workdir',
slavesrc='data',
writer=FakeRemote(self.fakemaster),
maxsize=None,
blocksize=512,
compress=None
))
d = self.run_command()
self.assertFailure(d, RuntimeError)
def check(_):
self.assertUpdates([
{'header': 'sending %s' % self.datadir},
'write(s)', 'unpack',
{'rc': 1}
])
d.addCallback(check)
return d
# this is just a subclass of SlaveUpload, so the remaining permutations
# are already tested
class TestDownloadFile(CommandTestMixin, unittest.TestCase):
def setUp(self):
self.setUpCommand()
self.fakemaster = FakeMasterMethods(self.add_update)
# the command will write to the basedir, so make sure it exists
if os.path.exists(self.basedir):
shutil.rmtree(self.basedir)
os.makedirs(self.basedir)
def tearDown(self):
self.tearDownCommand()
if os.path.exists(self.basedir):
shutil.rmtree(self.basedir)
def test_simple(self):
self.fakemaster.count_reads = True # get actual byte counts
self.fakemaster.data = test_data = '1234' * 13
assert(len(self.fakemaster.data) == 52)
self.make_command(transfer.SlaveFileDownloadCommand, dict(
workdir='.',
slavedest='data',
reader=FakeRemote(self.fakemaster),
maxsize=None,
blocksize=32,
mode=0o777,
))
d = self.run_command()
def check(_):
self.assertUpdates([
'read 32', 'read 32', 'read 32', 'close',
{'rc': 0}
])
datafile = os.path.join(self.basedir, 'data')
self.assertTrue(os.path.exists(datafile))
self.assertEqual(open(datafile).read(), test_data)
if runtime.platformType != 'win32':
self.assertEqual(os.stat(datafile).st_mode & 0o777, 0o777)
d.addCallback(check)
return d
def test_mkdir(self):
self.fakemaster.data = test_data = 'hi'
self.make_command(transfer.SlaveFileDownloadCommand, dict(
workdir='workdir',
slavedest=os.path.join('subdir', 'data'),
reader=FakeRemote(self.fakemaster),
maxsize=None,
blocksize=32,
mode=0o777,
))
d = self.run_command()
def check(_):
self.assertUpdates([
'read(s)', 'close',
{'rc': 0}
])
datafile = os.path.join(self.basedir, 'workdir', 'subdir', 'data')
self.assertTrue(os.path.exists(datafile))
self.assertEqual(open(datafile).read(), test_data)
d.addCallback(check)
return d
def test_failure(self):
self.fakemaster.data = 'hi'
os.makedirs(os.path.join(self.basedir, 'dir'))
self.make_command(transfer.SlaveFileDownloadCommand, dict(
workdir='.',
slavedest='dir', # but that's a directory!
reader=FakeRemote(self.fakemaster),
maxsize=None,
blocksize=32,
mode=0o777,
))
d = self.run_command()
def check(_):
self.assertUpdates([
'close',
{'rc': 1,
'stderr': "Cannot open file '%s' for download"
% os.path.join(self.basedir, '.', 'dir')}
])
d.addCallback(check)
return d
def test_truncated(self):
self.fakemaster.data = test_data = 'tenchars--' * 10
self.make_command(transfer.SlaveFileDownloadCommand, dict(
workdir='.',
slavedest='data',
reader=FakeRemote(self.fakemaster),
maxsize=50,
blocksize=32,
mode=0o777,
))
d = self.run_command()
def check(_):
self.assertUpdates([
'read(s)', 'close',
{'rc': 1,
'stderr': "Maximum filesize reached, truncating file '%s'"
% os.path.join(self.basedir, '.', 'data')}
])
datafile = os.path.join(self.basedir, 'data')
self.assertTrue(os.path.exists(datafile))
self.assertEqual(open(datafile).read(), test_data[:50])
d.addCallback(check)
return d
def test_interrupted(self):
self.fakemaster.data = 'tenchars--' * 100 # 1k
self.fakemaster.delay_read = True # read veery slowly
self.make_command(transfer.SlaveFileDownloadCommand, dict(
workdir='.',
slavedest='data',
reader=FakeRemote(self.fakemaster),
maxsize=100,
blocksize=2,
mode=0o777,
))
d = self.run_command()
# wait a jiffy..
interrupt_d = defer.Deferred()
reactor.callLater(0.01, interrupt_d.callback, None)
# and then interrupt the step
def do_interrupt(_):
return self.cmd.interrupt()
interrupt_d.addCallback(do_interrupt)
dl = defer.DeferredList([d, interrupt_d])
def check(_):
self.assertUpdates([
'read(s)', 'close', {'rc': 1}
])
dl.addCallback(check)
return dl

144
slave/buildslave/test/unit/test_commands_utils.py
View File

@@ -1,144 +0,0 @@
# This file is part of Buildbot. Buildbot is free software: you can
# redistribute it and/or modify it under the terms of the GNU General Public
# License as published by the Free Software Foundation, version 2.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 51
# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
# Copyright Buildbot Team Members
from __future__ import print_function
import os
import shutil
import sys
import twisted.python.procutils
from twisted.python import runtime
from twisted.trial import unittest
from buildslave.commands import utils
class GetCommand(unittest.TestCase):
def setUp(self):
# monkey-patch 'which' to return something appropriate
self.which_results = {}
def which(arg):
return self.which_results.get(arg, [])
self.patch(twisted.python.procutils, 'which', which)
# note that utils.py currently imports which by name, so we
# patch it there, too
self.patch(utils, 'which', which)
def set_which_results(self, results):
self.which_results = results
def test_getCommand_empty(self):
self.set_which_results({
'xeyes': [],
})
self.assertRaises(RuntimeError, lambda: utils.getCommand('xeyes'))
def test_getCommand_single(self):
self.set_which_results({
'xeyes': ['/usr/bin/xeyes'],
})
self.assertEqual(utils.getCommand('xeyes'), '/usr/bin/xeyes')
def test_getCommand_multi(self):
self.set_which_results({
'xeyes': ['/usr/bin/xeyes', '/usr/X11/bin/xeyes'],
})
self.assertEqual(utils.getCommand('xeyes'), '/usr/bin/xeyes')
def test_getCommand_single_exe(self):
self.set_which_results({
'xeyes': ['/usr/bin/xeyes'],
# it should not select this option, since only one matched
# to begin with
'xeyes.exe': [r'c:\program files\xeyes.exe'],
})
self.assertEqual(utils.getCommand('xeyes'), '/usr/bin/xeyes')
def test_getCommand_multi_exe(self):
self.set_which_results({
'xeyes': [r'c:\program files\xeyes.com', r'c:\program files\xeyes.exe'],
'xeyes.exe': [r'c:\program files\xeyes.exe'],
})
# this one will work out differently depending on platform..
if runtime.platformType == 'win32':
self.assertEqual(
utils.getCommand('xeyes'), r'c:\program files\xeyes.exe')
else:
self.assertEqual(
utils.getCommand('xeyes'), r'c:\program files\xeyes.com')
class RmdirRecursive(unittest.TestCase):
# this is more complicated than you'd think because Twisted doesn't
# rmdir its test directory very well, either..
def setUp(self):
self.target = 'testdir'
try:
if os.path.exists(self.target):
shutil.rmtree(self.target)
except Exception:
# this test will probably fail anyway
e = sys.exc_info()[0]
raise unittest.SkipTest("could not clean before test: %s" % (e,))
# fill it with some files
os.mkdir(os.path.join(self.target))
open(os.path.join(self.target, "a"), "w")
os.mkdir(os.path.join(self.target, "d"))
open(os.path.join(self.target, "d", "a"), "w")
os.mkdir(os.path.join(self.target, "d", "d"))
open(os.path.join(self.target, "d", "d", "a"), "w")
def tearDown(self):
try:
if os.path.exists(self.target):
shutil.rmtree(self.target)
except Exception:
print("\n"
"(target directory was not removed by test, and cleanup "
"failed too)"
"\n")
raise
def test_rmdirRecursive_easy(self):
utils.rmdirRecursive(self.target)
self.assertFalse(os.path.exists(self.target))
def test_rmdirRecursive_symlink(self):
# this was intended as a regression test for #792, but doesn't seem
# to trigger it. It can't hurt to check it, all the same.
if runtime.platformType == 'win32':
raise unittest.SkipTest("no symlinks on this platform")
os.mkdir("noperms")
open("noperms/x", "w")
os.chmod("noperms/x", 0)
try:
os.symlink("../noperms", os.path.join(self.target, "link"))
utils.rmdirRecursive(self.target)
# that shouldn't delete the target of the symlink
self.assertTrue(os.path.exists("noperms"))
finally:
# even Twisted can't clean this up very well, so try hard to
# clean it up ourselves..
os.chmod("noperms/x", 0o777)
os.unlink("noperms/x")
os.rmdir("noperms")
self.assertFalse(os.path.exists(self.target))

816
slave/buildslave/test/unit/test_runprocess.py
View File

@@ -1,816 +0,0 @@
# This file is part of Buildbot. Buildbot is free software: you can
# redistribute it and/or modify it under the terms of the GNU General Public
# License as published by the Free Software Foundation, version 2.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 51
# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
# Copyright Buildbot Team Members
import os
import re
import signal
import sys
import time
from mock import Mock
from twisted.internet import defer
from twisted.internet import reactor
from twisted.internet import task
from twisted.python import log
from twisted.python import runtime
from twisted.python import util
from twisted.trial import unittest
from buildslave import util as bsutil
from buildslave import runprocess
from buildslave.exceptions import AbandonChain
from buildslave.test.fake.slavebuilder import FakeSlaveBuilder
from buildslave.test.util import compat
from buildslave.test.util.misc import BasedirMixin
from buildslave.test.util.misc import nl
def catCommand():
return [sys.executable, '-c', 'import sys; sys.stdout.write(sys.stdin.read())']
def stdoutCommand(output):
return [sys.executable, '-c', 'import sys; sys.stdout.write("%s\\n")' % output]
def stderrCommand(output):
return [sys.executable, '-c', 'import sys; sys.stderr.write("%s\\n")' % output]
def sleepCommand(dur):
return [sys.executable, '-c', 'import time; time.sleep(%d)' % dur]
def scriptCommand(function, *args):
runprocess_scripts = util.sibpath(__file__, 'runprocess-scripts.py')
return [sys.executable, runprocess_scripts, function] + list(args)
def printArgsCommand():
return [sys.executable, '-c', 'import sys; sys.stdout.write(repr(sys.argv[1:]))']
# windows returns rc 1, because exit status cannot indicate "signalled";
# posix returns rc -1 for "signalled"
FATAL_RC = -1
if runtime.platformType == 'win32':
FATAL_RC = 1
# We would like to see debugging output in the test.log
runprocess.RunProcessPP.debug = True
class TestRunProcess(BasedirMixin, unittest.TestCase):
def setUp(self):
self.setUpBasedir()
def tearDown(self):
self.tearDownBasedir()
def testCommandEncoding(self):
b = FakeSlaveBuilder(False, self.basedir)
s = runprocess.RunProcess(b, u'abcd', self.basedir)
self.assertIsInstance(s.command, str)
self.assertIsInstance(s.fake_command, str)
def testCommandEncodingList(self):
b = FakeSlaveBuilder(False, self.basedir)
s = runprocess.RunProcess(b, [u'abcd', 'efg'], self.basedir)
self.assertIsInstance(s.command[0], str)
self.assertIsInstance(s.fake_command[0], str)
def testCommandEncodingObfuscated(self):
b = FakeSlaveBuilder(False, self.basedir)
s = runprocess.RunProcess(b,
[bsutil.Obfuscated(u'abcd', u'ABCD')],
self.basedir)
self.assertIsInstance(s.command[0], str)
self.assertIsInstance(s.fake_command[0], str)
def testStart(self):
b = FakeSlaveBuilder(False, self.basedir)
s = runprocess.RunProcess(b, stdoutCommand('hello'), self.basedir)
d = s.start()
def check(ign):
self.failUnless({'stdout': nl('hello\n')} in b.updates, b.show())
self.failUnless({'rc': 0} in b.updates, b.show())
d.addCallback(check)
return d
def testNoStdout(self):
b = FakeSlaveBuilder(False, self.basedir)
s = runprocess.RunProcess(
b, stdoutCommand('hello'), self.basedir, sendStdout=False)
d = s.start()
def check(ign):
self.failIf({'stdout': nl('hello\n')} in b.updates, b.show())
self.failUnless({'rc': 0} in b.updates, b.show())
d.addCallback(check)
return d
def testKeepStdout(self):
b = FakeSlaveBuilder(False, self.basedir)
s = runprocess.RunProcess(
b, stdoutCommand('hello'), self.basedir, keepStdout=True)
d = s.start()
def check(ign):
self.failUnless({'stdout': nl('hello\n')} in b.updates, b.show())
self.failUnless({'rc': 0} in b.updates, b.show())
self.failUnlessEquals(s.stdout, nl('hello\n'))
d.addCallback(check)
return d
def testStderr(self):
b = FakeSlaveBuilder(False, self.basedir)
s = runprocess.RunProcess(b, stderrCommand("hello"), self.basedir)
d = s.start()
def check(ign):
self.failIf({'stderr': nl('hello\n')} not in b.updates, b.show())
self.failUnless({'rc': 0} in b.updates, b.show())
d.addCallback(check)
return d
def testNoStderr(self):
b = FakeSlaveBuilder(False, self.basedir)
s = runprocess.RunProcess(
b, stderrCommand("hello"), self.basedir, sendStderr=False)
d = s.start()
def check(ign):
self.failIf({'stderr': nl('hello\n')} in b.updates, b.show())
self.failUnless({'rc': 0} in b.updates, b.show())
d.addCallback(check)
return d
def testKeepStderr(self):
b = FakeSlaveBuilder(False, self.basedir)
s = runprocess.RunProcess(
b, stderrCommand("hello"), self.basedir, keepStderr=True)
d = s.start()
def check(ign):
self.failUnless({'stderr': nl('hello\n')} in b.updates, b.show())
self.failUnless({'rc': 0} in b.updates, b.show())
self.failUnlessEquals(s.stderr, nl('hello\n'))
d.addCallback(check)
return d
def testStringCommand(self):
b = FakeSlaveBuilder(False, self.basedir)
# careful! This command must execute the same on windows and UNIX
s = runprocess.RunProcess(b, 'echo hello', self.basedir)
d = s.start()
def check(ign):
self.failUnless({'stdout': nl('hello\n')} in b.updates, b.show())
self.failUnless({'rc': 0} in b.updates, b.show())
d.addCallback(check)
return d
def testObfuscatedCommand(self):
b = FakeSlaveBuilder(False, self.basedir)
s = runprocess.RunProcess(b,
[('obfuscated', 'abcd', 'ABCD')],
self.basedir)
self.assertEqual(s.command, ['abcd'])
self.assertEqual(s.fake_command, ['ABCD'])
def testMultiWordStringCommand(self):
b = FakeSlaveBuilder(False, self.basedir)
# careful! This command must execute the same on windows and UNIX
s = runprocess.RunProcess(b, 'echo Happy Days and Jubilation',
self.basedir)
# no quoting occurs
exp = nl('Happy Days and Jubilation\n')
d = s.start()
def check(ign):
self.failUnless({'stdout': exp} in b.updates, b.show())
self.failUnless({'rc': 0} in b.updates, b.show())
d.addCallback(check)
return d
def testInitialStdinUnicode(self):
b = FakeSlaveBuilder(False, self.basedir)
s = runprocess.RunProcess(
b, catCommand(), self.basedir, initialStdin=u'hello')
d = s.start()
def check(ign):
self.failUnless({'stdout': nl('hello')} in b.updates, b.show())
self.failUnless({'rc': 0} in b.updates, b.show())
d.addCallback(check)
return d
def testMultiWordStringCommandQuotes(self):
b = FakeSlaveBuilder(False, self.basedir)
# careful! This command must execute the same on windows and UNIX
s = runprocess.RunProcess(b, 'echo "Happy Days and Jubilation"',
self.basedir)
if runtime.platformType == "win32":
# echo doesn't parse out the quotes, so they come through in the
# output
exp = nl('"Happy Days and Jubilation"\n')
else:
exp = nl('Happy Days and Jubilation\n')
d = s.start()
def check(ign):
self.failUnless({'stdout': exp} in b.updates, b.show())
self.failUnless({'rc': 0} in b.updates, b.show())
d.addCallback(check)
return d
def testTrickyArguments(self):
# make sure non-trivial arguments are passed verbatim
b = FakeSlaveBuilder(False, self.basedir)
args = [
'Happy Days and Jubilation', # spaces
r'''!"#$%&'()*+,-./:;<=>?@[\]^_`{|}~''', # special characters
'%PATH%', # Windows variable expansions
# Expansions get an argument of their own, because the Windows
# shell doesn't treat % as special unless it surrounds a
# variable name.
]
s = runprocess.RunProcess(b, printArgsCommand() + args, self.basedir)
d = s.start()
def check(ign):
self.failUnless({'stdout': nl(repr(args))} in b.updates, b.show())
self.failUnless({'rc': 0} in b.updates, b.show())
d.addCallback(check)
return d
@compat.skipUnlessPlatformIs("win32")
def testPipeString(self):
b = FakeSlaveBuilder(False, self.basedir)
# this is highly contrived, but it proves the point.
cmd = sys.executable + \
' -c "import sys; sys.stdout.write(\'b\\na\\n\')" | sort'
s = runprocess.RunProcess(b, cmd, self.basedir)
d = s.start()
def check(ign):
self.failUnless({'stdout': nl('a\nb\n')} in b.updates, b.show())
self.failUnless({'rc': 0} in b.updates, b.show())
d.addCallback(check)
return d
def testCommandTimeout(self):
b = FakeSlaveBuilder(False, self.basedir)
s = runprocess.RunProcess(b, sleepCommand(10), self.basedir, timeout=5)
clock = task.Clock()
s._reactor = clock
d = s.start()
def check(ign):
self.failUnless(
{'stdout': nl('hello\n')} not in b.updates, b.show())
self.failUnless({'rc': FATAL_RC} in b.updates, b.show())
d.addCallback(check)
clock.advance(6)
return d
def testCommandMaxTime(self):
b = FakeSlaveBuilder(False, self.basedir)
s = runprocess.RunProcess(b, sleepCommand(10), self.basedir, maxTime=5)
clock = task.Clock()
s._reactor = clock
d = s.start()
def check(ign):
self.failUnless(
{'stdout': nl('hello\n')} not in b.updates, b.show())
self.failUnless({'rc': FATAL_RC} in b.updates, b.show())
d.addCallback(check)
clock.advance(6) # should knock out maxTime
return d
@compat.skipUnlessPlatformIs("posix")
def test_stdin_closed(self):
b = FakeSlaveBuilder(False, self.basedir)
s = runprocess.RunProcess(b,
scriptCommand('assert_stdin_closed'),
self.basedir,
# if usePTY=True, stdin is never closed
usePTY=False,
logEnviron=False)
d = s.start()
def check(ign):
self.failUnless({'rc': 0} in b.updates, b.show())
d.addCallback(check)
return d
@compat.usesFlushLoggedErrors
def test_startCommand_exception(self):
b = FakeSlaveBuilder(False, self.basedir)
s = runprocess.RunProcess(b, ['whatever'], self.basedir)
# set up to cause an exception in _startCommand
def _startCommand(*args, **kwargs):
raise RuntimeError()
s._startCommand = _startCommand
d = s.start()
def check(err):
err.trap(AbandonChain)
stderr = []
# Here we're checking that the exception starting up the command
# actually gets propogated back to the master in stderr.
for u in b.updates:
if 'stderr' in u:
stderr.append(u['stderr'])
stderr = "".join(stderr)
self.failUnless("RuntimeError" in stderr, stderr)
d.addBoth(check)
d.addBoth(lambda _: self.flushLoggedErrors())
return d
def testLogEnviron(self):
b = FakeSlaveBuilder(False, self.basedir)
s = runprocess.RunProcess(b, stdoutCommand('hello'), self.basedir,
environ={"FOO": "BAR"})
d = s.start()
def check(ign):
headers = "".join([list(update.values())[0]
for update in b.updates if list(update) == ["header"]])
self.failUnless("FOO=BAR" in headers, "got:\n" + headers)
d.addCallback(check)
return d
def testNoLogEnviron(self):
b = FakeSlaveBuilder(False, self.basedir)
s = runprocess.RunProcess(b, stdoutCommand('hello'), self.basedir,
environ={"FOO": "BAR"}, logEnviron=False)
d = s.start()
def check(ign):
headers = "".join([list(update.values())[0]
for update in b.updates if list(update) == ["header"]])
self.failUnless("FOO=BAR" not in headers, "got:\n" + headers)
d.addCallback(check)
return d
def testEnvironExpandVar(self):
b = FakeSlaveBuilder(False, self.basedir)
environ = {"EXPND": "-${PATH}-",
"DOESNT_EXPAND": "-${---}-",
"DOESNT_FIND": "-${DOESNT_EXISTS}-"}
s = runprocess.RunProcess(
b, stdoutCommand('hello'), self.basedir, environ=environ)
d = s.start()
def check(ign):
headers = "".join([list(update.values())[0]
for update in b.updates if list(update) == ["header"]])
self.failUnless("EXPND=-$" not in headers, "got:\n" + headers)
self.failUnless("DOESNT_FIND=--" in headers, "got:\n" + headers)
self.failUnless(
"DOESNT_EXPAND=-${---}-" in headers, "got:\n" + headers)
d.addCallback(check)
return d
def testUnsetEnvironVar(self):
b = FakeSlaveBuilder(False, self.basedir)
s = runprocess.RunProcess(b, stdoutCommand('hello'), self.basedir,
environ={"PATH": None})
d = s.start()
def check(ign):
headers = "".join([list(update.values())[0]
for update in b.updates if list(update) == ["header"]])
self.failUnless(
not re.match('\bPATH=', headers), "got:\n" + headers)
d.addCallback(check)
return d
def testEnvironPythonPath(self):
b = FakeSlaveBuilder(False, self.basedir)
s = runprocess.RunProcess(b, stdoutCommand('hello'), self.basedir,
environ={"PYTHONPATH": 'a'})
d = s.start()
def check(ign):
headers = "".join([list(update.values())[0]
for update in b.updates if list(update) == ["header"]])
self.failUnless(not re.match('\bPYTHONPATH=a%s' % (os.pathsep), headers),
"got:\n" + headers)
d.addCallback(check)
return d
def testEnvironArray(self):
b = FakeSlaveBuilder(False, self.basedir)
s = runprocess.RunProcess(b, stdoutCommand('hello'), self.basedir,
environ={"FOO": ['a', 'b']})
d = s.start()
def check(ign):
headers = "".join([list(update.values())[0]
for update in b.updates if list(update) == ["header"]])
self.failUnless(not re.match('\bFOO=a%sb\b' % (os.pathsep), headers),
"got:\n" + headers)
d.addCallback(check)
return d
def testEnvironInt(self):
b = FakeSlaveBuilder(False, self.basedir)
self.assertRaises(RuntimeError, lambda:
runprocess.RunProcess(b, stdoutCommand('hello'), self.basedir,
environ={"BUILD_NUMBER": 13}))
class TestPOSIXKilling(BasedirMixin, unittest.TestCase):
if runtime.platformType != "posix":
skip = "not a POSIX platform"
def setUp(self):
self.pidfiles = []
self.setUpBasedir()
def tearDown(self):
# make sure all of the subprocesses are dead
for pidfile in self.pidfiles:
if not os.path.exists(pidfile):
continue
pid = open(pidfile).read()
if not pid:
return
pid = int(pid)
try:
os.kill(pid, signal.SIGKILL)
except OSError:
pass
# and clean up leftover pidfiles
for pidfile in self.pidfiles:
if os.path.exists(pidfile):
os.unlink(pidfile)
self.tearDownBasedir()
def newPidfile(self):
pidfile = os.path.abspath("test-%d.pid" % len(self.pidfiles))
if os.path.exists(pidfile):
os.unlink(pidfile)
self.pidfiles.append(pidfile)
return pidfile
def waitForPidfile(self, pidfile):
# wait for a pidfile, and return the pid via a Deferred
until = time.time() + 10
d = defer.Deferred()
def poll():
if reactor.seconds() > until:
d.errback(RuntimeError("pidfile %s never appeared" % pidfile))
return
if os.path.exists(pidfile):
try:
pid = int(open(pidfile).read())
except (IOError, TypeError, ValueError):
pid = None
if pid is not None:
d.callback(pid)
return
reactor.callLater(0.01, poll)
poll() # poll right away
return d
def assertAlive(self, pid):
try:
os.kill(pid, 0)
except OSError:
self.fail("pid %d still alive" % (pid,))
def assertDead(self, pid, timeout=5):
log.msg("checking pid %r" % (pid,))
def check():
try:
os.kill(pid, 0)
except OSError:
return True # dead
return False # alive
# check immediately
if check():
return
# poll every 100'th of a second; this allows us to test for
# processes that have been killed, but where the signal hasn't
# been delivered yet
until = time.time() + timeout
while time.time() < until:
time.sleep(0.01)
if check():
return
self.fail("pid %d still alive after %ds" % (pid, timeout))
# tests
def test_simple_interruptSignal(self):
return self.test_simple('TERM')
def test_simple(self, interruptSignal=None):
# test a simple process that just sleeps waiting to die
pidfile = self.newPidfile()
self.pid = None
b = FakeSlaveBuilder(False, self.basedir)
s = runprocess.RunProcess(b,
scriptCommand(
'write_pidfile_and_sleep', pidfile),
self.basedir)
if interruptSignal is not None:
s.interruptSignal = interruptSignal
runproc_d = s.start()
pidfile_d = self.waitForPidfile(pidfile)
def check_alive(pid):
self.pid = pid # for use in check_dead
# test that the process is still alive
self.assertAlive(pid)
# and tell the RunProcess object to kill it
s.kill("diaf")
pidfile_d.addCallback(check_alive)
def check_dead(_):
self.assertDead(self.pid)
runproc_d.addCallback(check_dead)
return defer.gatherResults([pidfile_d, runproc_d])
def test_sigterm(self, interruptSignal=None):
# Tests that the process will receive SIGTERM if sigtermTimeout
# is not None
pidfile = self.newPidfile()
self.pid = None
b = FakeSlaveBuilder(False, self.basedir)
s = runprocess.RunProcess(b,
scriptCommand(
'write_pidfile_and_sleep', pidfile),
self.basedir, sigtermTime=1)
runproc_d = s.start()
pidfile_d = self.waitForPidfile(pidfile)
self.receivedSIGTERM = False
def check_alive(pid):
# Create a mock process that will check if we recieve SIGTERM
mock_process = Mock(wraps=s.process)
mock_process.pgid = None # Skips over group SIGTERM
mock_process.pid = pid
process = s.process
def _mock_signalProcess(sig):
if sig == "TERM":
self.receivedSIGTERM = True
process.signalProcess(sig)
mock_process.signalProcess = _mock_signalProcess
s.process = mock_process
self.pid = pid # for use in check_dead
# test that the process is still alive
self.assertAlive(pid)
# and tell the RunProcess object to kill it
s.kill("diaf")
pidfile_d.addCallback(check_alive)
def check_dead(_):
self.failUnlessEqual(self.receivedSIGTERM, True)
self.assertDead(self.pid)
runproc_d.addCallback(check_dead)
return defer.gatherResults([pidfile_d, runproc_d])
def test_pgroup_usePTY(self):
return self.do_test_pgroup(usePTY=True)
def test_pgroup_no_usePTY(self):
return self.do_test_pgroup(usePTY=False)
def test_pgroup_no_usePTY_no_pgroup(self):
# note that this configuration is not *used*, but that it is
# still supported, and correctly fails to kill the child process
return self.do_test_pgroup(usePTY=False, useProcGroup=False,
expectChildSurvival=True)
def do_test_pgroup(self, usePTY, useProcGroup=True,
expectChildSurvival=False):
# test that a process group gets killed
parent_pidfile = self.newPidfile()
self.parent_pid = None
child_pidfile = self.newPidfile()
self.child_pid = None
b = FakeSlaveBuilder(False, self.basedir)
s = runprocess.RunProcess(b,
scriptCommand(
'spawn_child', parent_pidfile, child_pidfile),
self.basedir,
usePTY=usePTY,
useProcGroup=useProcGroup)
runproc_d = s.start()
# wait for both processes to start up, then call s.kill
parent_pidfile_d = self.waitForPidfile(parent_pidfile)
child_pidfile_d = self.waitForPidfile(child_pidfile)
pidfiles_d = defer.gatherResults([parent_pidfile_d, child_pidfile_d])
def got_pids(pids):
self.parent_pid, self.child_pid = pids
pidfiles_d.addCallback(got_pids)
def kill(_):
s.kill("diaf")
pidfiles_d.addCallback(kill)
# check that both processes are dead after RunProcess is done
d = defer.gatherResults([pidfiles_d, runproc_d])
def check_dead(_):
self.assertDead(self.parent_pid)
if expectChildSurvival:
self.assertAlive(self.child_pid)
else:
self.assertDead(self.child_pid)
d.addCallback(check_dead)
return d
def test_double_fork_usePTY(self):
return self.do_test_double_fork(usePTY=True)
def test_double_fork_no_usePTY(self):
return self.do_test_double_fork(usePTY=False)
def test_double_fork_no_usePTY_no_pgroup(self):
# note that this configuration is not *used*, but that it is
# still supported, and correctly fails to kill the child process
return self.do_test_double_fork(usePTY=False, useProcGroup=False,
expectChildSurvival=True)
def do_test_double_fork(self, usePTY, useProcGroup=True,
expectChildSurvival=False):
# when a spawned process spawns another process, and then dies itself
# (either intentionally or accidentally), we should be able to clean up
# the child.
parent_pidfile = self.newPidfile()
self.parent_pid = None
child_pidfile = self.newPidfile()
self.child_pid = None
b = FakeSlaveBuilder(False, self.basedir)
s = runprocess.RunProcess(b,
scriptCommand(
'double_fork', parent_pidfile, child_pidfile),
self.basedir,
usePTY=usePTY,
useProcGroup=useProcGroup)
runproc_d = s.start()
# wait for both processes to start up, then call s.kill
parent_pidfile_d = self.waitForPidfile(parent_pidfile)
child_pidfile_d = self.waitForPidfile(child_pidfile)
pidfiles_d = defer.gatherResults([parent_pidfile_d, child_pidfile_d])
def got_pids(pids):
self.parent_pid, self.child_pid = pids
pidfiles_d.addCallback(got_pids)
def kill(_):
s.kill("diaf")
pidfiles_d.addCallback(kill)
# check that both processes are dead after RunProcess is done
d = defer.gatherResults([pidfiles_d, runproc_d])
def check_dead(_):
self.assertDead(self.parent_pid)
if expectChildSurvival:
self.assertAlive(self.child_pid)
else:
self.assertDead(self.child_pid)
d.addCallback(check_dead)
return d
class TestLogging(BasedirMixin, unittest.TestCase):
def setUp(self):
self.setUpBasedir()
def tearDown(self):
self.tearDownBasedir()
def testSendStatus(self):
b = FakeSlaveBuilder(False, self.basedir)
s = runprocess.RunProcess(b, stdoutCommand('hello'), self.basedir)
s.sendStatus({'stdout': nl('hello\n')})
self.failUnlessEqual(b.updates, [{'stdout': nl('hello\n')}], b.show())
def testSendBuffered(self):
b = FakeSlaveBuilder(False, self.basedir)
s = runprocess.RunProcess(b, stdoutCommand('hello'), self.basedir)
s._addToBuffers('stdout', 'hello ')
s._addToBuffers('stdout', 'world')
s._sendBuffers()
self.failUnlessEqual(b.updates, [{'stdout': 'hello world'}], b.show())
def testSendBufferedInterleaved(self):
b = FakeSlaveBuilder(False, self.basedir)
s = runprocess.RunProcess(b, stdoutCommand('hello'), self.basedir)
s._addToBuffers('stdout', 'hello ')
s._addToBuffers('stderr', 'DIEEEEEEE')
s._addToBuffers('stdout', 'world')
s._sendBuffers()
self.failUnlessEqual(b.updates, [
{'stdout': 'hello '},
{'stderr': 'DIEEEEEEE'},
{'stdout': 'world'},
])
def testSendChunked(self):
b = FakeSlaveBuilder(False, self.basedir)
s = runprocess.RunProcess(b, stdoutCommand('hello'), self.basedir)
data = "x" * (runprocess.RunProcess.CHUNK_LIMIT * 3 / 2)
s._addToBuffers('stdout', data)
s._sendBuffers()
self.failUnlessEqual(len(b.updates), 2)
def testSendNotimeout(self):
b = FakeSlaveBuilder(False, self.basedir)
s = runprocess.RunProcess(b, stdoutCommand('hello'), self.basedir)
data = "x" * (runprocess.RunProcess.BUFFER_SIZE + 1)
s._addToBuffers('stdout', data)
self.failUnlessEqual(len(b.updates), 1)
class TestLogFileWatcher(BasedirMixin, unittest.TestCase):
def setUp(self):
self.setUpBasedir()
def tearDown(self):
self.tearDownBasedir()
def makeRP(self):
b = FakeSlaveBuilder(False, self.basedir)
rp = runprocess.RunProcess(b, stdoutCommand('hello'), self.basedir)
return rp
def test_statFile_missing(self):
rp = self.makeRP()
if os.path.exists('statfile.log'):
os.remove('statfile.log')
lf = runprocess.LogFileWatcher(rp, 'test', 'statfile.log', False)
self.assertFalse(lf.statFile(), "statfile.log doesn't exist")
def test_statFile_exists(self):
rp = self.makeRP()
open('statfile.log', 'w').write('hi')
lf = runprocess.LogFileWatcher(rp, 'test', 'statfile.log', False)
st = lf.statFile()
self.assertEqual(
st and st[2], 2, "statfile.log exists and size is correct")
os.remove('statfile.log')

103
slave/buildslave/test/unit/test_scripts_base.py
View File

@@ -1,103 +0,0 @@
# This file is part of Buildbot. Buildbot is free software: you can
# redistribute it and/or modify it under the terms of the GNU General Public
# License as published by the Free Software Foundation, version 2.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 51
# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
# Copyright Buildbot Team Members
import io
import os
import sys
from twisted.trial import unittest
from buildslave.scripts import base
from buildslave.test.util import misc
class TestIsBuildslaveDir(misc.FileIOMixin, unittest.TestCase):
"""Test buildslave.scripts.base.isBuildslaveDir()"""
def setUp(self):
# capture output to stdout
self.mocked_stdout = io.BytesIO()
self.patch(sys, "stdout", self.mocked_stdout)
# generate OS specific relative path to buildbot.tac inside basedir
self.tac_file_path = os.path.join("testdir", "buildbot.tac")
def assertReadErrorMessage(self, strerror):
expected_message = "error reading '%s': %s\n" \
"invalid buildslave directory 'testdir'\n" \
% (self.tac_file_path, strerror)
self.assertEqual(self.mocked_stdout.getvalue(),
expected_message,
"unexpected error message on stdout")
def test_open_error(self):
"""Test that open() errors are handled."""
# patch open() to raise IOError
self.setUpOpenError(1, "open-error", "dummy")
# check that isBuildslaveDir() flags directory as invalid
self.assertFalse(base.isBuildslaveDir("testdir"))
# check that correct error message was printed to stdout
self.assertReadErrorMessage("open-error")
# check that open() was called with correct path
self.open.assert_called_once_with(self.tac_file_path)
def test_read_error(self):
"""Test that read() errors on buildbot.tac file are handled."""
# patch open() to return file object that raises IOError on read()
self.setUpReadError(1, "read-error", "dummy")
# check that isBuildslaveDir() flags directory as invalid
self.assertFalse(base.isBuildslaveDir("testdir"))
# check that correct error message was printed to stdout
self.assertReadErrorMessage("read-error")
# check that open() was called with correct path
self.open.assert_called_once_with(self.tac_file_path)
def test_unexpected_tac_contents(self):
"""Test that unexpected contents in buildbot.tac is handled."""
# patch open() to return file with unexpected contents
self.setUpOpen("dummy-contents")
# check that isBuildslaveDir() flags directory as invalid
self.assertFalse(base.isBuildslaveDir("testdir"))
# check that correct error message was printed to stdout
self.assertEqual(self.mocked_stdout.getvalue(),
"unexpected content in '%s'\n" % self.tac_file_path +
"invalid buildslave directory 'testdir'\n",
"unexpected error message on stdout")
# check that open() was called with correct path
self.open.assert_called_once_with(self.tac_file_path)
def test_slavedir_good(self):
"""Test checking valid buildslave directory."""
# patch open() to return file with valid buildslave tac contents
self.setUpOpen("Application('buildslave')")
# check that isBuildslaveDir() flags directory as good
self.assertTrue(base.isBuildslaveDir("testdir"))
# check that open() was called with correct path
self.open.assert_called_once_with(self.tac_file_path)

776
slave/buildslave/test/unit/test_scripts_create_slave.py
View File

@@ -1,776 +0,0 @@
# This file is part of Buildbot. Buildbot is free software: you can
# redistribute it and/or modify it under the terms of the GNU General Public
# License as published by the Free Software Foundation, version 2.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 51
# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
# Copyright Buildbot Team Members
import os
import mock
from twisted.trial import unittest
from buildslave.scripts import create_slave
from buildslave.test.util import misc
def _regexp_path(name, *names):
"""
Join two or more path components and create a regexp that will match that
path.
"""
return os.path.join(name, *names).replace("\\", "\\\\")
class TestMakeBaseDir(misc.StdoutAssertionsMixin, unittest.TestCase):
"""
Test buildslave.scripts.create_slave._makeBaseDir()
"""
def setUp(self):
# capture stdout
self.setUpStdoutAssertions()
# patch os.mkdir() to do nothing
self.mkdir = mock.Mock()
self.patch(os, "mkdir", self.mkdir)
def testBasedirExists(self):
"""
test calling _makeBaseDir() on existing base directory
"""
self.patch(os.path, "exists", mock.Mock(return_value=True))
# call _makeBaseDir()
create_slave._makeBaseDir("dummy", False)
# check that correct message was printed to stdout
self.assertStdoutEqual("updating existing installation\n")
# check that os.mkdir was not called
self.assertFalse(self.mkdir.called,
"unexpected call to os.mkdir()")
def testBasedirExistsQuiet(self):
"""
test calling _makeBaseDir() on existing base directory with
quiet flag enabled
"""
self.patch(os.path, "exists", mock.Mock(return_value=True))
# call _makeBaseDir()
create_slave._makeBaseDir("dummy", True)
# check that nothing was printed to stdout
self.assertWasQuiet()
# check that os.mkdir was not called
self.assertFalse(self.mkdir.called,
"unexpected call to os.mkdir()")
def testBasedirCreated(self):
"""
test creating new base directory with _makeBaseDir()
"""
self.patch(os.path, "exists", mock.Mock(return_value=False))
# call _makeBaseDir()
create_slave._makeBaseDir("dummy", False)
# check that os.mkdir() was called with correct path
self.mkdir.assert_called_once_with("dummy")
# check that correct message was printed to stdout
self.assertStdoutEqual("mkdir dummy\n")
def testBasedirCreatedQuiet(self):
"""
test creating new base directory with _makeBaseDir()
and quiet flag enabled
"""
self.patch(os.path, "exists", mock.Mock(return_value=False))
# call _makeBaseDir()
create_slave._makeBaseDir("dummy", True)
# check that os.mkdir() was called with correct path
self.mkdir.assert_called_once_with("dummy")
# check that nothing was printed to stdout
self.assertWasQuiet()
def testMkdirError(self):
"""
test that _makeBaseDir() handles error creating directory correctly
"""
self.patch(os.path, "exists", mock.Mock(return_value=False))
# patch os.mkdir() to raise an exception
self.patch(os, "mkdir",
mock.Mock(side_effect=OSError(0, "dummy-error")))
# check that correct exception was raised
self.assertRaisesRegexp(create_slave.CreateSlaveError,
"error creating directory dummy: dummy-error",
create_slave._makeBaseDir, "dummy", False)
class TestMakeBuildbotTac(misc.StdoutAssertionsMixin,
misc.FileIOMixin,
unittest.TestCase):
"""
Test buildslave.scripts.create_slave._makeBuildbotTac()
"""
def setUp(self):
# capture stdout
self.setUpStdoutAssertions()
# patch os.chmod() to do nothing
self.chmod = mock.Mock()
self.patch(os, "chmod", self.chmod)
# generate OS specific relative path to buildbot.tac inside basedir
self.tac_file_path = _regexp_path("bdir", "buildbot.tac")
def testTacOpenError(self):
"""
test that _makeBuildbotTac() handles open() errors on buildbot.tac
"""
self.patch(os.path, "exists", mock.Mock(return_value=True))
# patch open() to raise exception
self.setUpOpenError()
# call _makeBuildbotTac() and check that correct exception is raised
expected_message = "error reading %s: dummy-msg" % self.tac_file_path
self.assertRaisesRegexp(create_slave.CreateSlaveError,
expected_message,
create_slave._makeBuildbotTac,
"bdir", "contents", False)
def testTacReadError(self):
"""
test that _makeBuildbotTac() handles read() errors on buildbot.tac
"""
self.patch(os.path, "exists", mock.Mock(return_value=True))
# patch read() to raise exception
self.setUpReadError()
# call _makeBuildbotTac() and check that correct exception is raised
expected_message = "error reading %s: dummy-msg" % self.tac_file_path
self.assertRaisesRegexp(create_slave.CreateSlaveError,
expected_message,
create_slave._makeBuildbotTac,
"bdir", "contents", False)
def testTacWriteError(self):
"""
test that _makeBuildbotTac() handles write() errors on buildbot.tac
"""
self.patch(os.path, "exists", mock.Mock(return_value=False))
# patch write() to raise exception
self.setUpWriteError(0)
# call _makeBuildbotTac() and check that correct exception is raised
expected_message = "could not write %s: dummy-msg" % self.tac_file_path
self.assertRaisesRegexp(create_slave.CreateSlaveError,
expected_message,
create_slave._makeBuildbotTac,
"bdir", "contents", False)
def checkTacFileCorrect(self, quiet):
"""
Utility function to test calling _makeBuildbotTac() on base directory
with existing buildbot.tac file, which does not need to be changed.
@param quiet: the value of 'quiet' argument for _makeBuildbotTac()
"""
# set-up mocks to simulate buildbot.tac file in the basedir
self.patch(os.path, "exists", mock.Mock(return_value=True))
self.setUpOpen("test-tac-contents")
# call _makeBuildbotTac()
create_slave._makeBuildbotTac("bdir", "test-tac-contents", quiet)
# check that write() was not called
self.assertFalse(self.fileobj.write.called,
"unexpected write() call")
# check output to stdout
if quiet:
self.assertWasQuiet()
else:
self.assertStdoutEqual(
"buildbot.tac already exists and is correct\n")
def testTacFileCorrect(self):
"""
call _makeBuildbotTac() on base directory which contains a buildbot.tac
file, which does not need to be changed
"""
self.checkTacFileCorrect(False)
def testTacFileCorrectQuiet(self):
"""
call _makeBuildbotTac() on base directory which contains a buildbot.tac
file, which does not need to be changed. Check that quite flag works
"""
self.checkTacFileCorrect(True)
def checkDiffTacFile(self, quiet):
"""
Utility function to test calling _makeBuildbotTac() on base directory
with a buildbot.tac file, with does needs to be changed.
@param quiet: the value of 'quiet' argument for _makeBuildbotTac()
"""
# set-up mocks to simulate buildbot.tac file in basedir
self.patch(os.path, "exists", mock.Mock(return_value=True))
self.setUpOpen("old-tac-contents")
# call _makeBuildbotTac()
create_slave._makeBuildbotTac("bdir", "new-tac-contents", quiet)
# check that buildbot.tac.new file was created with expected contents
tac_file_path = os.path.join("bdir", "buildbot.tac")
self.open.assert_has_calls([mock.call(tac_file_path, "rt"),
mock.call(tac_file_path + ".new", "wt")])
self.fileobj.write.assert_called_once_with("new-tac-contents")
self.chmod.assert_called_once_with(tac_file_path + ".new", 0o600)
# check output to stdout
if quiet:
self.assertWasQuiet()
else:
self.assertStdoutEqual("not touching existing buildbot.tac\n"
"creating buildbot.tac.new instead\n")
def testDiffTacFile(self):
"""
call _makeBuildbotTac() on base directory which contains a buildbot.tac
file, with does needs to be changed.
"""
self.checkDiffTacFile(False)
def testDiffTacFileQuiet(self):
"""
call _makeBuildbotTac() on base directory which contains a buildbot.tac
file, with does needs to be changed. Check that quite flag works
"""
self.checkDiffTacFile(True)
def testNoTacFile(self):
"""
call _makeBuildbotTac() on base directory with no buildbot.tac file
"""
self.patch(os.path, "exists", mock.Mock(return_value=False))
# capture calls to open() and write()
self.setUpOpen()
# call _makeBuildbotTac()
create_slave._makeBuildbotTac("bdir", "test-tac-contents", False)
# check that buildbot.tac file was created with expected contents
tac_file_path = os.path.join("bdir", "buildbot.tac")
self.open.assert_called_once_with(tac_file_path, "wt")
self.fileobj.write.assert_called_once_with("test-tac-contents")
self.chmod.assert_called_once_with(tac_file_path, 0o600)
class TestMakeInfoFiles(misc.StdoutAssertionsMixin,
misc.FileIOMixin,
unittest.TestCase):
"""
Test buildslave.scripts.create_slave._makeInfoFiles()
"""
def setUp(self):
# capture stdout
self.setUpStdoutAssertions()
def checkMkdirError(self, quiet):
"""
Utility function to test _makeInfoFiles() when os.mkdir() fails.
Patch os.mkdir() to raise an exception, and check that _makeInfoFiles()
handles mkdir errors correctly.
@param quiet: the value of 'quiet' argument for _makeInfoFiles()
"""
self.patch(os.path, "exists", mock.Mock(return_value=False))
# patch os.mkdir() to raise an exception
self.patch(os, "mkdir", mock.Mock(side_effect=OSError(0, "err-msg")))
# call _makeInfoFiles() and check that correct exception is raised
self.assertRaisesRegexp(create_slave.CreateSlaveError,
"error creating directory %s: err-msg" %
_regexp_path("bdir", "info"),
create_slave._makeInfoFiles,
"bdir", quiet)
# check output to stdout
if quiet:
self.assertWasQuiet()
else:
self.assertStdoutEqual("mkdir %s\n" % os.path.join("bdir", "info"))
def testMkdirError(self):
"""
test _makeInfoFiles() when os.mkdir() fails
"""
self.checkMkdirError(False)
def testMkdirErrorQuiet(self):
"""
test _makeInfoFiles() when os.mkdir() fails and quiet flag is enabled
"""
self.checkMkdirError(True)
def checkIOError(self, error_type, quiet):
"""
Utility function to test _makeInfoFiles() when open() or write() fails.
Patch file IO functions to raise an exception, and check that
_makeInfoFiles() handles file IO errors correctly.
@param error_type: type of error to emulate,
'open' - patch open() to fail
'write' - patch write() to fail
@param quiet: the value of 'quiet' argument for _makeInfoFiles()
"""
# patch os.path.exists() to simulate that 'info' directory exists
# but not 'admin' or 'host' files
self.patch(os.path, "exists", lambda path: path.endswith("info"))
# set-up requested IO error
if error_type == "open":
self.setUpOpenError(strerror="info-err-msg")
elif error_type == "write":
self.setUpWriteError(strerror="info-err-msg")
else:
self.fail("unexpected error_type '%s'" % error_type)
# call _makeInfoFiles() and check that correct exception is raised
self.assertRaisesRegexp(create_slave.CreateSlaveError,
"could not write %s: info-err-msg" %
_regexp_path("bdir", "info", "admin"),
create_slave._makeInfoFiles,
"bdir", quiet)
# check output to stdout
if quiet:
self.assertWasQuiet()
else:
self.assertStdoutEqual(
"Creating %s, you need to edit it appropriately.\n" %
os.path.join("info", "admin"))
def testOpenError(self):
"""
test _makeInfoFiles() when open() fails
"""
self.checkIOError("open", False)
def testOpenErrorQuiet(self):
"""
test _makeInfoFiles() when open() fails and quiet flag is enabled
"""
self.checkIOError("open", True)
def testWriteError(self):
"""
test _makeInfoFiles() when write() fails
"""
self.checkIOError("write", False)
def testWriteErrorQuiet(self):
"""
test _makeInfoFiles() when write() fails and quiet flag is enabled
"""
self.checkIOError("write", True)
def checkCreatedSuccessfully(self, quiet):
"""
Utility function to test _makeInfoFiles() when called on
base directory that does not have 'info' sub-directory.
@param quiet: the value of 'quiet' argument for _makeInfoFiles()
"""
# patch os.path.exists() to report the no dirs/files exists
self.patch(os.path, "exists", mock.Mock(return_value=False))
# patch os.mkdir() to do nothing
mkdir_mock = mock.Mock()
self.patch(os, "mkdir", mkdir_mock)
# capture calls to open() and write()
self.setUpOpen()
# call _makeInfoFiles()
create_slave._makeInfoFiles("bdir", quiet)
# check calls to os.mkdir()
info_path = os.path.join("bdir", "info")
mkdir_mock.assert_called_once_with(info_path)
# check open() calls
self.open.assert_has_calls(
[mock.call(os.path.join(info_path, "admin"), "wt"),
mock.call(os.path.join(info_path, "host"), "wt")])
# check write() calls
self.fileobj.write.assert_has_calls(
[mock.call("Your Name Here <admin@youraddress.invalid>\n"),
mock.call("Please put a description of this build host here\n")])
# check output to stdout
if quiet:
self.assertWasQuiet()
else:
self.assertStdoutEqual(
"mkdir %s\n"
"Creating %s, you need to edit it appropriately.\n"
"Creating %s, you need to edit it appropriately.\n"
"Not creating %s - add it if you wish\n"
"Please edit the files in %s appropriately.\n" %
(info_path, os.path.join("info", "admin"),
os.path.join("info", "host"),
os.path.join("info", "access_uri"),
info_path))
def testCreatedSuccessfully(self):
"""
test calling _makeInfoFiles() on basedir without 'info' directory
"""
self.checkCreatedSuccessfully(False)
def testCreatedSuccessfullyQuiet(self):
"""
test calling _makeInfoFiles() on basedir without 'info' directory
and quiet flag is enabled
"""
self.checkCreatedSuccessfully(True)
def testInfoDirExists(self):
"""
test calling _makeInfoFiles() on basedir with fully populated
'info' directory
"""
self.patch(os.path, "exists", mock.Mock(return_value=True))
create_slave._makeInfoFiles("bdir", False)
# there should be no messages to stdout
self.assertWasQuiet()
class TestCreateSlave(misc.StdoutAssertionsMixin, unittest.TestCase):
"""
Test buildslave.scripts.create_slave.createSlave()
"""
# default options and required arguments
options = {
# flags
"no-logrotate": False,
"relocatable": False,
"quiet": False,
# options
"basedir": "bdir",
"allow-shutdown": None,
"umask": None,
"usepty": 0,
"log-size": 16,
"log-count": 8,
"keepalive": 4,
"maxdelay": 2,
"numcpus": None,
# arguments
"host": "masterhost",
"port": 1234,
"name": "slavename",
"passwd": "orange"
}
def setUp(self):
# capture stdout
self.setUpStdoutAssertions()
def setUpMakeFunctions(self, exception=None):
"""
patch create_slave._make*() functions with a mocks
@param exception: if not None, the mocks will raise this exception.
"""
self._makeBaseDir = mock.Mock(side_effect=exception)
self.patch(create_slave,
"_makeBaseDir",
self._makeBaseDir)
self._makeBuildbotTac = mock.Mock(side_effect=exception)
self.patch(create_slave,
"_makeBuildbotTac",
self._makeBuildbotTac)
self._makeInfoFiles = mock.Mock(side_effect=exception)
self.patch(create_slave,
"_makeInfoFiles",
self._makeInfoFiles)
def assertMakeFunctionsCalls(self, basedir, tac_contents, quiet):
"""
assert that create_slave._make*() were called with specified arguments
"""
self._makeBaseDir.assert_called_once_with(basedir, quiet)
self._makeBuildbotTac.assert_called_once_with(basedir,
tac_contents,
quiet)
self._makeInfoFiles.assert_called_once_with(basedir, quiet)
def testCreateError(self):
"""
test that errors while creating buildslave directory are handled
correctly by createSlave()
"""
# patch _make*() functions to raise an exception
self.setUpMakeFunctions(create_slave.CreateSlaveError("err-msg"))
# call createSlave() and check that we get error exit code
self.assertEquals(create_slave.createSlave(self.options), 1,
"unexpected exit code")
# check that correct error message was printed on stdout
self.assertStdoutEqual("err-msg\n"
"failed to configure buildslave in bdir\n")
def testMinArgs(self):
"""
test calling createSlave() with only required arguments
"""
# patch _make*() functions to do nothing
self.setUpMakeFunctions()
# call createSlave() and check that we get success exit code
self.assertEquals(create_slave.createSlave(self.options), 0,
"unexpected exit code")
# check _make*() functions were called with correct arguments
expected_tac_contents = \
"".join(create_slave.slaveTACTemplate) % self.options
self.assertMakeFunctionsCalls(self.options["basedir"],
expected_tac_contents,
self.options["quiet"])
# check that correct info message was printed
self.assertStdoutEqual("buildslave configured in bdir\n")
def assertTACFileContents(self, options):
"""
Check that TAC file generated with provided options is valid Python
script and does typical for TAC file logic.
"""
# import modules for mocking
import twisted.application.service
import twisted.python.logfile
import buildslave.bot
# mock service.Application class
application_mock = mock.Mock()
application_class_mock = mock.Mock(return_value=application_mock)
self.patch(twisted.application.service, "Application",
application_class_mock)
# mock logging stuff
logfile_mock = mock.Mock()
self.patch(twisted.python.logfile.LogFile, "fromFullPath",
logfile_mock)
# mock BuildSlave class
buildslave_mock = mock.Mock()
buildslave_class_mock = mock.Mock(return_value=buildslave_mock)
self.patch(buildslave.bot, "BuildSlave", buildslave_class_mock)
expected_tac_contents = \
"".join(create_slave.slaveTACTemplate) % options
# Executed .tac file with mocked functions with side effect.
# This will raise exception if .tac file is not valid Python file.
glb = {}
exec(expected_tac_contents, glb, glb)
# only one Application must be created in .tac
application_class_mock.assert_called_once_with("buildslave")
# check that BuildSlave created with passed options
buildslave_class_mock.assert_called_once_with(
options["host"],
options["port"],
options["name"],
options["passwd"],
options["basedir"],
options["keepalive"],
options["usepty"],
umask=options["umask"],
numcpus=options["numcpus"],
maxdelay=options["maxdelay"],
allow_shutdown=options["allow-shutdown"])
# check that BuildSlave instance attached to application
self.assertEqual(buildslave_mock.method_calls,
[mock.call.setServiceParent(application_mock)])
# .tac file must define global variable "application", instance of
# Application
self.assertTrue('application' in glb,
".tac file doesn't define \"application\" variable")
self.assertTrue(glb['application'] is application_mock,
"defined \"application\" variable in .tac file is not "
"Application instance")
def testDefaultTACContents(self):
"""
test that with default options generated TAC file is valid.
"""
self.assertTACFileContents(self.options)
def testBackslashInBasedir(self):
"""
test that using backslash (typical for Windows platform) in basedir
won't break generated TAC file.
"""
p = mock.patch.dict(self.options, {"basedir": r"C:\builslave dir\\"})
p.start()
try:
self.assertTACFileContents(self.options)
finally:
p.stop()
def testQuotesInBasedir(self):
"""
test that using quotes in basedir won't break generated TAC file.
"""
p = mock.patch.dict(self.options, {"basedir": r"Buildbot's \"dir"})
p.start()
try:
self.assertTACFileContents(self.options)
finally:
p.stop()
def testDoubleQuotesInBasedir(self):
"""
test that using double quotes at begin and end of basedir won't break
generated TAC file.
"""
p = mock.patch.dict(self.options, {"basedir": r"\"\"Buildbot''"})
p.start()
try:
self.assertTACFileContents(self.options)
finally:
p.stop()
def testSpecialCharactersInOptions(self):
"""
test that using special characters in options strings won't break
generated TAC file.
"""
test_string = ("\"\" & | ^ # @ \\& \\| \\^ \\# \\@ \\n"
" \x07 \" \\\" ' \\' ''")
p = mock.patch.dict(self.options, {
"basedir": test_string,
"host": test_string,
"passwd": test_string,
"name": test_string,
})
p.start()
try:
self.assertTACFileContents(self.options)
finally:
p.stop()
def testNoLogRotate(self):
"""
test that when --no-logrotate options is used, correct tac file
is generated.
"""
options = self.options.copy()
options["no-logrotate"] = True
# patch _make*() functions to do nothing
self.setUpMakeFunctions()
# call createSlave() and check that we get success exit code
self.assertEquals(create_slave.createSlave(options), 0,
"unexpected exit code")
# check _make*() functions were called with correct arguments
expected_tac_contents = (create_slave.slaveTACTemplate[0] +
create_slave.slaveTACTemplate[2]) % options
self.assertMakeFunctionsCalls(self.options["basedir"],
expected_tac_contents,
self.options["quiet"])
# check that correct info message was printed
self.assertStdoutEqual("buildslave configured in bdir\n")
def testWithOpts(self):
"""
test calling createSlave() with --relocatable and --allow-shutdown
options specified.
"""
options = self.options.copy()
options["relocatable"] = True
options["allow-shutdown"] = "signal"
# patch _make*() functions to do nothing
self.setUpMakeFunctions()
# call createSlave() and check that we get success exit code
self.assertEquals(create_slave.createSlave(options), 0,
"unexpected exit code")
# check _make*() functions were called with correct arguments
options["allow-shutdown"] = "'signal'"
expected_tac_contents = \
"".join(create_slave.slaveTACTemplate) % options
self.assertMakeFunctionsCalls(self.options["basedir"],
expected_tac_contents,
options["quiet"])
# check that correct info message was printed
self.assertStdoutEqual("buildslave configured in bdir\n")
def testQuiet(self):
"""
test calling createSlave() with --quiet flag
"""
options = self.options.copy()
options["quiet"] = True
# patch _make*() functions to do nothing
self.setUpMakeFunctions()
# call createSlave() and check that we get success exit code
self.assertEquals(create_slave.createSlave(options), 0,
"unexpected exit code")
# check _make*() functions were called with correct arguments
expected_tac_contents = \
"".join(create_slave.slaveTACTemplate) % options
self.assertMakeFunctionsCalls(options["basedir"],
expected_tac_contents,
options["quiet"])
# there should be no output on stdout
self.assertWasQuiet()

91
slave/buildslave/test/unit/test_scripts_restart.py
View File

@@ -1,91 +0,0 @@
# This file is part of Buildbot. Buildbot is free software: you can
# redistribute it and/or modify it under the terms of the GNU General Public
# License as published by the Free Software Foundation, version 2.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 51
# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
# Copyright Buildbot Team Members
import mock
from twisted.trial import unittest
from buildslave.scripts import restart
from buildslave.scripts import start
from buildslave.scripts import stop
from buildslave.test.util import misc
class TestRestart(misc.IsBuildslaveDirMixin,
misc.StdoutAssertionsMixin,
unittest.TestCase):
"""
Test buildslave.scripts.restart.restart()
"""
config = {"basedir": "dummy", "nodaemon": False, "quiet": False}
def setUp(self):
self.setUpStdoutAssertions()
# patch start.startSlave() to do nothing
self.startSlave = mock.Mock()
self.patch(start, "startSlave", self.startSlave)
def test_bad_basedir(self):
"""
test calling restart() with invalid basedir path
"""
# patch isBuildslaveDir() to fail
self.setupUpIsBuildslaveDir(False)
# call startCommand() and check that correct exit code is returned
self.assertEqual(restart.restart(self.config), 1,
"unexpected exit code")
# check that isBuildslaveDir was called with correct argument
self.isBuildslaveDir.assert_called_once_with(self.config["basedir"])
def test_no_slave_running(self):
"""
test calling restart() when no slave is running
"""
# patch basedir check to always succeed
self.setupUpIsBuildslaveDir(True)
# patch stopSlave() to raise an exception
mock_stopSlave = mock.Mock(side_effect=stop.SlaveNotRunning())
self.patch(stop, "stopSlave", mock_stopSlave)
# check that restart() calls startSlave() and prints correct messages
restart.restart(self.config)
self.startSlave.assert_called_once_with(self.config["basedir"],
self.config["quiet"],
self.config["nodaemon"])
self.assertStdoutEqual("no old buildslave process found to stop\n"
"now restarting buildslave process..\n")
def test_restart(self):
"""
test calling restart() when slave is running
"""
# patch basedir check to always succeed
self.setupUpIsBuildslaveDir(True)
# patch stopSlave() to do nothing
mock_stopSlave = mock.Mock()
self.patch(stop, "stopSlave", mock_stopSlave)
# check that restart() calls startSlave() and prints correct messages
restart.restart(self.config)
self.startSlave.assert_called_once_with(self.config["basedir"],
self.config["quiet"],
self.config["nodaemon"])
self.assertStdoutEqual("now restarting buildslave process..\n")

424
slave/buildslave/test/unit/test_scripts_runner.py
View File

@@ -1,424 +0,0 @@
# This file is part of Buildbot. Buildbot is free software: you can
# redistribute it and/or modify it under the terms of the GNU General Public
# License as published by the Free Software Foundation, version 2.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 51
# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
# Copyright Buildbot Team Members
import os
import sys
import mock
from twisted.python import log
from twisted.python import usage
from twisted.trial import unittest
from buildslave.scripts import runner
from buildslave.test.util import misc
class OptionsMixin(object):
def assertOptions(self, opts, exp):
got = dict([(k, opts[k]) for k in exp])
if got != exp:
msg = []
for k in exp:
if opts[k] != exp[k]:
msg.append(" %s: expected %r, got %r" %
(k, exp[k], opts[k]))
self.fail("did not get expected options\n" + ("\n".join(msg)))
class BaseDirTestsMixin(object):
"""
Common tests for Options classes with 'basedir' parameter
"""
GETCWD_PATH = "test-dir"
ABSPATH_PREFIX = "test-prefix-"
MY_BASEDIR = "my-basedir"
# the options class to instantiate for test cases
options_class = None
def setUp(self):
self.patch(os, "getcwd", lambda: self.GETCWD_PATH)
self.patch(os.path, "abspath", lambda path: self.ABSPATH_PREFIX + path)
def parse(self, *args):
assert self.options_class is not None
opts = self.options_class()
opts.parseOptions(args)
return opts
def test_defaults(self):
opts = self.parse()
self.assertEqual(opts["basedir"],
self.ABSPATH_PREFIX + self.GETCWD_PATH,
"unexpected basedir path")
def test_basedir_arg(self):
opts = self.parse(self.MY_BASEDIR)
self.assertEqual(opts["basedir"],
self.ABSPATH_PREFIX + self.MY_BASEDIR,
"unexpected basedir path")
def test_too_many_args(self):
self.assertRaisesRegexp(usage.UsageError,
"I wasn't expecting so many arguments",
self.parse, "arg1", "arg2")
class TestMakerBase(BaseDirTestsMixin, unittest.TestCase):
"""
Test buildslave.scripts.runner.MakerBase class.
"""
options_class = runner.MakerBase
class TestStopOptions(BaseDirTestsMixin, unittest.TestCase):
"""
Test buildslave.scripts.runner.StopOptions class.
"""
options_class = runner.StopOptions
def test_synopsis(self):
opts = runner.StopOptions()
self.assertIn('buildslave stop', opts.getSynopsis())
class TestStartOptions(OptionsMixin, BaseDirTestsMixin, unittest.TestCase):
"""
Test buildslave.scripts.runner.StartOptions class.
"""
options_class = runner.StartOptions
def test_synopsis(self):
opts = runner.StartOptions()
self.assertIn('buildslave start', opts.getSynopsis())
def test_all_args(self):
opts = self.parse("--quiet", "--nodaemon", self.MY_BASEDIR)
self.assertOptions(opts,
dict(quiet=True, nodaemon=True,
basedir=self.ABSPATH_PREFIX + self.MY_BASEDIR))
class TestRestartOptions(OptionsMixin, BaseDirTestsMixin, unittest.TestCase):
"""
Test buildslave.scripts.runner.RestartOptions class.
"""
options_class = runner.RestartOptions
def test_synopsis(self):
opts = runner.RestartOptions()
self.assertIn('buildslave restart', opts.getSynopsis())
def test_all_args(self):
opts = self.parse("--quiet", "--nodaemon", self.MY_BASEDIR)
self.assertOptions(opts,
dict(quiet=True, nodaemon=True,
basedir=self.ABSPATH_PREFIX + self.MY_BASEDIR))
class TestUpgradeSlaveOptions(BaseDirTestsMixin, unittest.TestCase):
"""
Test buildslave.scripts.runner.UpgradeSlaveOptions class.
"""
options_class = runner.UpgradeSlaveOptions
def test_synopsis(self):
opts = runner.UpgradeSlaveOptions()
self.assertIn('buildslave upgrade-slave', opts.getSynopsis())
class TestCreateSlaveOptions(OptionsMixin, unittest.TestCase):
"""
Test buildslave.scripts.runner.CreateSlaveOptions class.
"""
req_args = ["bdir", "mstr:5678", "name", "pswd"]
def parse(self, *args):
opts = runner.CreateSlaveOptions()
opts.parseOptions(args)
return opts
def test_defaults(self):
self.assertRaisesRegexp(usage.UsageError,
"incorrect number of arguments",
self.parse)
def test_synopsis(self):
opts = runner.CreateSlaveOptions()
self.assertIn('buildslave create-slave', opts.getSynopsis())
def test_min_args(self):
# patch runner.MakerBase.postOptions() so that 'basedir'
# argument will not be converted to absolute path
self.patch(runner.MakerBase, "postOptions", mock.Mock())
self.assertOptions(self.parse(*self.req_args),
dict(basedir="bdir", host="mstr", port=5678,
name="name", passwd="pswd"))
def test_all_args(self):
# patch runner.MakerBase.postOptions() so that 'basedir'
# argument will not be converted to absolute path
self.patch(runner.MakerBase, "postOptions", mock.Mock())
opts = self.parse("--force", "--relocatable", "--no-logrotate",
"--keepalive=4", "--usepty=0", "--umask=022",
"--maxdelay=3", "--numcpus=4", "--log-size=2", "--log-count=1",
"--allow-shutdown=file", *self.req_args)
self.assertOptions(opts,
{"force": True,
"relocatable": True,
"no-logrotate": True,
"usepty": 0,
"umask": "022",
"maxdelay": 3,
"numcpus": "4",
"log-size": 2,
"log-count": "1",
"allow-shutdown": "file",
"basedir": "bdir",
"host": "mstr",
"port": 5678,
"name": "name",
"passwd": "pswd"})
def test_master_url(self):
self.assertRaisesRegexp(usage.UsageError,
"<master> is not a URL - do not use URL",
self.parse, "a", "http://b.c", "d", "e")
def test_inv_keepalive(self):
self.assertRaisesRegexp(usage.UsageError,
"keepalive parameter needs to be an number",
self.parse, "--keepalive=X", *self.req_args)
def test_inv_usepty(self):
self.assertRaisesRegexp(usage.UsageError,
"usepty parameter needs to be an number",
self.parse, "--usepty=X", *self.req_args)
def test_inv_maxdelay(self):
self.assertRaisesRegexp(usage.UsageError,
"maxdelay parameter needs to be an number",
self.parse, "--maxdelay=X", *self.req_args)
def test_inv_log_size(self):
self.assertRaisesRegexp(usage.UsageError,
"log-size parameter needs to be an number",
self.parse, "--log-size=X", *self.req_args)
def test_inv_log_count(self):
self.assertRaisesRegexp(usage.UsageError,
"log-count parameter needs to be an number or None",
self.parse, "--log-count=X", *self.req_args)
def test_inv_numcpus(self):
self.assertRaisesRegexp(usage.UsageError,
"numcpus parameter needs to be an number or None",
self.parse, "--numcpus=X", *self.req_args)
def test_inv_umask(self):
self.assertRaisesRegexp(usage.UsageError,
"umask parameter needs to be an number or None",
self.parse, "--umask=X", *self.req_args)
def test_inv_allow_shutdown(self):
self.assertRaisesRegexp(usage.UsageError,
"allow-shutdown needs to be one of 'signal' or 'file'",
self.parse, "--allow-shutdown=X", *self.req_args)
def test_too_few_args(self):
self.assertRaisesRegexp(usage.UsageError,
"incorrect number of arguments",
self.parse, "arg1", "arg2")
def test_too_many_args(self):
self.assertRaisesRegexp(usage.UsageError,
"incorrect number of arguments",
self.parse, "extra_arg", *self.req_args)
def test_validateMasterArgument_no_port(self):
"""
test calling CreateSlaveOptions.validateMasterArgument()
on <master> argument without port specified.
"""
opts = runner.CreateSlaveOptions()
self.assertEqual(opts.validateMasterArgument("mstrhost"),
("mstrhost", 9989),
"incorrect master host and/or port")
def test_validateMasterArgument_empty_master(self):
"""
test calling CreateSlaveOptions.validateMasterArgument()
on <master> without host part specified.
"""
opts = runner.CreateSlaveOptions()
self.assertRaisesRegexp(usage.UsageError,
"invalid <master> argument ':1234'",
opts.validateMasterArgument, ":1234")
def test_validateMasterArgument_inv_port(self):
"""
test calling CreateSlaveOptions.validateMasterArgument()
on <master> without with unparsable port part
"""
opts = runner.CreateSlaveOptions()
self.assertRaisesRegexp(usage.UsageError,
"invalid master port 'apple', "
"needs to be an number",
opts.validateMasterArgument, "host:apple")
def test_validateMasterArgument_ok(self):
"""
test calling CreateSlaveOptions.validateMasterArgument()
on <master> without host and port parts specified.
"""
opts = runner.CreateSlaveOptions()
self.assertEqual(opts.validateMasterArgument("mstrhost:4321"),
("mstrhost", 4321),
"incorrect master host and/or port")
class TestOptions(misc.StdoutAssertionsMixin, unittest.TestCase):
"""
Test buildslave.scripts.runner.Options class.
"""
def setUp(self):
self.setUpStdoutAssertions()
def parse(self, *args):
opts = runner.Options()
opts.parseOptions(args)
return opts
def test_defaults(self):
self.assertRaisesRegexp(usage.UsageError,
"must specify a command",
self.parse)
def test_version(self):
exception = self.assertRaises(SystemExit, self.parse, '--version')
self.assertEqual(exception.code, 0, "unexpected exit code")
self.assertInStdout('Buildslave version:')
def test_verbose(self):
self.patch(log, 'startLogging', mock.Mock())
self.assertRaises(usage.UsageError, self.parse, "--verbose")
log.startLogging.assert_called_once_with(sys.stderr)
# used by TestRun.test_run_good to patch in a callback
functionPlaceholder = None
class TestRun(misc.StdoutAssertionsMixin, unittest.TestCase):
"""
Test buildslave.scripts.runner.run()
"""
def setUp(self):
self.setUpStdoutAssertions()
class TestSubCommand(usage.Options):
subcommandFunction = __name__ + ".functionPlaceholder"
optFlags = [["test-opt", None, None]]
class TestOptions(usage.Options):
"""
Option class that emulates usage error. The 'suboptions' flag
enables emulation of usage error in a sub-option.
"""
optFlags = [["suboptions", None, None]]
def postOptions(self):
if self["suboptions"]:
self.subOptions = "SubOptionUsage"
raise usage.UsageError("usage-error-message")
def __str__(self):
return "GeneralUsage"
def test_run_good(self):
"""
Test successful invocation of buildslave command.
"""
self.patch(sys, "argv", ["command", 'test', '--test-opt'])
# patch runner module to use our test subcommand class
self.patch(runner.Options, 'subCommands',
[['test', None, self.TestSubCommand, None]])
# trace calls to subcommand function
subcommand_func = mock.Mock(return_value=42)
self.patch(sys.modules[__name__],
"functionPlaceholder",
subcommand_func)
# check that subcommand function called with correct arguments
# and that it's return value is used as exit code
exception = self.assertRaises(SystemExit, runner.run)
subcommand_func.assert_called_once_with({'test-opt': 1})
self.assertEqual(exception.code, 42, "unexpected exit code")
def test_run_bad_noargs(self):
"""
Test handling of invalid command line arguments.
"""
self.patch(sys, "argv", ["command"])
# patch runner module to use test Options class
self.patch(runner, "Options", self.TestOptions)
exception = self.assertRaises(SystemExit, runner.run)
self.assertEqual(exception.code, 1, "unexpected exit code")
self.assertStdoutEqual("command: usage-error-message\n\n"
"GeneralUsage\n",
"unexpected error message on stdout")
def test_run_bad_suboption(self):
"""
Test handling of invalid command line arguments in a suboption.
"""
self.patch(sys, "argv", ["command", "--suboptions"])
# patch runner module to use test Options class
self.patch(runner, "Options", self.TestOptions)
exception = self.assertRaises(SystemExit, runner.run)
self.assertEqual(exception.code, 1, "unexpected exit code")
# check that we get error message for a sub-option
self.assertStdoutEqual("command: usage-error-message\n\n"
"SubOptionUsage\n",
"unexpected error message on stdout")

64
slave/buildslave/test/unit/test_scripts_start.py
View File

@@ -1,64 +0,0 @@
# This file is part of Buildbot. Buildbot is free software: you can
# redistribute it and/or modify it under the terms of the GNU General Public
# License as published by the Free Software Foundation, version 2.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 51
# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
# Copyright Buildbot Team Members
import mock
from twisted.trial import unittest
from buildslave.scripts import start
from buildslave.test.util import misc
class TestStartCommand(unittest.TestCase, misc.IsBuildslaveDirMixin):
"""
Test buildslave.scripts.startup.startCommand()
"""
def test_start_command_bad_basedir(self):
"""
test calling startCommand() with invalid basedir path
"""
# patch isBuildslaveDir() to fail
self.setupUpIsBuildslaveDir(False)
# call startCommand() and check that correct exit code is returned
config = {"basedir": "dummy"}
self.assertEqual(start.startCommand(config), 1, "unexpected exit code")
# check that isBuildslaveDir was called with correct argument
self.isBuildslaveDir.assert_called_once_with("dummy")
def test_start_command_good(self):
"""
test successful startCommand() call
"""
# patch basedir check to always succeed
self.setupUpIsBuildslaveDir(True)
# patch startSlave() to do nothing
mocked_startSlave = mock.Mock(return_value=0)
self.patch(start, "startSlave", mocked_startSlave)
config = {"basedir": "dummy", "nodaemon": False, "quiet": False}
self.assertEqual(start.startCommand(config), 0, "unexpected exit code")
# check that isBuildslaveDir() and startSlave() were called
# with correct argument
self.isBuildslaveDir.assert_called_once_with("dummy")
mocked_startSlave.assert_called_once_with(config["basedir"],
config["quiet"],
config["nodaemon"])

140
slave/buildslave/test/unit/test_scripts_stop.py
View File

@@ -1,140 +0,0 @@
# This file is part of Buildbot. Buildbot is free software: you can
# redistribute it and/or modify it under the terms of the GNU General Public
# License as published by the Free Software Foundation, version 2.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 51
# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
# Copyright Buildbot Team Members
import errno
import os
import signal
import time
import mock
from twisted.trial import unittest
from buildslave.scripts import stop
from buildslave.test.util import compat
from buildslave.test.util import misc
class TestStopSlave(misc.FileIOMixin,
misc.StdoutAssertionsMixin,
unittest.TestCase):
"""
Test buildslave.scripts.stop.stopSlave()
"""
PID = 9876
def setUp(self):
self.setUpStdoutAssertions()
# patch os.chdir() to do nothing
self.patch(os, "chdir", mock.Mock())
def test_no_pid_file(self):
"""
test calling stopSlave() when no pid file is present
"""
# patch open() to raise 'file not found' exception
self.setUpOpenError(2)
# check that stop() raises SlaveNotRunning exception
self.assertRaises(stop.SlaveNotRunning,
stop.stopSlave, None, False)
@compat.skipUnlessPlatformIs("posix")
def test_successful_stop(self):
"""
test stopSlave() on a successful slave stop
"""
def emulated_kill(pid, sig):
if sig == 0:
# when probed if a signal can be send to the process
# emulate that it is dead with 'No such process' error
raise OSError(errno.ESRCH, "dummy")
# patch open() to return a pid file
self.setUpOpen(str(self.PID))
# patch os.kill to emulate successful kill
mocked_kill = mock.Mock(side_effect=emulated_kill)
self.patch(os, "kill", mocked_kill)
# don't waste time
self.patch(time, "sleep", mock.Mock())
# check that stopSlave() sends expected signal to right PID
# and print correct message to stdout
stop.stopSlave(None, False)
mocked_kill.assert_has_calls([mock.call(self.PID, signal.SIGTERM),
mock.call(self.PID, 0)])
self.assertStdoutEqual("buildslave process %s is dead\n" % self.PID)
class TestStop(misc.IsBuildslaveDirMixin,
misc.StdoutAssertionsMixin,
unittest.TestCase):
"""
Test buildslave.scripts.stop.stop()
"""
config = {"basedir": "dummy", "quiet": False}
def test_bad_basedir(self):
"""
test calling stop() with invalid basedir path
"""
# patch isBuildslaveDir() to fail
self.setupUpIsBuildslaveDir(False)
# call startCommand() and check that correct exit code is returned
self.assertEqual(stop.stop(self.config), 1, "unexpected exit code")
# check that isBuildslaveDir was called with correct argument
self.isBuildslaveDir.assert_called_once_with(self.config["basedir"])
def test_no_slave_running(self):
"""
test calling stop() when no slave is running
"""
self.setUpStdoutAssertions()
# patch basedir check to always succeed
self.setupUpIsBuildslaveDir(True)
# patch stopSlave() to raise an exception
mock_stopSlave = mock.Mock(side_effect=stop.SlaveNotRunning())
self.patch(stop, "stopSlave", mock_stopSlave)
stop.stop(self.config)
self.assertStdoutEqual("buildslave not running\n")
def test_successful_stop(self):
"""
test calling stop() when slave is running
"""
# patch basedir check to always succeed
self.setupUpIsBuildslaveDir(True)
# patch stopSlave() to do nothing
mock_stopSlave = mock.Mock()
self.patch(stop, "stopSlave", mock_stopSlave)
stop.stop(self.config)
mock_stopSlave.assert_called_once_with(self.config["basedir"],
self.config["quiet"],
"TERM")

118
slave/buildslave/test/unit/test_scripts_upgrade_slave.py
View File

@@ -1,118 +0,0 @@
# This file is part of Buildbot. Buildbot is free software: you can
# redistribute it and/or modify it under the terms of the GNU General Public
# License as published by the Free Software Foundation, version 2.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 51
# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
# Copyright Buildbot Team Members
import os
import mock
from twisted.trial import unittest
from buildslave.scripts import upgrade_slave
from buildslave.test.util import misc
MODERN_BUILDBOT_TAC = \
"""# dummy buildbot.tac
import os
from buildslave.bot import BuildSlave
"""
OLD_BUILDBOT_TAC = \
"""# dummy buildbot.tac
import os
from buildbot.slave.bot import BuildSlave
"""
class TestUpgradeSlave(misc.IsBuildslaveDirMixin,
misc.StdoutAssertionsMixin,
misc.FileIOMixin,
unittest.TestCase):
"""
Test buildslave.scripts.runner.upgradeSlave()
"""
config = {"basedir": "dummy"}
def setUp(self):
self.setUpStdoutAssertions()
# expected buildbot.tac relative path
self.buildbot_tac = os.path.join(self.config["basedir"],
"buildbot.tac")
def test_upgradeSlave_bad_basedir(self):
"""
test calling upgradeSlave() with bad base directory
"""
# override isBuildslaveDir() to always fail
self.setupUpIsBuildslaveDir(False)
# call upgradeSlave() and check that correct exit code is returned
self.assertEqual(upgrade_slave.upgradeSlave(self.config), 1,
"unexpected exit code")
# check that isBuildslaveDir was called with correct argument
self.isBuildslaveDir.assert_called_once_with("dummy")
def test_upgradeSlave_no_changes(self):
"""
test calling upgradeSlave() on a buildbot.tac that don't need to be
upgraded
"""
# patch basedir check to always succeed
self.setupUpIsBuildslaveDir(True)
# patch open() to return a modern buildbot.tac file
self.setUpOpen(MODERN_BUILDBOT_TAC)
# call upgradeSlave() and check the success exit code is returned
self.assertEqual(upgrade_slave.upgradeSlave(self.config), 0,
"unexpected exit code")
# check message to stdout
self.assertStdoutEqual("No changes made\n")
# check that open() was called with correct path
self.open.assert_called_once_with(self.buildbot_tac)
# check that no writes where made
self.assertFalse(self.fileobj.write.called,
"unexpected write to buildbot.tac file")
def test_upgradeSlave_updated(self):
"""
test calling upgradeSlave() on an older buildbot.tac, that need to
be updated
"""
# patch basedir check to always succeed
self.setupUpIsBuildslaveDir(True)
# patch open() to return older buildbot.tac file
self.setUpOpen(OLD_BUILDBOT_TAC)
# call upgradeSlave() and check the success exit code is returned
self.assertEqual(upgrade_slave.upgradeSlave(self.config), 0,
"unexpected exit code")
# check message to stdout
self.assertStdoutEqual("buildbot.tac updated\n")
# check calls to open()
self.open.assert_has_calls([mock.call(self.buildbot_tac),
mock.call(self.buildbot_tac, "w")])
# check that we wrote correct updated buildbot.tac file
self.fileobj.write.assert_called_once_with(MODERN_BUILDBOT_TAC)

82
slave/buildslave/test/unit/test_util.py
View File

@@ -1,82 +0,0 @@
# This file is part of Buildbot. Buildbot is free software: you can
# redistribute it and/or modify it under the terms of the GNU General Public
# License as published by the Free Software Foundation, version 2.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 51
# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
# Copyright Buildbot Team Members
from twisted.trial import unittest
from buildslave import util
class remove_userpassword(unittest.TestCase):
def assertUrl(self, real_url, expected_url):
new_url = util.remove_userpassword(real_url)
self.assertEqual(expected_url, new_url)
def test_url_with_no_user_and_password(self):
self.assertUrl('http://myurl.com/myrepo', 'http://myurl.com/myrepo')
def test_url_with_user_and_password(self):
self.assertUrl(
'http://myuser:mypass@myurl.com/myrepo', 'http://myurl.com/myrepo')
def test_another_url_with_no_user_and_password(self):
self.assertUrl(
'http://myurl2.com/myrepo2', 'http://myurl2.com/myrepo2')
def test_another_url_with_user_and_password(self):
self.assertUrl(
'http://myuser2:mypass2@myurl2.com/myrepo2', 'http://myurl2.com/myrepo2')
def test_with_different_protocol_without_user_and_password(self):
self.assertUrl('ssh://myurl3.com/myrepo3', 'ssh://myurl3.com/myrepo3')
def test_with_different_protocol_with_user_and_password(self):
self.assertUrl(
'ssh://myuser3:mypass3@myurl3.com/myrepo3', 'ssh://myurl3.com/myrepo3')
def test_file_path(self):
self.assertUrl('/home/me/repos/my-repo', '/home/me/repos/my-repo')
def test_file_path_with_at_sign(self):
self.assertUrl('/var/repos/speci@l', '/var/repos/speci@l')
def test_win32file_path(self):
self.assertUrl('c:\\repos\\my-repo', 'c:\\repos\\my-repo')
class TestObfuscated(unittest.TestCase):
def testSimple(self):
c = util.Obfuscated('real', '****')
self.failUnlessEqual(str(c), '****')
self.failUnlessEqual(repr(c), "'****'")
def testObfuscatedCommand(self):
cmd = ['echo', util.Obfuscated('password', '*******')]
self.failUnlessEqual(
['echo', 'password'], util.Obfuscated.get_real(cmd))
self.failUnlessEqual(
['echo', '*******'], util.Obfuscated.get_fake(cmd))
def testObfuscatedNonString(self):
cmd = ['echo', 1]
self.failUnlessEqual(['echo', '1'], util.Obfuscated.get_real(cmd))
self.failUnlessEqual(['echo', '1'], util.Obfuscated.get_fake(cmd))
def testObfuscatedNonList(self):
cmd = 1
self.failUnlessEqual(1, util.Obfuscated.get_real(cmd))
self.failUnlessEqual(1, util.Obfuscated.get_fake(cmd))

0
slave/buildslave/test/util/__init__.py
View File

147
slave/buildslave/test/util/command.py
View File

@@ -1,147 +0,0 @@
# This file is part of Buildbot. Buildbot is free software: you can
# redistribute it and/or modify it under the terms of the GNU General Public
# License as published by the Free Software Foundation, version 2.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 51
# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
# Copyright Buildbot Team Members
import os
import shutil
import buildslave.runprocess
from buildslave.commands import utils
from buildslave.test.fake import runprocess
from buildslave.test.fake import slavebuilder
class CommandTestMixin(object):
"""
Support for testing Command subclasses.
"""
def setUpCommand(self):
"""
Get things ready to test a Command
Sets:
self.basedir -- the basedir (an abs path)
self.basedir_workdir -- os.path.join(self.basedir, 'workdir')
self.basedir_source -- os.path.join(self.basedir, 'source')
"""
self.basedir = os.path.abspath('basedir')
self.basedir_workdir = os.path.join(self.basedir, 'workdir')
self.basedir_source = os.path.join(self.basedir, 'source')
# clean up the basedir unconditionally
if os.path.exists(self.basedir):
shutil.rmtree(self.basedir)
def tearDownCommand(self):
"""
Call this from the tearDown method to clean up any leftover workdirs and do
any additional cleanup required.
"""
# clean up the basedir unconditionally
if os.path.exists(self.basedir):
shutil.rmtree(self.basedir)
# finish up the runprocess
if hasattr(self, 'runprocess_patched') and self.runprocess_patched:
runprocess.FakeRunProcess.test_done()
def make_command(self, cmdclass, args, makedirs=False):
"""
Create a new command object, creating the necessary arguments. The
cmdclass argument is the Command class, and args is the args dict
to pass to its constructor.
This always creates the SlaveBuilder with a basedir (self.basedir). If
makedirs is true, it will create the basedir and a workdir directory
inside (named 'workdir').
The resulting command is returned, but as a side-effect, the following
attributes are set:
self.cmd -- the command
self.builder -- the (fake) SlaveBuilder
"""
# set up the workdir and basedir
if makedirs:
basedir_abs = os.path.abspath(os.path.join(self.basedir))
workdir_abs = os.path.abspath(
os.path.join(self.basedir, 'workdir'))
if os.path.exists(basedir_abs):
shutil.rmtree(basedir_abs)
os.makedirs(workdir_abs)
b = self.builder = slavebuilder.FakeSlaveBuilder(basedir=self.basedir)
self.cmd = cmdclass(b, 'fake-stepid', args)
return self.cmd
def run_command(self):
"""
Run the command created by make_command. Returns a deferred that will fire
on success or failure.
"""
return self.cmd.doStart()
def get_updates(self):
"""
Return the updates made so far
"""
return self.builder.updates
def assertUpdates(self, updates, msg=None):
"""
Asserts that self.get_updates() matches updates, ignoring elapsed time data
"""
my_updates = []
for update in self.get_updates():
try:
if "elapsed" in update:
continue
except Exception:
pass
my_updates.append(update)
self.assertEqual(my_updates, updates, msg)
def add_update(self, upd):
self.builder.updates.append(upd)
def patch_runprocess(self, *expectations):
"""
Patch a fake RunProcess class in, and set the given expectations.
"""
self.patch(
buildslave.runprocess, 'RunProcess', runprocess.FakeRunProcess)
buildslave.runprocess.RunProcess.expect(*expectations)
self.runprocess_patched = True
def patch_getCommand(self, name, result):
"""
Patch utils.getCommand to return RESULT for NAME
"""
old_getCommand = utils.getCommand
def new_getCommand(n):
if n == name:
return result
return old_getCommand(n)
self.patch(utils, 'getCommand', new_getCommand)
def clean_environ(self):
"""
Temporarily clean out os.environ to { 'PWD' : '.' }
"""
self.patch(os, 'environ', {'PWD': '.'})

37
slave/buildslave/test/util/compat.py
View File

@@ -1,37 +0,0 @@
# This file is part of Buildbot. Buildbot is free software: you can
# redistribute it and/or modify it under the terms of the GNU General Public
# License as published by the Free Software Foundation, version 2.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 51
# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
# Copyright Buildbot Team Members
import sys
import twisted
from twisted.python import runtime
from twisted.python import versions
def usesFlushLoggedErrors(test):
"Decorate a test method that uses flushLoggedErrors with this decorator"
if (sys.version_info[:2] == (2, 7)
and twisted.version <= versions.Version('twisted', 9, 0, 0)):
test.skip = \
"flushLoggedErrors is broken on Python==2.7 and Twisted<=9.0.0"
return test
def skipUnlessPlatformIs(platform):
def closure(test):
if runtime.platformType != platform:
test.skip = "not a %s platform" % platform
return test
return closure

169
slave/buildslave/test/util/misc.py
View File

@@ -1,169 +0,0 @@
# This file is part of Buildbot. Buildbot is free software: you can
# redistribute it and/or modify it under the terms of the GNU General Public
# License as published by the Free Software Foundation, version 2.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 51
# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
# Copyright Buildbot Team Members
import __builtin__
import errno
import io
import os
import shutil
import sys
import mock
from buildslave.scripts import base
def nl(s):
"""Convert the given string to the native newline format, assuming it is
already in normal UNIX newline format (\n). Use this to create the
appropriate expectation in a failUnlessEqual"""
if not isinstance(s, basestring):
return s
return s.replace('\n', os.linesep)
class BasedirMixin(object):
"""Mix this in and call setUpBasedir and tearDownBasedir to set up
a clean basedir with a name given in self.basedir."""
def setUpBasedir(self):
self.basedir = "test-basedir"
if os.path.exists(self.basedir):
shutil.rmtree(self.basedir)
def tearDownBasedir(self):
if os.path.exists(self.basedir):
shutil.rmtree(self.basedir)
class IsBuildslaveDirMixin(object):
"""
Mixin for setting up mocked base.isBuildslaveDir() function
"""
def setupUpIsBuildslaveDir(self, return_value):
self.isBuildslaveDir = mock.Mock(return_value=return_value)
self.patch(base, "isBuildslaveDir", self.isBuildslaveDir)
class PatcherMixin(object):
"""
Mix this in to get a few special-cased patching methods
"""
def patch_os_uname(self, replacement):
# twisted's 'patch' doesn't handle the case where an attribute
# doesn't exist..
if hasattr(os, 'uname'):
self.patch(os, 'uname', replacement)
else:
def cleanup():
del os.uname
self.addCleanup(cleanup)
os.uname = replacement
class FileIOMixin(object):
"""
Mixin for patching open(), read() and write() to simulate successful
I/O operations and various I/O errors.
"""
def setUpOpen(self, file_contents="dummy-contents"):
"""
patch open() to return file object with provided contents.
@param file_contents: contents that will be returned by file object's
read() method
"""
# create mocked file object that returns 'file_contents' on read()
# and tracks any write() calls
self.fileobj = mock.Mock()
self.fileobj.read = mock.Mock(return_value=file_contents)
self.fileobj.write = mock.Mock()
# patch open() to return mocked object
self.open = mock.Mock(return_value=self.fileobj)
self.patch(__builtin__, "open", self.open)
def setUpOpenError(self, errno=errno.ENOENT, strerror="dummy-msg",
filename="dummy-file"):
"""
patch open() to raise IOError
@param errno: exception's errno value
@param strerror: exception's strerror value
@param filename: exception's filename value
"""
self.open = mock.Mock(side_effect=IOError(errno, strerror, filename))
self.patch(__builtin__, "open", self.open)
def setUpReadError(self, errno=errno.EIO, strerror="dummy-msg",
filename="dummy-file"):
"""
patch open() to return a file object that will raise IOError on read()
@param errno: exception's errno value
@param strerror: exception's strerror value
@param filename: exception's filename value
"""
self.fileobj = mock.Mock()
self.fileobj.read = mock.Mock(side_effect=IOError(errno, strerror,
filename))
self.open = mock.Mock(return_value=self.fileobj)
self.patch(__builtin__, "open", self.open)
def setUpWriteError(self, errno=errno.ENOSPC, strerror="dummy-msg",
filename="dummy-file"):
"""
patch open() to return a file object that will raise IOError on write()
@param errno: exception's errno value
@param strerror: exception's strerror value
@param filename: exception's filename value
"""
self.fileobj = mock.Mock()
self.fileobj.write = mock.Mock(side_effect=IOError(errno, strerror,
filename))
self.open = mock.Mock(return_value=self.fileobj)
self.patch(__builtin__, "open", self.open)
class StdoutAssertionsMixin(object):
"""
Mix this in to be able to assert on stdout during the test
"""
def setUpStdoutAssertions(self):
self.stdout = io.BytesIO()
self.patch(sys, 'stdout', self.stdout)
def assertWasQuiet(self):
self.assertEqual(self.stdout.getvalue(), '')
def assertInStdout(self, exp):
self.assertIn(exp, self.stdout.getvalue())
def assertStdoutEqual(self, exp, msg=None):
self.assertEqual(exp, self.stdout.getvalue(), msg)
def getStdout(self):
return self.stdout.getvalue().strip()

85
slave/buildslave/test/util/sourcecommand.py
View File

@@ -1,85 +0,0 @@
# This file is part of Buildbot. Buildbot is free software: you can
# redistribute it and/or modify it under the terms of the GNU General Public
# License as published by the Free Software Foundation, version 2.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 51
# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
# Copyright Buildbot Team Members
from buildslave import runprocess
from buildslave.test.util import command
class SourceCommandTestMixin(command.CommandTestMixin):
"""
Support for testing Source Commands; an extension of CommandTestMixin
"""
def make_command(self, cmdclass, args, makedirs=False, initial_sourcedata=''):
"""
Same as the parent class method, but this also adds some source-specific
patches:
* writeSourcedata - writes to self.sourcedata (self is the TestCase)
* readSourcedata - reads from self.sourcedata
* doClobber - invokes RunProcess(['clobber', DIRECTORY])
* doCopy - invokes RunProcess(['copy', cmd.srcdir, cmd.workdir])
"""
cmd = command.CommandTestMixin.make_command(self, cmdclass, args, makedirs)
# note that these patches are to an *instance*, not a class, so there
# is no need to use self.patch() to reverse them
self.sourcedata = initial_sourcedata
def readSourcedata():
if self.sourcedata is None:
raise IOError("File not found")
return self.sourcedata
cmd.readSourcedata = readSourcedata
def writeSourcedata(res):
self.sourcedata = cmd.sourcedata
return res
cmd.writeSourcedata = writeSourcedata
# patch out a bunch of actions with invocations of RunProcess that will
# end up being Expect-able by the tests.
def doClobber(_, dirname):
r = runprocess.RunProcess(self.builder,
['clobber', dirname],
self.builder.basedir)
return r.start()
cmd.doClobber = doClobber
def doCopy(_):
r = runprocess.RunProcess(self.builder,
['copy', cmd.srcdir, cmd.workdir],
self.builder.basedir)
return r.start()
cmd.doCopy = doCopy
def setFileContents(filename, contents):
r = runprocess.RunProcess(self.builder,
['setFileContents', filename, contents],
self.builder.basedir)
return r.start()
cmd.setFileContents = setFileContents
def check_sourcedata(self, _, expected_sourcedata):
"""
Assert that the sourcedata (from the patched functions - see
make_command) is correct. Use this as a deferred callback.
"""
self.assertEqual(self.sourcedata, expected_sourcedata)
return _

87
slave/buildslave/util.py
View File

@@ -1,87 +0,0 @@
# This file is part of Buildbot. Buildbot is free software: you can
# redistribute it and/or modify it under the terms of the GNU General Public
# License as published by the Free Software Foundation, version 2.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 51
# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
# Copyright Buildbot Team Members
import time
def remove_userpassword(url):
if '@' not in url:
return url
if '://' not in url:
return url
# urlparse would've been nice, but doesn't support ssh... sigh
(protocol, repo_url) = url.split('://')
repo_url = repo_url.split('@')[-1]
return protocol + '://' + repo_url
def now(_reactor=None):
if _reactor and hasattr(_reactor, "seconds"):
return _reactor.seconds()
else:
return time.time()
class Obfuscated(object):
"""An obfuscated string in a command"""
def __init__(self, real, fake):
self.real = real
self.fake = fake
def __str__(self):
return self.fake
def __repr__(self):
return repr(self.fake)
def __eq__(self, other):
return other.__class__ is self.__class__ and \
other.real == self.real and \
other.fake == self.fake
@staticmethod
def to_text(s):
if isinstance(s, basestring):
return s
else:
return str(s)
@staticmethod
def get_real(command):
rv = command
if isinstance(command, list):
rv = []
for elt in command:
if isinstance(elt, Obfuscated):
rv.append(elt.real)
else:
rv.append(Obfuscated.to_text(elt))
return rv
@staticmethod
def get_fake(command):
rv = command
if isinstance(command, list):
rv = []
for elt in command:
if isinstance(elt, Obfuscated):
rv.append(elt.fake)
else:
rv.append(Obfuscated.to_text(elt))
return rv

11
slave/contrib/README.txt
View File

@@ -1,11 +0,0 @@
Utility scripts, things contributed by users but not strictly a part of
buildbot:
zsh/_buildslave: zsh tab-completion file for 'buildslave' command. Put it in
one of the directories appearing in $fpath to enable
tab-completion in zsh.
bash/buildslave: bash tab-completion file for 'buildslave' command. Source this
file to enable completions in your bash session. This is
typically accomplished by placing the file into the
appropriate 'bash_completion.d' directory.

50
slave/contrib/bash/buildslave
View File

@@ -1,50 +0,0 @@
#
# This file installs BASH completions for 'buildslave' command.
#
_buildslave()
{
local buildslave_subcommands="
create-slave upgrade-slave start stop restart"
local cur=${COMP_WORDS[COMP_CWORD]}
local subcommand=
local subcommand_args=
local i=1
#
# 'parse' the command line so far
# figure out if we have subcommand specified and any arguments to it
#
# skip global options
while [[ "${COMP_WORDS[$i]}" == -* ]];
do
i=$(($i+1))
done
# save subcommand
subcommand=${COMP_WORDS[$i]}
i=$(($i+1))
# skip subcommand options
while [[ "${COMP_WORDS[$i]}" == -* ]];
do
i=$(($i+1))
done
# save subcommand arguments
subcommand_args=${COMP_WORDS[@]:$i:${#COMP_WORDS[@]}}
if [ "$cur" == "$subcommand" ]; then
# suggest buildbot subcommands
COMPREPLY=( $(compgen -W "$buildslave_subcommands" $cur) )
elif [ "$cur" == "$subcommand_args" ]; then
# we are at first subcommand argument
# all subcommands can have slave base directory as first argument
# suggest directories
COMPREPLY=( $(compgen -A directory $cur) )
fi
}
complete -F _buildslave buildslave

12
slave/contrib/init-scripts/buildslave.default
View File

@@ -1,12 +0,0 @@
SLAVE_RUNNER=/usr/bin/buildslave
# NOTE: SLAVE_ENABLED has changed its behaviour in version 0.8.4. Use
# 'true|yes|1' to enable instance and 'false|no|0' to disable. Other
# values will be considered as syntax error.
SLAVE_ENABLED[1]=0 # 1-enabled, 0-disabled
SLAVE_NAME[1]="buildslave #1" # short name printed on start/stop
SLAVE_USER[1]="buildbot" # user to run slave as
SLAVE_BASEDIR[1]="" # basedir to slave (absolute path)
SLAVE_OPTIONS[1]="" # buildbot options
SLAVE_PREFIXCMD[1]="" # prefix command, i.e. nice, linux32, dchroot

210
slave/contrib/init-scripts/buildslave.init.sh
View File

@@ -1,210 +0,0 @@
#!/bin/bash
### Maintain compatibility with chkconfig
# chkconfig: 2345 83 17
# description: buildslave
### BEGIN INIT INFO
# Provides: buildslave
# Required-Start: $remote_fs
# Required-Stop: $remote_fs
# Default-Start: 2 3 4 5
# Default-Stop: 0 1 6
# Short-Description: Buildbot slave init script
# Description: This file allows running buildbot slave instances at
# startup
### END INIT INFO
PATH=/sbin:/bin:/usr/sbin:/usr/bin
SLAVE_RUNNER=/usr/bin/buildslave
# Source buildslave configuration
[[ -r /etc/default/buildslave ]] && . /etc/default/buildslave
#[[ -r /etc/sysconfig/buildslave ]] && . /etc/sysconfig/buildslave
# Or define/override the configuration here
#SLAVE_ENABLED[1]=0 # 0-enabled, other-disabled
#SLAVE_NAME[1]="buildslave #1" # short name printed on start/stop
#SLAVE_USER[1]="buildbot" # user to run slave as
#SLAVE_BASEDIR[1]="" # basedir to slave (absolute path)
#SLAVE_OPTIONS[1]="" # buildbot options
#SLAVE_PREFIXCMD[1]="" # prefix command, i.e. nice, linux32, dchroot
# Get some LSB-like functions
if [ -r /lib/lsb/init-functions ]; then
. /lib/lsb/init-functions
else
function log_success_msg() {
echo "$@"
}
function log_failure_msg() {
echo "$@"
}
function log_warning_msg() {
echo "$@"
}
fi
# Some systems don't have seq (e.g. Solaris)
if type seq >/dev/null 2>&1; then
:
else
function seq() {
for ((i=1; i<=$1; i+=1)); do
echo $i
done
}
fi
if [[ ! -x ${SLAVE_RUNNER} ]]; then
log_failure_msg "does not exist or not an executable file: ${SLAVE_RUNNER}"
exit 1
fi
function is_enabled() {
ANSWER=`echo $1|tr "[:upper:]" "[:lower:]"`
[[ "$ANSWER" == "yes" ]] || [[ "$ANSWER" == "true" ]] || [[ "$ANSWER" == "1" ]]
return $?
}
function is_disabled() {
ANSWER=`echo $1|tr "[:upper:]" "[:lower:]"`
[[ "$ANSWER" == "no" ]] || [[ "$ANSWER" == "false" ]] || [[ "$ANSWER" == "0" ]]
return $?
}
function slave_config_valid() {
# Function validates buildslave instance startup variables based on array
# index
local errors=0
local index=$1
if ! is_enabled "${SLAVE_ENABLED[$index]}" && ! is_disabled "${SLAVE_ENABLED[$index]}" ; then
log_warning_msg "buildslave #${index}: invalid enabled status"
errors=$(($errors+1))
fi
if [[ -z ${SLAVE_NAME[$index]} ]]; then
log_failure_msg "buildslave #${index}: no name"
errors=$(($errors+1))
fi
if [[ -z ${SLAVE_USER[$index]} ]]; then
log_failure_msg "buildslave #${index}: no run user specified"
errors=$( ($errors+1) )
elif ! getent passwd ${SLAVE_USER[$index]} >/dev/null; then
log_failure_msg "buildslave #${index}: unknown user ${SLAVE_USER[$index]}"
errors=$(($errors+1))
fi
if [[ ! -d "${SLAVE_BASEDIR[$index]}" ]]; then
log_failure_msg "buildslave ${index}: basedir does not exist ${SLAVE_BASEDIR[$index]}"
errors=$(($errors+1))
fi
return $errors
}
function check_config() {
itemcount="${#SLAVE_ENABLED[@]}
${#SLAVE_NAME[@]}
${#SLAVE_USER[@]}
${#SLAVE_BASEDIR[@]}
${#SLAVE_OPTIONS[@]}
${#SLAVE_PREFIXCMD[@]}"
if [[ $(echo "$itemcount" | tr -d ' ' | sort -u | wc -l) -ne 1 ]]; then
log_failure_msg "SLAVE_* arrays must have an equal number of elements!"
return 1
fi
errors=0
for i in $( seq ${#SLAVE_ENABLED[@]} ); do
if is_disabled "${SLAVE_ENABLED[$i]}" ; then
log_warning_msg "buildslave #${i}: disabled"
continue
fi
slave_config_valid $i
errors=$(($errors+$?))
done
[[ $errors == 0 ]]; return $?
}
check_config || exit $?
function iscallable () { type $1 2>/dev/null | grep -q 'shell function'; }
function slave_op () {
op=$1 ; mi=$2
if [ `uname` = SunOS ]; then
suopt=""
else
suopt="-s /bin/sh"
fi
${SLAVE_PREFIXCMD[$mi]} \
su $suopt - ${SLAVE_USER[$mi]} \
-c "$SLAVE_RUNNER $op ${SLAVE_OPTIONS[$mi]} ${SLAVE_BASEDIR[$mi]} > /dev/null"
return $?
}
function do_op () {
errors=0
for i in $( seq ${#SLAVE_ENABLED[@]} ); do
if [ -n "$4" ] && [ "$4" != "${SLAVE_NAME[$i]}" ] ; then
continue
elif is_disabled "${SLAVE_ENABLED[$i]}" && [ -z "$4" ] ; then
continue
fi
# Some rhels don't come with all the lsb goodies
if iscallable log_daemon_msg; then
log_daemon_msg "$3 \"${SLAVE_NAME[$i]}\""
if eval $1 $2 $i; then
log_end_msg 0
else
log_end_msg 1
errors=$(($errors+1))
fi
else
if eval $1 $2 $i; then
log_success_msg "$3 \"${SLAVE_NAME[$i]}\""
else
log_failure_msg "$3 \"${SLAVE_NAME[$i]}\""
errors=$(($errors+1))
fi
fi
done
return $errors
}
case "$1" in
start)
do_op "slave_op" "start" "Starting buildslave" "$2"
exit $?
;;
stop)
do_op "slave_op" "stop" "Stopping buildslave" "$2"
exit $?
;;
reload)
do_op "slave_op" "reload" "Reloading buildslave" "$2"
exit $?
;;
restart|force-reload)
do_op "slave_op" "restart" "Restarting buildslave" "$2"
exit $?
;;
*)
echo "Usage: $0 {start|stop|restart|reload|force-reload}"
exit 1
;;
esac
exit 0

23
slave/contrib/os-x/README
View File

@@ -1,23 +0,0 @@
Mark Pauley contributed the two launchd plist files for OS-X (10.4+) to start
a buildmaster or buildslave automatically at startup:
contrib/OS-X/net.sourceforge.buildbot.master.plist
contrib/OS-X/net.sourceforge.buildbot.slave.plist
His email message is as follows:
Message-Id: <C0E57556-0432-4EB6-9A6C-22CDC72208E9@apple.com>
From: Mark Pauley <mpauley@apple.com>
To: buildbot-devel <buildbot-devel@lists.sourceforge.net>
Date: Wed, 24 Jan 2007 11:05:44 -0800
Subject: [Buildbot-devel] Sample buildbot launchd plists for MacOS 10.4+
Hi guys,
I've had these kicking around for a while and thought that maybe
someone would like to see them. Installing either of these two to /
Library/LaunchDaemons will cause the bulidbot slave or master to auto-
start as whatever user you like on launch. This is the "right way to
do this" going forward, startupitems are deprecated. Please note that
this means any tests that require a windowserver connection on os x
won't work.

36
slave/contrib/os-x/net.sourceforge.buildbot.slave.plist
View File

@@ -1,36 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple Computer//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd
">
<plist version="1.0">
<dict>
<key>Label</key>
<string>net.sourceforge.buildbot.slave</string>
<!-- Change this to the user you want to run buildbot as -->
<key>UserName</key>
<string>buildbot</string>
<!-- Change this to your buildbot working directory -->
<key>WorkingDirectory</key>
<string>/Users/buildbot/Buildbot_Slave</string>
<key>ProgramArguments</key>
<array>
<string>/usr/bin/twistd</string>
<string>--nodaemon</string>
<string>--python=buildbot.tac</string>
<string>--logfile=buildbot.log</string>
<string>--prefix=slave</string>
</array>
<key>KeepAlive</key>
<dict>
<key>SuccessfulExit</key>
<false/>
</dict>
<key>RunAtLoad</key>
<true/>
</dict>
</plist>

17
slave/contrib/systemd/buildslave.service
View File

@@ -1,17 +0,0 @@
[Unit]
Description=Buildbot Slave
Wants=network.target
After=network.target
[Service]
Type=forking
PIDFile=/srv/buildslave/linux-slave/twistd.pid
WorkingDirectory=/srv/buildslave
ExecStart=/usr/bin/buildslave start linux-slave
ExecReload=/usr/bin/buildslave restart linux-slave
ExecStop=/usr/bin/buildslave stop linux-slave
Restart=always
User=buildslave
[Install]
WantedBy=multi-user.target

556
slave/contrib/windows/buildbot_service.py
View File

@@ -1,556 +0,0 @@
# Runs the build-bot as a Windows service.
# To use:
# * Install and configure buildbot as per normal (ie, running
# 'setup.py install' from the source directory).
#
# * Configure any number of build-bot directories (slaves or masters), as
# per the buildbot instructions. Test these directories normally by
# using the (possibly modified) "buildbot.bat" file and ensure everything
# is working as expected.
#
# * Install the buildbot service. Execute the command:
# % python buildbot_service.py
# To see installation options. You probably want to specify:
# + --username and --password options to specify the user to run the
# + --startup auto to have the service start at boot time.
#
# For example:
# % python buildbot_service.py --user mark --password secret \
# --startup auto install
# Alternatively, you could execute:
# % python buildbot_service.py install
# to install the service with default options, then use Control Panel
# to configure it.
#
# * Start the service specifying the name of all buildbot directories as
# service args. This can be done one of 2 ways:
# - Execute the command:
# % python buildbot_service.py start "dir_name1" "dir_name2"
# or:
# - Start Control Panel->Administrative Tools->Services
# - Locate the previously installed buildbot service.
# - Open the "properties" for the service.
# - Enter the directory names into the "Start Parameters" textbox. The
# directory names must be fully qualified, and surrounded in quotes if
# they include spaces.
# - Press the "Start"button.
# Note that the service will automatically use the previously specified
# directories if no arguments are specified. This means the directories
# need only be specified when the directories to use have changed (and
# therefore also the first time buildbot is configured)
#
# * The service should now be running. You should check the Windows
# event log. If all goes well, you should see some information messages
# telling you the buildbot has successfully started.
#
# * If you change the buildbot configuration, you must restart the service.
# There is currently no way to ask a running buildbot to reload the
# config. You can restart by executing:
# % python buildbot_service.py restart
#
# Troubleshooting:
# * Check the Windows event log for any errors.
# * Check the "twistd.log" file in your buildbot directories - once each
# bot has been started it just writes to this log as normal.
# * Try executing:
# % python buildbot_service.py debug
# This will execute the buildbot service in "debug" mode, and allow you to
# see all messages etc generated. If the service works in debug mode but
# not as a real service, the error probably relates to the environment or
# permissions of the user configured to run the service (debug mode runs as
# the currently logged in user, not the service user)
# * Ensure you have the latest pywin32 build available, at least version 206.
# Written by Mark Hammond, 2006.
import os
import sys
import threading
import pywintypes
import servicemanager
import win32api
import win32con
import win32event
import win32file
import win32pipe
import win32process
import win32security
import win32service
import win32serviceutil
import winerror
# Are we running in a py2exe environment?
is_frozen = hasattr(sys, "frozen")
# Taken from the Zope service support - each "child" is run as a sub-process
# (trying to run multiple twisted apps in the same process is likely to screw
# stdout redirection etc).
# Note that unlike the Zope service, we do *not* attempt to detect a failed
# client and perform restarts - buildbot itself does a good job
# at reconnecting, and Windows itself provides restart semantics should
# everything go pear-shaped.
# We execute a new thread that captures the tail of the output from our child
# process. If the child fails, it is written to the event log.
# This process is unconditional, and the output is never written to disk
# (except obviously via the event log entry)
# Size of the blocks we read from the child process's output.
CHILDCAPTURE_BLOCK_SIZE = 80
# The number of BLOCKSIZE blocks we keep as process output.
CHILDCAPTURE_MAX_BLOCKS = 200
class BBService(win32serviceutil.ServiceFramework):
_svc_name_ = 'BuildBot'
_svc_display_name_ = _svc_name_
_svc_description_ = 'Manages local buildbot slaves and masters - ' \
'see http://buildbot.sourceforge.net'
def __init__(self, args):
win32serviceutil.ServiceFramework.__init__(self, args)
# Create an event which we will use to wait on. The "service stop"
# request will set this event.
# * We must make it inheritable so we can pass it to the child
# process via the cmd-line
# * Must be manual reset so each child process and our service
# all get woken from a single set of the event.
sa = win32security.SECURITY_ATTRIBUTES()
sa.bInheritHandle = True
self.hWaitStop = win32event.CreateEvent(sa, True, False, None)
self.args = args
self.dirs = None
self.runner_prefix = None
# Patch up the service messages file in a frozen exe.
# (We use the py2exe option that magically bundles the .pyd files
# into the .zip file - so servicemanager.pyd doesn't exist.)
if is_frozen and servicemanager.RunningAsService():
msg_file = os.path.join(os.path.dirname(sys.executable),
"buildbot.msg")
if os.path.isfile(msg_file):
servicemanager.Initialize("BuildBot", msg_file)
else:
self.warning("Strange - '%s' does not exist" % (msg_file, ))
def _checkConfig(self):
# Locate our child process runner (but only when run from source)
if not is_frozen:
# Running from source
python_exe = os.path.join(sys.prefix, "python.exe")
if not os.path.isfile(python_exe):
# for ppl who build Python itself from source.
python_exe = os.path.join(sys.prefix, "PCBuild", "python.exe")
if not os.path.isfile(python_exe):
# virtualenv support
python_exe = os.path.join(sys.prefix, "Scripts", "python.exe")
if not os.path.isfile(python_exe):
self.error("Can not find python.exe to spawn subprocess")
return False
me = __file__
if me.endswith(".pyc") or me.endswith(".pyo"):
me = me[:-1]
self.runner_prefix = '"%s" "%s"' % (python_exe, me)
else:
# Running from a py2exe built executable - our child process is
# us (but with the funky cmdline args!)
self.runner_prefix = '"' + sys.executable + '"'
# Now our arg processing - this may be better handled by a
# twisted/buildbot style config file - but as of time of writing,
# MarkH is clueless about such things!
# Note that the "arguments" you type into Control Panel for the
# service do *not* persist - they apply only when you click "start"
# on the service. When started by Windows, args are never presented.
# Thus, it is the responsibility of the service to persist any args.
# so, when args are presented, we save them as a "custom option". If
# they are not presented, we load them from the option.
self.dirs = []
if len(self.args) > 1:
dir_string = os.pathsep.join(self.args[1:])
save_dirs = True
else:
dir_string = win32serviceutil.GetServiceCustomOption(self,
"directories")
save_dirs = False
if not dir_string:
self.error("You must specify the buildbot directories as "
"parameters to the service.\nStopping the service.")
return False
dirs = dir_string.split(os.pathsep)
for d in dirs:
d = os.path.abspath(d)
sentinal = os.path.join(d, "buildbot.tac")
if os.path.isfile(sentinal):
self.dirs.append(d)
else:
msg = "Directory '%s' is not a buildbot dir - ignoring" \
% (d, )
self.warning(msg)
if not self.dirs:
self.error("No valid buildbot directories were specified.\n"
"Stopping the service.")
return False
if save_dirs:
dir_string = os.pathsep.join(self.dirs).encode("mbcs")
win32serviceutil.SetServiceCustomOption(self, "directories",
dir_string)
return True
def SvcStop(self):
# Tell the SCM we are starting the stop process.
self.ReportServiceStatus(win32service.SERVICE_STOP_PENDING)
# Set the stop event - the main loop takes care of termination.
win32event.SetEvent(self.hWaitStop)
# SvcStop only gets triggered when the user explicitly stops (or restarts)
# the service. To shut the service down cleanly when Windows is shutting
# down, we also need to hook SvcShutdown.
SvcShutdown = SvcStop
def SvcDoRun(self):
if not self._checkConfig():
# stopped status set by caller.
return
self.logmsg(servicemanager.PYS_SERVICE_STARTED)
child_infos = []
for bbdir in self.dirs:
self.info("Starting BuildBot in directory '%s'" % (bbdir, ))
hstop = self.hWaitStop
cmd = '%s --spawn %d start --nodaemon %s' % (self.runner_prefix, hstop, bbdir)
# print "cmd is", cmd
h, t, output = self.createProcess(cmd)
child_infos.append((bbdir, h, t, output))
while child_infos:
handles = [self.hWaitStop] + [i[1] for i in child_infos]
rc = win32event.WaitForMultipleObjects(handles,
0, # bWaitAll
win32event.INFINITE)
if rc == win32event.WAIT_OBJECT_0:
# user sent a stop service request
break
else:
# A child process died. For now, just log the output
# and forget the process.
index = rc - win32event.WAIT_OBJECT_0 - 1
bbdir, dead_handle, dead_thread, output_blocks = \
child_infos[index]
status = win32process.GetExitCodeProcess(dead_handle)
output = "".join(output_blocks)
if not output:
output = "The child process generated no output. " \
"Please check the twistd.log file in the " \
"indicated directory."
self.warning("BuildBot for directory %r terminated with "
"exit code %d.\n%s" % (bbdir, status, output))
del child_infos[index]
if not child_infos:
self.warning("All BuildBot child processes have "
"terminated. Service stopping.")
# Either no child processes left, or stop event set.
self.ReportServiceStatus(win32service.SERVICE_STOP_PENDING)
# The child processes should have also seen our stop signal
# so wait for them to terminate.
for bbdir, h, t, output in child_infos:
for i in range(10): # 30 seconds to shutdown...
self.ReportServiceStatus(win32service.SERVICE_STOP_PENDING)
rc = win32event.WaitForSingleObject(h, 3000)
if rc == win32event.WAIT_OBJECT_0:
break
# Process terminated - no need to try harder.
if rc == win32event.WAIT_OBJECT_0:
break
self.ReportServiceStatus(win32service.SERVICE_STOP_PENDING)
# If necessary, kill it
if win32process.GetExitCodeProcess(h) == win32con.STILL_ACTIVE:
self.warning("BuildBot process at %r failed to terminate - "
"killing it" % (bbdir, ))
win32api.TerminateProcess(h, 3)
self.ReportServiceStatus(win32service.SERVICE_STOP_PENDING)
# Wait for the redirect thread - it should have died as the remote
# process terminated.
# As we are shutting down, we do the join with a little more care,
# reporting progress as we wait (even though we never will <wink>)
for i in range(5):
t.join(1)
self.ReportServiceStatus(win32service.SERVICE_STOP_PENDING)
if not t.isAlive():
break
else:
self.warning("Redirect thread did not stop!")
# All done.
self.logmsg(servicemanager.PYS_SERVICE_STOPPED)
#
# Error reporting/logging functions.
#
def logmsg(self, event):
# log a service event using servicemanager.LogMsg
try:
servicemanager.LogMsg(servicemanager.EVENTLOG_INFORMATION_TYPE,
event,
(self._svc_name_,
" (%s)" % self._svc_display_name_))
except win32api.error, details:
# Failed to write a log entry - most likely problem is
# that the event log is full. We don't want this to kill us
try:
print "FAILED to write INFO event", event, ":", details
except IOError:
# No valid stdout! Ignore it.
pass
def _dolog(self, func, msg):
try:
func(msg)
except win32api.error, details:
# Failed to write a log entry - most likely problem is
# that the event log is full. We don't want this to kill us
try:
print "FAILED to write event log entry:", details
print msg
except IOError:
pass
def info(self, s):
self._dolog(servicemanager.LogInfoMsg, s)
def warning(self, s):
self._dolog(servicemanager.LogWarningMsg, s)
def error(self, s):
self._dolog(servicemanager.LogErrorMsg, s)
# Functions that spawn a child process, redirecting any output.
# Although buildbot itself does this, it is very handy to debug issues
# such as ImportErrors that happen before buildbot has redirected.
def createProcess(self, cmd):
hInputRead, hInputWriteTemp = self.newPipe()
hOutReadTemp, hOutWrite = self.newPipe()
pid = win32api.GetCurrentProcess()
# This one is duplicated as inheritable.
hErrWrite = win32api.DuplicateHandle(pid, hOutWrite, pid, 0, 1,
win32con.DUPLICATE_SAME_ACCESS)
# These are non-inheritable duplicates.
hOutRead = self.dup(hOutReadTemp)
hInputWrite = self.dup(hInputWriteTemp)
# dup() closed hOutReadTemp, hInputWriteTemp
si = win32process.STARTUPINFO()
si.hStdInput = hInputRead
si.hStdOutput = hOutWrite
si.hStdError = hErrWrite
si.dwFlags = win32process.STARTF_USESTDHANDLES | \
win32process.STARTF_USESHOWWINDOW
si.wShowWindow = win32con.SW_HIDE
# pass True to allow handles to be inherited. Inheritance is
# problematic in general, but should work in the controlled
# circumstances of a service process.
create_flags = win32process.CREATE_NEW_CONSOLE
# info is (hProcess, hThread, pid, tid)
info = win32process.CreateProcess(None, cmd, None, None, True,
create_flags, None, None, si)
# (NOTE: these really aren't necessary for Python - they are closed
# as soon as they are collected)
hOutWrite.Close()
hErrWrite.Close()
hInputRead.Close()
# We don't use stdin
hInputWrite.Close()
# start a thread collecting output
blocks = []
t = threading.Thread(target=self.redirectCaptureThread,
args=(hOutRead, blocks))
t.start()
return info[0], t, blocks
def redirectCaptureThread(self, handle, captured_blocks):
# One of these running per child process we are watching. It
# handles both stdout and stderr on a single handle. The read data is
# never referenced until the thread dies - so no need for locks
# around self.captured_blocks.
# self.info("Redirect thread starting")
while True:
try:
ec, data = win32file.ReadFile(handle, CHILDCAPTURE_BLOCK_SIZE)
except pywintypes.error, err:
# ERROR_BROKEN_PIPE means the child process closed the
# handle - ie, it terminated.
if err[0] != winerror.ERROR_BROKEN_PIPE:
self.warning("Error reading output from process: %s" % err)
break
captured_blocks.append(data)
del captured_blocks[CHILDCAPTURE_MAX_BLOCKS:]
handle.Close()
# self.info("Redirect capture thread terminating")
def newPipe(self):
sa = win32security.SECURITY_ATTRIBUTES()
sa.bInheritHandle = True
return win32pipe.CreatePipe(sa, 0)
def dup(self, pipe):
# create a duplicate handle that is not inherited, so that
# it can be closed in the parent. close the original pipe in
# the process.
pid = win32api.GetCurrentProcess()
dup = win32api.DuplicateHandle(pid, pipe, pid, 0, 0,
win32con.DUPLICATE_SAME_ACCESS)
pipe.Close()
return dup
# Service registration and startup
def RegisterWithFirewall(exe_name, description):
# Register our executable as an exception with Windows Firewall.
# taken from http://msdn.microsoft.com/library/default.asp?url=\
# /library/en-us/ics/ics/wf_adding_an_application.asp
from win32com.client import Dispatch
# Scope
NET_FW_SCOPE_ALL = 0
# IP Version - ANY is the only allowable setting for now
NET_FW_IP_VERSION_ANY = 2
fwMgr = Dispatch("HNetCfg.FwMgr")
# Get the current profile for the local firewall policy.
profile = fwMgr.LocalPolicy.CurrentProfile
app = Dispatch("HNetCfg.FwAuthorizedApplication")
app.ProcessImageFileName = exe_name
app.Name = description
app.Scope = NET_FW_SCOPE_ALL
# Use either Scope or RemoteAddresses, but not both
# app.RemoteAddresses = "*"
app.IpVersion = NET_FW_IP_VERSION_ANY
app.Enabled = True
# Use this line if you want to add the app, but disabled.
# app.Enabled = False
profile.AuthorizedApplications.Add(app)
# A custom install function.
def CustomInstall(opts):
# Register this process with the Windows Firewaall
import pythoncom
try:
RegisterWithFirewall(sys.executable, "BuildBot")
except pythoncom.com_error, why:
print "FAILED to register with the Windows firewall"
print why
# Magic code to allow shutdown. Note that this code is executed in
# the *child* process, by way of the service process executing us with
# special cmdline args (which includes the service stop handle!)
def _RunChild(runfn):
del sys.argv[1] # The --spawn arg.
# Create a new thread that just waits for the event to be signalled.
t = threading.Thread(target=_WaitForShutdown,
args=(int(sys.argv[1]), )
)
del sys.argv[1] # The stop handle
# This child process will be sent a console handler notification as
# users log off, or as the system shuts down. We want to ignore these
# signals as the service parent is responsible for our shutdown.
def ConsoleHandler(what):
# We can ignore *everything* - ctrl+c will never be sent as this
# process is never attached to a console the user can press the
# key in!
return True
win32api.SetConsoleCtrlHandler(ConsoleHandler, True)
t.setDaemon(True) # we don't want to wait for this to stop!
t.start()
if hasattr(sys, "frozen"):
# py2exe sets this env vars that may screw our child process - reset
del os.environ["PYTHONPATH"]
# Start the buildbot/buildslave app
runfn()
print "Service child process terminating normally."
def _WaitForShutdown(h):
win32event.WaitForSingleObject(h, win32event.INFINITE)
print "Shutdown requested"
from twisted.internet import reactor
reactor.callLater(0, reactor.stop)
def DetermineRunner(bbdir):
'''Checks if the given directory is a buildslave or a master and returns the
appropriate run function.'''
try:
import buildslave.scripts.runner
tacfile = os.path.join(bbdir, 'buildbot.tac')
if os.path.exists(tacfile):
with open(tacfile, 'r') as f:
contents = f.read()
if 'import BuildSlave' in contents:
return buildslave.scripts.runner.run
except ImportError:
# Use the default
pass
import buildbot.scripts.runner
return buildbot.scripts.runner.run
# This function is also called by the py2exe startup code.
def HandleCommandLine():
if len(sys.argv) > 1 and sys.argv[1] == "--spawn":
# Special command-line created by the service to execute the
# child-process.
# First arg is the handle to wait on
# Fourth arg is the config directory to use for the buildbot/slave
_RunChild(DetermineRunner(sys.argv[5]))
else:
win32serviceutil.HandleCommandLine(BBService,
customOptionHandler=CustomInstall)
if __name__ == '__main__':
HandleCommandLine()

22
slave/contrib/windows/buildslave.bat
View File

@@ -1,22 +0,0 @@
@echo off
REM This file is used to run buildslave when installed into a python installation or deployed in virtualenv
setlocal
set BB_BUILDSLAVE="%~dp0buildslave"
IF EXIST "%~dp0..\python.exe" (
REM Normal system install of python (buildslave.bat is in scripts dir, just below python.exe)
set BB_PYTHON="%~dp0..\python"
) ELSE IF EXIST "%~dp0python.exe" (
REM virtualenv install (buildslave.bat is in same dir as python.exe)
set BB_PYTHON="%~dp0python"
) ELSE (
REM Not found nearby. Use system version and hope for the best
echo Warning! Unable to find python.exe near buildslave.bat. Using python on PATH, which might be a mismatch.
echo.
set BB_PYTHON=python
)
%BB_PYTHON% %BB_BUILDSLAVE% %*
exit /b %ERRORLEVEL%

30
slave/contrib/zsh/_buildslave
View File

@@ -1,30 +0,0 @@
#compdef buildslave
#
# This is the ZSH completion file for 'buildslave' command. It calls
# 'buildslave' command with the special "--_shell-completion" option which is
# handled by twisted.python.usage. t.p.usage then generates zsh code on stdout
# to handle the completions.
#
# This file is derived from twisted/python/twisted-completion.zsh from twisted
# distribution.
#
# redirect stderr to /dev/null otherwise deprecation warnings may get puked all
# over the user's terminal if completing options for a deprecated command.
# Redirect stderr to a file to debug errors.
local cmd output
cmd=("$words[@]" --_shell-completion zsh:$CURRENT)
output=$("$cmd[@]" 2>/dev/null)
if [[ $output == "#compdef "* ]]; then
# Looks like we got a valid completion function - so eval it to produce
# the completion matches.
eval $output
else
echo "\nCompletion error running command:" ${(qqq)cmd}
echo -n "If output below is unhelpful you may need to edit this file and "
echo "redirect stderr to a file."
echo "Expected completion function, but instead got:"
echo $output
return 1
fi

192
slave/docs/buildslave.1
View File

@@ -1,192 +0,0 @@
.\" This file is part of Buildbot. Buildbot is free software: you can
.\" redistribute it and/or modify it under the terms of the GNU General Public
.\" License as published by the Free Software Foundation, version 2.
.\"
.\" This program is distributed in the hope that it will be useful, but WITHOUT
.\" ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
.\" FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
.\" details.
.\"
.\" You should have received a copy of the GNU General Public License along with
.\" this program; if not, write to the Free Software Foundation, Inc., 51
.\" Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
.\"
.\" Copyright Buildbot Team Members
.TH BUILDSLAVE "1" "August 2010" "Buildbot" "User Commands"
.SH NAME
buildslave \- a tool for managing buildbot slave instances
.SH SYNOPSIS
.PP
.B buildslave
[
.BR "global options"
]
.I command
[
.BR "command options"
]
.PP
.B buildslave
create-slave
[
.BR \-q | \-\-quiet
]
[
.BR \-f | \-\-force
]
[
.BR \-r | \-\-relocatable
]
[
.BR \-n | \-\-no-logrotate
]
[
.BR \-k | \-\-keepalive
.I TIME
]
[
.BR --usepty
{0|1}
]
[
.BR \-\-umask
.I UMASK
]
[
.BR \-s | \-\-log-size
.I SIZE
]
[
.BR \-l | \-\-log-count
.I COUNT
]
[
.BR \-\-verbose
]
.I PATH
.I MASTER
.I USERNAME
.I PASSWORD
.PP
.B buildslave
[
.BR \-\-verbose
]
{
.BR start | stop | restart
}
[
.I PATH
]
.PP
.B buildslave
[
.BR \-\-verbose
]
{
.BR \-\-help | \-\-version
}
.PP
.B buildslave
.I command
.BR \-h | \-\-help
.SH DESCRIPTION
.\" Putting a newline after each sentence can generate better output.
The `buildslave' command-line tool can be used to start or stop a
buildslave or create a new buildslave instance.
.SH OPTIONS
.SS Commands
.TP
.BR create-slave
Create and populate a directory for a new buildslave
.TP
.BR start
Start a buildslave
.TP
.BR stop
Stop a buildslave
.TP
.BR restart
Restart a buildslave
.SS Global options
.TP
.BR \-h | \-\-help
Print the list of available commands and global options.
All subsequent commands are ignored.
.TP
.BR --version
Print twistd and buildslave version.
All subsequent commands are ignored.
.TP
.BR --verbose
Verbose output.
.SS create-slave command options
.TP
.BR \-f | \-\-force
Re-use an existing directory.
.TP
.BR \-h | \-\-help
Show help for current command and exit.
All subsequent commands are ignored.
.TP
.BR \-k | \-\-keepalive
Send keepalive requests to buildmaster every
.I TIME
seconds.
Default value is 600 seconds.
.TP
.BR \-l | \-\-log-count
Limit the number of kept old twisted log files to
.IR COUNT .
All files are kept by default.
.TP
.BR \-q | \-\-quiet
Do not emit the commands being run.
.TP
.BR \-r | \-\-relocatable
Create a relocatable buildbot.tac.
.TP
.BR \-n | \-\-no-logrotate
Do not permit buildslave rotate logs by itself.
.TP
.BR \-s | \-\-log-size
Set size at which twisted lof file is rotated to
.I SIZE
bytes.
Default value is 1000000 bytes.
.TP
.BR \-\-umask
Set umask for files created by buildslave.
Default value is 077 which means only owner can access the files.
See
.BR umask (2)
for more details.
.TP
.BR \-\-usepty
Set wether child processes should be run in a pty (0 means do not run in a
pty).
Default value is 0.
.TP
.I PATH
Path to buildslave base directory.
.TP
.I MASTER
Set the host and port of buildbot master to attach to in form
.IR HOST:PORT .
This should be provided by buildmaster administrator.
.TP
.I USERNAME
Buildslave name to connect with.
This should be provided by buildmaster administrator.
.TP
.I PASSWORD
Buildslave password to connect with.
This should be provided by buildmaster administrator.
.SH "SEE ALSO"
.BR buildbot (1),
.BR umask (2),
.PP
The complete documentation is available in texinfo format. To use it, run
.BR "info buildbot" .

2
slave/setup.cfg
View File

@@ -1,2 +0,0 @@
[aliases]
test = trial -m buildslave

153
slave/setup.py
View File

@@ -1,153 +0,0 @@
#!/usr/bin/env python
#
# This file is part of Buildbot. Buildbot is free software: you can
# redistribute it and/or modify it under the terms of the GNU General Public
# License as published by the Free Software Foundation, version 2.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 51
# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
# Copyright Buildbot Team Members
"""
Standard setup script.
"""
import os
import sys
from distutils.command.install_data import install_data
from distutils.command.sdist import sdist
from distutils.core import setup
from buildslave import version
scripts = ["bin/buildslave"]
# sdist is usually run on a non-Windows platform, but the buildslave.bat file
# still needs to get packaged.
if 'sdist' in sys.argv or sys.platform == 'win32':
scripts.append("contrib/windows/buildslave.bat")
scripts.append("contrib/windows/buildbot_service.py")
class our_install_data(install_data):
def finalize_options(self):
self.set_undefined_options('install',
('install_lib', 'install_dir'),
)
install_data.finalize_options(self)
def run(self):
install_data.run(self)
# ensure there's a buildslave/VERSION file
fn = os.path.join(self.install_dir, 'buildslave', 'VERSION')
open(fn, 'w').write(version)
self.outfiles.append(fn)
class our_sdist(sdist):
def make_release_tree(self, base_dir, files):
sdist.make_release_tree(self, base_dir, files)
# ensure there's a buildslave/VERSION file
fn = os.path.join(base_dir, 'buildslave', 'VERSION')
open(fn, 'w').write(version)
# ensure that NEWS has a copy of the latest release notes, copied from
# the master tree, with the proper version substituted
src_fn = os.path.join('..', 'master', 'docs', 'relnotes/index.rst')
src = open(src_fn).read()
src = src.replace('|version|', version)
dst_fn = os.path.join(base_dir, 'NEWS')
open(dst_fn, 'w').write(src)
setup_args = {
'name': "buildbot-slave",
'version': version,
'description': "Buildbot Slave Daemon",
'long_description': "See the 'buildbot' package for details",
'author': "Brian Warner",
'author_email': "warner-buildbot@lothar.com",
'maintainer': "Dustin J. Mitchell",
'maintainer_email': "dustin@v.igoro.us",
'url': "http://buildbot.net/",
'license': "GNU GPL",
'classifiers': [
'Development Status :: 5 - Production/Stable',
'Environment :: No Input/Output (Daemon)',
'Intended Audience :: Developers',
'License :: OSI Approved :: GNU General Public License (GPL)',
'Topic :: Software Development :: Build Tools',
'Topic :: Software Development :: Testing',
],
'packages': [
"buildslave",
"buildslave.commands",
"buildslave.scripts",
"buildslave.monkeypatches",
"buildslave.test",
"buildslave.test.fake",
"buildslave.test.util",
"buildslave.test.unit",
],
'scripts': scripts,
# mention data_files, even if empty, so install_data is called and
# VERSION gets copied
'data_files': [("buildslave", [])],
'cmdclass': {
'install_data': our_install_data,
'sdist': our_sdist
}
}
# set zip_safe to false to force Windows installs to always unpack eggs
# into directories, which seems to work better --
# see http://buildbot.net/trac/ticket/907
if sys.platform == "win32":
setup_args['zip_safe'] = False
try:
# If setuptools is installed, then we'll add setuptools-specific arguments
# to the setup args.
import setuptools # @UnusedImport
except ImportError:
pass
else:
setup_args['install_requires'] = [
'twisted >= 8.0.0',
'future',
]
# Unit test hard dependencies.
test_deps = [
'mock',
]
setup_args['tests_require'] = test_deps
setup_args['extras_require'] = {
'test': [
'setuptools_trial',
'pep8',
'pylint==1.1.0',
'pyflakes',
] + test_deps,
}
if '--help-commands' in sys.argv or 'trial' in sys.argv or 'test' in sys.argv:
setup_args['setup_requires'] = [
'setuptools_trial',
]
if os.getenv('NO_INSTALL_REQS'):
setup_args['install_requires'] = None
setup_args['extras_require'] = None
setup(**setup_args)

10
slave/tox.ini
View File

@@ -1,10 +0,0 @@
# Tox (http://tox.testrun.org/) is a tool for running tests
# in multiple virtualenvs. This configuration file will run the
# test suite on all supported python versions. To use it, "pip install tox"
# and then run "tox" from this directory.
[tox]
envlist = py24, py25, py26, py27
[testenv]
commands = python setup.py test