dockerfiles: do not store the config in a volume

This new docker method will store the configuration in a github repository

while it is a little bit more difficult to edit for the total newbie, it is a much better
option for serious continuous deployment
This commit is contained in:
Pierre Tardy
2016-10-05 17:31:45 +02:00
parent 1b0fe163c7
commit 3d0f864d5f
18 changed files with 244 additions and 133 deletions

View File

@@ -1,9 +1,5 @@
# BBTravis CI configuration file
services:
- mysql
- postgresql
language: python
# Available Python versions:

View File

@@ -1,4 +1,5 @@
# developer utilities
DOCKERBUILD := docker build --build-arg http_proxy=$$http_proxy --build-arg https_proxy=$$https_proxy
.PHONY: docs pylint flake8
@@ -55,8 +56,14 @@ isort:
git diff --name-only --stat "HEAD" | grep '.py$$' | xargs autopep8 -i
git commit -a -m "isort+autopep8 run"
docker:
git ls-files | grep Dockerfile | while read line ;\
do \
(cd `dirname $${line}`; docker build . ) || exit 1 ;\
done
docker: docker-buildbot-worker docker-buildbot-worker-node docker-buildbot-master docker-buildbot-master-ubuntu
echo done
docker-buildbot-worker:
$(DOCKERBUILD) -t buildbot/buildbot-worker:master worker
docker-buildbot-worker-node:
$(DOCKERBUILD) -t buildbot/buildbot-worker-node:master master/contrib/docker/pythonnode_worker
docker-buildbot-master:
$(DOCKERBUILD) -t buildbot/buildbot-master:master master
docker-buildbot-master-ubuntu:
$(DOCKERBUILD) -t buildbot/buildbot-master-ubuntu:master -f master/Dockerfile.ubuntu master

View File

@@ -1,35 +1,49 @@
# buildbot/buildbot-master
# please follow docker best practices
# https://docs.docker.com/engine/userguide/eng-image/dockerfile_best-practices/
#
# Provides a base alpine 3.4 image with latest buildbot master installed
# Alpine base build is ~130MB vs ~500MB for the ubuntu build
# Note that the UI and worker packages are the latest version published on pypi
# This is to avoid pulling node inside this container
FROM alpine:3.4
MAINTAINER Buildbot maintainers
# Last build date - this can be updated whenever there are security updates so
# that everything is rebuilt
ENV security_updates_as_of 2016-08-25
ENV security_updates_as_of 2016-10-07
COPY . /usr/src/buildbot
# We install as much as possible python packages from the distro in order to avoid
# having to pull gcc for building native extensions
# Some packages are at the moment (08/2016) only available on @testing
# Some packages are at the moment (10/2016) only available on @testing
RUN \
echo @testing http://nl.alpinelinux.org/alpine/edge/testing >> /etc/apk/repositories && \
echo @edge http://nl.alpinelinux.org/alpine/edge/community >> /etc/apk/repositories && \
apk add --no-cache \
python \
py-pip \
py-psycopg2 py-twisted py-cryptography@edge \
py-service_identity@edge py-sqlalchemy@edge \
gosu@testing dumb-init@edge\
py-jinja2 && \
python -m ensurepip && \
rm -r /usr/lib/python*/ensurepip && \
python \
py-pip \
py-psycopg2 \
py-twisted \
py-cffi \
py-openssl@edge \
py-cryptography@edge \
py-service_identity@edge \
py-sqlalchemy@edge \
gosu@testing \
dumb-init@edge\
py-jinja2 \
tar \
curl && \
# install pip dependencies
pip install --upgrade pip setuptools && \
pip install --pre "buildbot[bundle]" && \
pip install "/usr/src/buildbot" && \
pip install "buildbot[bundle,tls]" && \
pip install "/usr/src/buildbot" && \
rm -r /root/.cache
WORKDIR /var/lib/buildbot
VOLUME /var/lib/buildbot
CMD ["/usr/src/buildbot/contrib/docker/master/start_buildbot.sh"]
CMD ["dumb-init", "/usr/src/buildbot/contrib/docker/master/start_buildbot.sh"]

View File

@@ -1,31 +1,37 @@
# buildbot/buildbot-master-ubuntu
# please follow docker best practices
# https://docs.docker.com/engine/userguide/eng-image/dockerfile_best-practices/
# Provides a base Ubuntu (16.04) image with latest buildbot master installed
# (based on twisted dockerfile https://github.com/cyli/docker-twisted)
FROM ubuntu:16.04
MAINTAINER Buildbot maintainers
# Last build date - this can be updated whenever there are security updates so
# that everything is rebuilt
ENV security_updates_as_of 2016-07-30
ENV security_updates_as_of 2016-10-07
ADD . /usr/src/buildbot
# This will make apt-get install without question (only for the time of the docker build)
ARG DEBIAN_FRONTEND=noninteractive
COPY . /usr/src/buildbot
# Install security updates and required packages
RUN apt-get update && \
apt-get -y upgrade && \
apt-get -y install -q \
build-essential \
python-dev libffi-dev libssl-dev python-pip \
python-psycopg2
curl \
python-dev \
libffi-dev \
libssl-dev \
python-pip \
python-psycopg2 && \
rm -rf /var/lib/apt/lists/* && \
# Install required python packages, and twisted
RUN pip install --upgrade pip setuptools && \
pip install service_identity pycrypto && \
pip install twisted && \
pip install --pre "buildbot[bundle]" && \
pip install "/usr/src/buildbot"
pip install --upgrade pip setuptools && \
pip install "buildbot[bundle,tls]" && \
pip install "/usr/src/buildbot"
WORKDIR /var/lib/buildbot
VOLUME /var/lib/buildbot
CMD ["/usr/src/buildbot/contrib/docker/master/start_buildbot.sh"]

View File

@@ -24,6 +24,22 @@ Documentation
See http://docs.buildbot.net/current/ for documentation of the current version of Buildbot.
Docker container
----------------
Buildbot comes with a ready to use docker container available at buildbot/buildbot-master
Following environment variables are supported for configuration:
* ``BUILDBOT_CONFIG_URL``: http url to a config tarball.
The tarball must be in the .tar.gz format.
The tarball must contain a directory, which will contain a master.cfg file in it.
The tarball may contain a twisted.tac file in it, which can be used to configure the twisted logging system (e.g to log in logstash instead of the default stdout).
The tarball will be extracted in a directory named ``$BUILDBOT_CONFIG_DIR`` in the master directory, and can contain additional python module that the master.cfg can load.
If ``BUILDBOT_CONFIG_URL`` does not end with .tar.gz, it is considered to be an URL to the direct ``master.cfg``
* ``BUILDBOT_CONFIG_DIR`` directory where to extract the config tarball within the master directory.
It is important so that you can do relative imports in your master.cfg like it is done in the metabbotcfg (https://github.com/buildbot/metabbotcfg)
Requirements
------------

View File

@@ -1,4 +0,0 @@
# database parameters are shared between containers
POSTGRES_PASSWORD=change_me
POSTGRES_USER=buildbot
POSTGRES_DB=buildbot

View File

@@ -1,38 +0,0 @@
postgres:
env_file: db.env
image: "postgres:9.4"
volumes:
- ./buildbot_db:/var/lib/postgresql/data
expose:
- 5432
buildbot:
# switch between local build and dockerhub image
# build: ../../Dockerfile
# build: ../../Dockerfile.ubuntu
image: "buildbot/buildbot-master:master"
env_file: db.env
ports:
- "8010:8010"
# for external non-docker workers, uncomment the following line
# - "9989:9989"
expose:
- 9989
links:
- postgres
volumes:
- /var/lib/buildbot/:/var/lib/buildbot/
worker:
# switch between local build and dockerhub image
# build: ../../../worker/Dockerfile
image: "buildbot/buildbot-worker:master"
environment:
BUILDMASTER: buildbot
BUILDMASTER_PORT: 9989
WORKERNAME: example-worker
WORKERPASS: pass
WORKER_ENVIRONMENT_BLACKLIST: DOCKER_BUILDBOT* BUILDBOT_ENV_* BUILDBOT_1*
links:
- buildbot

View File

@@ -1,22 +1,54 @@
#!/bin/sh
# startup script for purely stateless master
# we download the config from an arbitrary curl accessible tar.gz file (which github can generate for us)
B=`pwd`
if [ -z "$BUILDBOT_CONFIG_URL" ]
then
if [ ! -f "$B/master.cfg" ]
then
echo No master.cfg found nor $$BUILDBOT_CONFIG_URL !
echo Please provide a master.cfg file in $B or provide a $$BUILDBOT_CONFIG_URL variable via -e
exit 1
fi
else
BUILDBOT_CONFIG_DIR=${BUILDBOT_CONFIG_DIR:-config}
mkdir -p $B/$BUILDBOT_CONFIG_DIR
# if it ends with .tar.gz then its a tarball, else its directly the file
if echo "$BUILDBOT_CONFIG_URL" | grep '.tar.gz$' >/dev/null
then
until curl -sL $BUILDBOT_CONFIG_URL | tar -xz --strip-components=1 --directory=$B/$BUILDBOT_CONFIG_DIR
do
echo "Can't download from \$BUILDBOT_CONFIG_URL: $BUILDBOT_CONFIG_URL"
sleep 1
done
if [ -f $B/$BUILDBOT_CONFIG_DIR/buildbot.tac ]
then
ln -sf $B/$BUILDBOT_CONFIG_DIR/buildbot.tac $B/buildbot.tac
fi
else
until curl -sL $BUILDBOT_CONFIG_URL > $B/master.cfg
do
echo "Can't download from $$BUILDBOT_CONFIG_URL: $BUILDBOT_CONFIG_URL"
done
fi
fi
# copy the default buildbot.tac if not provided by the config
if [ ! -f $B/buildbot.tac ]
then
buildbot create-master --db="postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@postgres/$POSTGRES_DB" $B
mv /usr/src/buildbot/buildbot/scripts/sample.cfg $B/master.cfg
cp /usr/src/buildbot/contrib/docker/master/buildbot.tac $B
echo
echo buildbot is now setup on the docker host in /var/lib/buildbot
echo
echo You can now edit the configuration file there to sweet your needs!
echo
echo
fi
# wait for pg to start by trying to upgrade the master
for i in `seq 100`
# wait for db to start by trying to upgrade the master
until buildbot upgrade-master $B
do
buildbot upgrade-master $B && break
echo "Can't upgrade master yet. Waiting for database ready?"
sleep 1
done
# we use exec so that twistd use the pid 1 of the container, and so that signals are properly forwarded
exec twistd -ny $B/buildbot.tac

View File

@@ -3,13 +3,18 @@
# This example docker file show how to customize the base worker docker image
# to add build dependencies to build the python+nodejs buildbot_www package
FROM buildbot/buildbot-worker
FROM buildbot/buildbot-worker:master
MAINTAINER Buildbot maintainers
# Install required npm packages
RUN apt-get install -y python-software-properties software-properties-common && \
add-apt-repository -y ppa:chris-lea/node.js && \
apt-get update
# This will make apt-get install without question
ARG DEBIAN_FRONTEND=noninteractive
RUN apt-get install -y -o APT::Install-Recommends=false -o APT::Install-Suggests=false \
nodejs git libmysqlclient-dev
user root
# Install required npm packages
RUN curl -sL https://deb.nodesource.com/setup_6.x | bash - && \
apt-get install -y -o APT::Install-Recommends=false -o APT::Install-Suggests=false \
nodejs \
git && \
rm -rf /var/lib/apt/lists/*
user buildbot

View File

@@ -391,8 +391,8 @@ Here is an nginx configuration that is known to work (nginx 1.6.2):
server {
# Enable SSL and SPDY
listen 443 ssl spdy default_server;
# Enable SSL and http2
listen 443 ssl http2 default_server;
server_name yourdomain.com;
@@ -406,7 +406,6 @@ Here is an nginx configuration that is known to work (nginx 1.6.2):
# put a one day session timeout for websockets to stay longer
ssl_session_cache shared:SSL:1440m;
ssl_session_timeout 1440m;
add_header Alternate-Protocol 443:npn-spdy/3;
# please consult latest nginx documentation for current secure encryption settings
ssl_protocols ..

View File

@@ -37,6 +37,14 @@ Installation
------------
* Use the `Docker installation instructions <https://docs.docker.com/engine/installation/>`_ for your operating system.
* Make sure you install docker-compose.
As root or inside a virtualenv, run:
.. code-block:: bash
pip install docker-compose
* Test docker is happy in your environment:
.. code-block:: bash
@@ -48,11 +56,11 @@ Building and running Buildbot
.. code-block:: bash
# Download Buildbot docker-compose.yml.
wget https://raw.github.com/buildbot/buildbot/master/master/contrib/docker/docker-compose.yml
wget https://raw.github.com/buildbot/buildbot/master/master/contrib/docker/db.env
# clone the example repository
git clone --depth 1 https://github.com/buildbot/buildbot-docker-example-config
# Build the Buildbot container (it will take a few minutes to download packages)
cd buildbot-docker-example-config/simple
docker-compose up
@@ -61,7 +69,7 @@ You should now be able to go to http://localhost:8010 and see a web page similar
.. image:: _images/index.png
:alt: index page
Click on the `Waterfall Display link <http://localhost:8010/waterfall>`_ and you get this:
Click on the `Waterfall Display link <http://localhost:8010/#/waterfall>`_ and you get this:
.. image:: _images/waterfall-empty.png
:alt: empty waterfall.
@@ -79,18 +87,23 @@ This docker-compose configuration is made as a basis for what you would put in p
- Containers are linked together so that the only port exposed to external is the web server
- The default master container is based on Alpine linux for minimal footprint
- The default worker container is based on more widely known Ubuntu distribution, as this is the container you want to customize.
- Download the config from a tarball accessible via a web server
Playing with your Buildbot containers
-------------------------------------
If you've come this far, you have a Buildbot environment that you can freely experiment with.
The container is storing all its valuable information in /var/lib/buildbot and /var/lib/buildbot_db
You can access your buildbot configuration in /var/lib/buildbot
In order to modify the configuration, you need to fork the project on github https://github.com/buildbot/buildbot-docker-example-config
Then you can clone your own fork, and start the docker-compose again.
.. code-block:: bash
To modify your config, edit the master.cfg file, commit your changes, and push to your fork.
You can use the command buildbot check-config in order to make sure the config is valid before the push.
You will need to change ``docker-compose.yml`` the variable ``BUILDBOT_CONFIG_URL`` in order to point to your github fork.
vi /var/lib/buildbot/master.cfg
The ``BUILDBOT_CONFIG_URL`` may point to a ``.tar.gz`` file accessible from HTTP.
Several git servers like github can generate that tarball automatically from the master branch of a git repository
If the ``BUILDBOT_CONFIG_URL`` does not end with ``.tar.gz``, it is considered to be the URL to a ``master.cfg`` file accessible from HTTP.
Customize your Worker container
-------------------------------
@@ -99,6 +112,17 @@ An example DockerFile is available in the contrib directory of buildbot:
https://github.com/buildbot/buildbot/blob/master/master/contrib/docker/pythonnode_worker/Dockerfile
Multi-master
------------
A multi-master environment can be setup using the ``multimaster/docker-compose.yml`` file in the example repository
# Build the Buildbot container (it will take a few minutes to download packages)
cd buildbot-docker-example-config/simple
docker-compose up -d
docker-compose scale buildbot=4
Going forward
-------------
You've got a taste now, but you're probably curious for more.
Let's step it up a little in the second tutorial by changing the configuration and doing an actual build.

View File

@@ -31,7 +31,7 @@ c['protocols'] = {'pb': {'port': 9989}}
c['change_source'] = []
c['change_source'].append(changes.GitPoller(
'git://github.com/buildbot/pyflakes.git',
'https://github.com/buildbot/pyflakes.git',
workdir='gitpoller-workdir', branch='master',
pollinterval=300))
@@ -58,7 +58,7 @@ c['schedulers'].append(schedulers.ForceScheduler(
factory = util.BuildFactory()
# check out the source
factory.addStep(steps.Git(repourl='git://github.com/buildbot/pyflakes.git', mode='incremental'))
factory.addStep(steps.Git(repourl='https://github.com/buildbot/pyflakes.git', mode='incremental'))
# run the tests (note that this will require that 'trial' is installed)
factory.addStep(steps.ShellCommand(command=["trial", "pyflakes"]))
@@ -103,3 +103,5 @@ c['db'] = {
# this at its default for all but the largest installations.
'db_url' : "sqlite:///state.sqlite",
}
c['buildbotNetUsageData'] = None

View File

@@ -4,6 +4,7 @@
"description": "smoke tests for buildbot with protractor",
"main": "index.js",
"dependencies": {
"coffee-script": "^1.11.1",
"protractor": "^4.0.3"
},
"devDependencies": {},

32
smokes/run_docker.sh Executable file
View File

@@ -0,0 +1,32 @@
#!/bin/bash
docker rm -f buildmaster
docker rm -f buildworker
function finish {
docker rm -f buildmaster
docker rm -f buildworker
}
trap finish EXIT
set -e
set -v
cd `dirname $0`
npm install
./node_modules/protractor/bin/webdriver-manager update
MASTER_IMAGE=${MASTER_IMAGE:-buildbot/buildbot-master:master}
WORKER_IMAGE=${WORKER_IMAGE:-buildbot/buildbot-worker:master}
BUILDBOT_CONFIG_URL=${BUILDBOT_CONFIG_URL:-https://raw.githubusercontent.com/buildbot/buildbot/master/smokes/master.cfg}
MASTERENV="-e http_proxy=$http_proxy -e https_proxy=$https_proxy -e BUILDBOT_CONFIG_DIR=config -e BUILDBOT_CONFIG_URL=$BUILDBOT_CONFIG_URL"
docker run --name buildmaster $MASTERENV -d -p 8010:8010 -p 9989:9989 -h buildmaster $MASTER_IMAGE
WORKERENV="-e https_proxy=$https_proxy -e BUILDMASTER=buildmaster -e BUILDMASTER_PORT=9989 -e WORKERNAME=example-worker -e WORKERPASS=pass"
docker run --name buildworker --link buildmaster -d $WORKERENV $WORKER_IMAGE
until curl http://localhost:8010 >/dev/null 2>/dev/null
do
docker logs buildmaster
sleep 1
done
./node_modules/protractor/bin/protractor protractor.conf.js

View File

@@ -1,9 +1,12 @@
# buildbot/buildbot-worker
# Provides a base Ubuntu (14.04) image with latest buildbot worker installed
# (based on twisted dockerfile https://github.com/cyli/docker-twisted)
# please follow docker best practices
# https://docs.docker.com/engine/userguide/eng-image/dockerfile_best-practices/
FROM ubuntu:14.04
# Provides a base Ubuntu (16.04) image with latest buildbot worker installed
# the worker image is not optimized for size, but rather uses ubuntu for wider package availability
FROM ubuntu:16.04
MAINTAINER Buildbot maintainers
COPY . /usr/src/buildbot-worker
@@ -11,33 +14,37 @@ COPY docker/buildbot.tac /buildbot/buildbot.tac
# Last build date - this can be updated whenever there are security updates so
# that everything is rebuilt
ENV security_updates_as_of 2016-08-23
ENV security_updates_as_of 2016-10-07
# This will make apt-get install without question
ENV DEBIAN_FRONTEND noninteractive
ARG DEBIAN_FRONTEND=noninteractive
# Install security updates and required packages
RUN apt-get update && \
apt-get -y upgrade && \
apt-get -y install -q \
build-essential \
git \
subversion \
python-dev libffi-dev libssl-dev python-pip python-virtualenv curl && \
apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
apt-get -y install -q -o APT::Get::Install-Recommends="false" -o APT::Get::Install-Suggests="false" \
build-essential \
git \
subversion \
python-dev \
libffi-dev \
libssl-dev \
python-pip \
python-virtualenv \
curl && \
rm -rf /var/lib/apt/lists/* && \
# Test runs produce a great quantity of dead grandchild processes. In a
# non-docker environment, these are automatically reaped by init (process 1),
# so we need to simulate that here. See https://github.com/Yelp/dumb-init
RUN curl -Lo /usr/local/bin/dumb-init https://github.com/Yelp/dumb-init/releases/download/v1.0.0/dumb-init_1.0.0_amd64
RUN chmod +x /usr/local/bin/dumb-init
curl -Lo /usr/local/bin/dumb-init https://github.com/Yelp/dumb-init/releases/download/v1.1.3/dumb-init_1.1.3_amd64 && \
chmod +x /usr/local/bin/dumb-init && \
# Install required python packages, and twisted
RUN pip install service_identity pycrypto && \
pip install twisted && \
pip install /usr/src/buildbot-worker
pip --no-cache-dir install \
'twisted[tls]' && \
pip install /usr/src/buildbot-worker && \
useradd -ms /bin/bash buildbot && chown -R buildbot /buildbot
RUN useradd -ms /bin/bash buildbot && chown -R buildbot /buildbot
USER buildbot
WORKDIR /buildbot

View File

@@ -35,3 +35,14 @@ package contains the buildmaster as well as a complete set of documentation.
See http://buildbot.net for more information and for an online version of the
Buildbot documentation.
Docker Image:
Here are the list of configuration variable for the buildbot/buildbot-worker image
- BUILDMASTER: the dns or IP address of the master to connect to
- BUILDMASTER_PORT: the port of the worker protocol
- WORKERNAME: the name of the worker as declared in the master configuration
- WORKERPASS: the password of the worker as declared in the master configuration
- WORKER_ENVIRONMENT_BLACKLIST: the worker environment variable to remove before starting the worker
As the environment variables are accessible from the build, and displayed in the log, it is better to remove secret variables like $WORKERPASS

View File

@@ -1,11 +1,12 @@
import fnmatch
import os
import sys
import fnmatch
from twisted.application import service
from twisted.python.log import ILogObserver, FileLogObserver
from buildbot_worker.bot import Worker
from twisted.python.log import FileLogObserver
from twisted.python.log import ILogObserver
from buildbot_worker.bot import Worker
# setup worker
basedir = os.path.abspath(os.path.dirname(__file__))
@@ -15,7 +16,7 @@ application = service.Application('buildbot-worker')
application.setComponent(ILogObserver, FileLogObserver(sys.stdout).emit)
# and worker on the same process!
buildmaster_host = os.environ.get("BUILDMASTER", 'localhost')
port = int(os.environ.get("BUILDMASTER_PORT", 19989))
port = int(os.environ.get("BUILDMASTER_PORT", 9989))
workername = os.environ.get("WORKERNAME", 'docker')
passwd = os.environ.get("WORKERPASS")

View File

@@ -99,7 +99,7 @@ setup_args = {
},
'entry_points': {
'console_scripts': [
'buildbot_worker=buildbot_worker.scripts.runner:run',
'buildbot-worker=buildbot_worker.scripts.runner:run',
# this will also be shipped on non windows :-(
'buildbot_worker_windows_service=buildbot_worker.scripts.windows_service:HandleCommandLine',
]}