pyinstaller binary generation for windows and linux-amd64

binaries generated  are tested on circle ci
binaries can be tested on windows, but tests are not stable enough yet on appveyor

binaries are pushed to github releases

Makefile

@@ -115,3 +115,7 @@ release: virtualenv
 	cp -r master/docs/_build/html ../bbdocs/docs/$(VERSION)
 	. .venv/bin/activate && cd ../bbdocs && make && git add . && git commit -m $(VERSION) && git push
 	echo twine upload --sign dist/*
+
+pyinstaller: virtualenv
+	$(PIP) install pyinstaller
+	$(VENV_NAME)/bin/pyinstaller -F pyinstaller/buildbot-worker.spec

appveyor.yml

@@ -18,10 +18,6 @@ install:
   # Twisted requires pywin32 on Windows in order to spawn processes.
   - "python -m pip install pypiwin32"
 
-  # AppVeyor has SQLite 3.6.21, and Buildbot prefers at least 3.7,
-  # install third-party SQLite.
-  #- "%PYTHON%\\python.exe -m pip install pysqlite"
-
   # TODO: There is no binary wheel for pysqlite and building on Windows is
   # painful. Let's just substitude sqlite3.dll with never version.
   - ps: |
@@ -32,16 +28,18 @@ install:
 
   - "python -m pip install -U pip"
   - "python -m pip install -r requirements-ci.txt"
-  - "python -m pip install -e pkg"
-  - "python -m pip install -e master[tls,test]"
-  - "python -m pip install -e worker"
-  - "python -m pip install coverage codecov"
   - "python -m pip list"
 
 build: false
 
 test_script:
   - "coverage run --rcfile=common/coveragerc -m twisted.trial --reporter=text --rterrors buildbot.test buildbot_worker.test"
+  - ps: |
+          echo $ENV:PYTHON
+          if ($env:PYTHON -imatch 'C:\\Python27') {
+            iex 'pyinstaller -F pyinstaller/buildbot-worker.spec'
+            iex 'appveyor PushArtifact dist\\buildbot-worker.exe'
+          }
 
 on_success:
   - "coverage xml --rcfile=common/coveragerc -o coverage.xml -i"
@@ -62,3 +60,16 @@ on_failure:
 # For detail see: https://www.appveyor.com/docs/how-to/rdp-to-build-worker
 #on_finish:
 #- ps: $blockRdp = $true; iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1'))
+
+deploy:
+  release: $(APPVEYOR_REPO_TAG_NAME)
+  description: 'windows binary for buildbot-worker'
+  provider: GitHub
+  auth_token:
+    secure: HQNlcAyaY9Jznbl77rfNatZG62Gg+qFY7emzj5n3Wu16fkr8dLlFNTKOJlfXO5uK
+  artifact: "buildbot-worker.exe"
+  draft: false
+  prerelease: false
+  on:
+    appveyor_repo_tag: true        # deploy on tag push only
+    PYTHON: "C:\\Python27"

circle.yml

@@ -4,16 +4,39 @@
 machine:
   python:
     version: 2.7.11
+  node:
+    version: 6.11.4
 
 dependencies:
+  pre:
+    - sudo apt-get update; sudo apt-get install libenchant-dev python3-dev
+    - go get github.com/tcnksm/ghr
   override:
-    - pyenv global 2.7.11
+    - pip install -U pip
     - pip install -r requirements-ci.txt
+    - pip install -r requirements-cidocs.txt
+    - (cd www/base; yarn install)
 
 test:
   override:
     - make docs
+    - make tarballs -j4
+    # pyinstaller: build the buildbotworker in a special virtualenv
+    # circle ci pythons do not include the necessary dynamic libraries for pyinstaller
+    - virtualenv -p /usr/bin/python3.4 .venv/
+    - .venv/bin/pip install twisted[tls] -e worker pyinstaller
+    - .venv/bin/pyinstaller -F pyinstaller/buildbot-worker.spec
+    # we test the new generated binary with the global virtualenv
+    - SANDBOXED_WORKER_PATH=`pwd`/dist/buildbot-worker trial --reporter=text --rterrors buildbot.test.integration.interop
 
 general:
   artifacts:
     - "master/docs/_build/html/"
+    - "dist"
+
+deployment:
+  release:
+    tag: /v.*/
+    commands:
+      - mv dist/buildbot-worker dist/buildbot-worker-linux-amd64-`git describe --tags`.bin
+      - ghr -t $GITHUB_TOKEN -u $CIRCLE_PROJECT_USERNAME -r $CIRCLE_PROJECT_REPONAME --replace `git describe --tags` dist/

master/buildbot/test/util/integration.py

@@ -110,7 +110,7 @@ class RunMasterBase(unittest.TestCase):
             elif self.proto == 'null':
                 proto = {"null": {}}
                 workerclass = worker.LocalWorker
-            config_dict['workers'] = [workerclass("local1", "localpw")]
+            config_dict['workers'] = [workerclass("local1", "localpw", missing_timeout=0)]
             config_dict['protocols'] = proto
 
         m = yield getMaster(self, reactor, config_dict)
@@ -140,11 +140,13 @@ class RunMasterBase(unittest.TestCase):
                 self.w = SandboxedWorker(
                     "127.0.0.1", workerPort, "local1", "localpw", worker_dir.path,
                     sandboxed_worker_path)
+                self.addCleanup(self.w.shutdownWorker)
+
         elif self.proto == 'null':
             self.w = None
+
         if self.w is not None:
-            self.w.startService()
-            self.addCleanup(self.w.stopService)
+            self.w.setServiceParent(m)
 
         @defer.inlineCallbacks
         def dump():

master/buildbot/test/util/sandboxed_worker.py

@@ -19,6 +19,7 @@ from __future__ import print_function
 
 from subprocess import check_call
 
+from twisted.internet import defer
 from twisted.internet import protocol
 from twisted.internet import reactor
 
@@ -26,12 +27,21 @@ from buildbot.util.service import AsyncService
 
 
 class WorkerProcessProtocol(protocol.ProcessProtocol):
+    def __init__(self):
+        self.finished_deferred = defer.Deferred()
+
     def outReceived(self, data):
         print(data)
 
     def errReceived(self, data):
         print(data)
 
+    def processEnded(self, _):
+        self.finished_deferred.callback(None)
+
+    def waitForFinish(self):
+        return self.finished_deferred
+
 
 class SandboxedWorker(AsyncService):
     def __init__(self, masterhost, port, name, passwd, workerdir, sandboxed_worker_path):
@@ -49,11 +59,22 @@ class SandboxedWorker(AsyncService):
         check_call([self.sandboxed_worker_path, "create-worker", '-q', self.workerdir,
                     self.masterhost + ":" + str(self.port), self.workername, self.workerpasswd])
 
-        processProtocol = WorkerProcessProtocol()
+        self.processprotocol = processProtocol = WorkerProcessProtocol()
         # we need to spawn the worker asynchronously though
         self.process = reactor.spawnProcess(
             processProtocol, self.sandboxed_worker_path, args=['bbw', 'start', '--nodaemon', self.workerdir])
 
+        self.worker = self.master.workers.getWorkerByName(self.workername)
+        return AsyncService.startService(self)
+
+    @defer.inlineCallbacks
+    def shutdownWorker(self):
+        # on windows, we killing a process does not work well.
+        # we use the graceful shutdown feature of buildbot-worker instead to kill the worker
+        # but we must do that before the master is stopping.
+        yield self.worker.shutdown()
+        # wait for process to disappear
+        yield self.processprotocol.waitForFinish()
+
     def stopService(self):
-        self.process.signalProcess("TERM")
-        self.process.loseConnection()
+        return AsyncService.stopService(self)

pyinstaller/buildbot-worker.py

@@ -0,0 +1,6 @@
+from __future__ import absolute_import
+from __future__ import print_function
+
+from buildbot_worker.scripts.runner import run
+
+run()

pyinstaller/buildbot-worker.spec

@@ -0,0 +1,29 @@
+# -*- mode: python -*-
+
+block_cipher = None
+
+
+a = Analysis(['buildbot-worker.py'],
+             pathex=[],
+             binaries=[],
+             datas=[],
+             hiddenimports=["buildbot_worker", "buildbot_worker.scripts.create_worker", "buildbot_worker.scripts.start", "buildbot_worker.scripts.stop", "buildbot_worker.scripts.restart", "buildbot_worker.bot"],
+             hookspath=[],
+             runtime_hooks=[],
+             excludes=[],
+             win_no_prefer_redirects=False,
+             win_private_assemblies=False,
+             cipher=block_cipher)
+pyz = PYZ(a.pure, a.zipped_data,
+             cipher=block_cipher)
+exe = EXE(pyz,
+          a.scripts,
+          a.binaries,
+          a.zipfiles,
+          a.datas,
+          name='buildbot-worker',
+          debug=False,
+          strip=False,
+          upx=True,
+          runtime_tmpdir=None,
+          console=True )

requirements-ci.txt

@@ -54,6 +54,7 @@ pycodestyle==2.3.1
 pycparser==2.18
 pyenchant==2.0.0
 pyflakes==1.6.0
+pyinstaller
 pyjade==4.0.0
 PyJWT==1.5.3
 pylint==1.8.1