GitHub Action: Support Caching for faster builds (#124)

* Update docker-publish.yml

* started script to generate dockerfiles

* autogenerated the dockerfiles

* new and improved dockerignore

closes #122

.dockerignore

@@ -1,8 +1,19 @@
-rhasspy/
-config/
-__pycache__/
+## ignore all
+**
+
+## but include these directories/files
+!/docs/**
+!/home_intent/**
+!/setup/**
+!/tmp/**
+!/ui/**
+!/requirements.txt
+
+## but ignore these dirs/files in those directories
+**/__pycache__/
+**/.pytest_cache
+**/.git
 docs/site
 ui/frontend/node_modules
 ui/frontend/build
-ui/frontend/.svelte-kit
-.git/
+ui/frontend/.svelte-kit

.github/workflows/docker-publish.yml

@@ -39,6 +39,14 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v1
 
+      - name: Cache Docker layers
+        uses: actions/cache@v2
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-buildx-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-
+
       # Login against a Docker registry except on PR
       # https://github.com/docker/login-action
       - name: Log into registry ${{ env.REGISTRY }}
@@ -66,16 +74,34 @@ jobs:
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
 
+      # Build and push Docker image with Buildx (don't push on PR)
+      # https://github.com/docker/build-push-action
+      - name: Build static content image
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          file: ./Dockerfile.static
+          push: false
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max
+          outputs: type=local,dest=./tmp/static
+
+      - name: List ./tmp/static directory (debug...)
+        run: ls -R ./tmp/static
+
       # Build and push Docker image with Buildx (don't push on PR)
       # https://github.com/docker/build-push-action
       - name: Build and push rhasspy-external Docker image
         uses: docker/build-push-action@v2
         with:
           context: .
+          file: ./Dockerfile.rhasspy-external
           push: ${{ github.event_name != 'pull_request' }}
           tags: ${{ steps.rhasspy-external-meta.outputs.tags }}
           labels: ${{ steps.rhasspy-external-meta.outputs.labels }}
           platforms: linux/amd64,linux/arm/v7,linux/arm64
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max
 
       # Build and push Docker image with Buildx (don't push on PR)
       # https://github.com/docker/build-push-action
@@ -83,7 +109,18 @@ jobs:
         uses: docker/build-push-action@v2
         with:
           context: .
+          file: ./Dockerfile.gh-build
           push: ${{ github.event_name != 'pull_request' }}
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
-          platforms: linux/amd64,linux/arm/v7,linux/arm64
+          platforms: linux/amd64,linux/arm/v7,linux/arm64          
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max
+
+      # Temp fix
+      # https://github.com/docker/build-push-action/issues/252
+      # https://github.com/moby/buildkit/issues/1896
+      - name: Move cache
+        run: |
+          rm -rf /tmp/.buildx-cache
+          mv /tmp/.buildx-cache-new /tmp/.buildx-cache

Dockerfile

@@ -1,3 +1,7 @@
+
+### NOTE: THIS FILE IS AUTOGENERATED FROM generate_dockerfiles.py ###
+
+
 FROM python:3.7
 ENV PYTHONDONTWRITEBYTECODE=1
 ENV PYTHONUNBUFFERED=1
@@ -43,6 +47,8 @@ WORKDIR /usr/src/app
 ENV PYTHONUNBUFFERED=1
 ENV PATH="/venv/bin:$PATH"
 
+
+
 COPY --from=1 /venv /venv
 COPY --from=0 /usr/src/app/docs/site ./docs/site
 COPY setup setup
@@ -51,3 +57,4 @@ COPY ui ui
 COPY --from=2 /usr/src/app/ui/frontend/build ./ui/frontend/build
 
 ENTRYPOINT [ "bash", "/usr/src/app/setup/setup.sh" ]
+

Dockerfile.gh-build

@@ -0,0 +1,36 @@
+
+### NOTE: THIS FILE IS AUTOGENERATED FROM generate_dockerfiles.py ###
+
+
+FROM python:3.7
+WORKDIR /usr/src/app
+
+ENV LANG=C.UTF-8
+ENV PYTHONDONTWRITEBYTECODE=1
+ENV PYTHONUNBUFFERED=1
+
+ENV PATH="/venv/bin:$PATH"
+
+RUN python -m venv /venv --copies
+COPY requirements.txt ./
+RUN pip3 install --no-cache-dir -r requirements.txt
+
+
+
+FROM rhasspy/rhasspy:2.5.10
+WORKDIR /usr/src/app
+
+ENV PYTHONUNBUFFERED=1
+ENV PATH="/venv/bin:$PATH"
+
+
+
+COPY --from=0 /venv /venv
+COPY tmp/static/docs/site ./docs/site
+COPY setup setup
+COPY home_intent home_intent
+COPY ui ui
+COPY tmp/static/ui/frontend/build ./ui/frontend/build
+
+ENTRYPOINT [ "bash", "/usr/src/app/setup/setup.sh" ]
+

Dockerfile.rhasspy-external

@@ -1,14 +1,5 @@
-FROM python:3.7
-ENV PYTHONDONTWRITEBYTECODE=1
-ENV PYTHONUNBUFFERED=1
-WORKDIR /usr/src/app/docs
-
-COPY docs/requirements.txt ./
-RUN pip3 install --no-cache-dir -r requirements.txt
-
-COPY docs .
-RUN mkdocs build
 
+### NOTE: THIS FILE IS AUTOGENERATED FROM generate_dockerfiles.py ###
 
 
 FROM python:3.7
@@ -26,17 +17,6 @@ RUN pip3 install --no-cache-dir -r requirements.txt
 
 
 
-FROM node:lts
-WORKDIR /usr/src/app/ui/frontend
-
-COPY ui/frontend/package.json ui/frontend/package-lock.json ./
-RUN npm ci
-
-COPY ui/frontend .
-RUN npm run build
-
-
-
 FROM python:3.7-slim
 WORKDIR /usr/src/app
 
@@ -45,11 +25,12 @@ ENV PATH="/venv/bin:$PATH"
 
 RUN pip3 install --no-cache-dir supervisor
 
-COPY --from=1 /venv /venv
-COPY --from=0 /usr/src/app/docs/site ./docs/site
+COPY --from=0 /venv /venv
+COPY tmp/static/docs/site ./docs/site
 COPY setup setup
 COPY home_intent home_intent
 COPY ui ui
-COPY --from=2 /usr/src/app/ui/frontend/build ./ui/frontend/build
+COPY tmp/static/ui/frontend/build ./ui/frontend/build
 
 ENTRYPOINT [ "supervisord", "--configuration", "/usr/src/app/setup/supervisord.conf" ]
+

Dockerfile.static

@@ -0,0 +1,30 @@
+
+### NOTE: THIS FILE IS AUTOGENERATED FROM generate_dockerfiles.py ###
+
+
+FROM python:3.7
+ENV PYTHONDONTWRITEBYTECODE=1
+ENV PYTHONUNBUFFERED=1
+WORKDIR /usr/src/app/docs
+
+COPY docs/requirements.txt ./
+RUN pip3 install --no-cache-dir -r requirements.txt
+
+COPY docs .
+RUN mkdocs build
+
+
+
+FROM node:lts
+WORKDIR /usr/src/app/ui/frontend
+
+COPY ui/frontend/package.json ui/frontend/package-lock.json ./
+RUN npm ci
+
+COPY ui/frontend .
+RUN npm run build
+
+
+FROM scratch
+COPY --from=0 /usr/src/app/docs/site /docs/site
+COPY --from=1 /usr/src/app/ui/frontend/build /ui/frontend/build

generate_dockerfiles.py

@@ -0,0 +1,169 @@
+from pathlib import Path
+
+RHASSPY_VERSION = "2.5.10"
+
+BUILD_DOCS = """
+FROM python:3.7
+ENV PYTHONDONTWRITEBYTECODE=1
+ENV PYTHONUNBUFFERED=1
+WORKDIR /usr/src/app/docs
+
+COPY docs/requirements.txt ./
+RUN pip3 install --no-cache-dir -r requirements.txt
+
+COPY docs .
+RUN mkdocs build
+"""
+
+BUILD_FRONTEND = """
+FROM node:lts
+WORKDIR /usr/src/app/ui/frontend
+
+COPY ui/frontend/package.json ui/frontend/package-lock.json ./
+RUN npm ci
+
+COPY ui/frontend .
+RUN npm run build
+"""
+
+
+BUILD_VENV = """
+FROM python:3.7
+WORKDIR /usr/src/app
+
+ENV LANG=C.UTF-8
+ENV PYTHONDONTWRITEBYTECODE=1
+ENV PYTHONUNBUFFERED=1
+
+ENV PATH="/venv/bin:$PATH"
+
+RUN python -m venv /venv --copies
+COPY requirements.txt ./
+RUN pip3 install --no-cache-dir -r requirements.txt
+"""
+
+
+FINAL_BULD = """
+FROM {build_image}
+WORKDIR /usr/src/app
+
+ENV PYTHONUNBUFFERED=1
+ENV PATH="/venv/bin:$PATH"
+
+{extra}
+
+COPY {venv_location} /venv
+COPY {docs_location} ./docs/site
+COPY setup setup
+COPY home_intent home_intent
+COPY ui ui
+COPY {frontend_location} ./ui/frontend/build
+
+ENTRYPOINT {entrypoint}
+"""
+
+WARNING = """### NOTE: THIS FILE IS AUTOGENERATED FROM generate_dockerfiles.py ###"""
+
+
+def main():
+    make_local_dockerfile()
+    make_gh_actions_dockerfiles()
+
+
+def write_dockerfile(filename: str, contents: str):
+    file = Path(filename)
+    if file.is_file():
+        file.chmod(0o644)
+
+    file.write_text(contents)
+    file.chmod(0o444)
+
+
+def make_local_dockerfile():
+    final_build = FINAL_BULD.format(
+        build_image=f"rhasspy/rhasspy:{RHASSPY_VERSION}",
+        venv_location="--from=1 /venv",
+        docs_location="--from=0 /usr/src/app/docs/site",
+        frontend_location="--from=2 /usr/src/app/ui/frontend/build",
+        extra="",
+        entrypoint='[ "bash", "/usr/src/app/setup/setup.sh" ]',
+    )
+
+    dockerfile = f"""
+{WARNING}
+
+{BUILD_DOCS}
+
+{BUILD_VENV}
+
+{BUILD_FRONTEND}
+
+{final_build}
+"""
+    write_dockerfile("Dockerfile", dockerfile)
+
+
+def make_gh_actions_dockerfiles():
+    make_static_dockerfile()
+    make_rhasspy_external_dockerfile()
+    make_gh_build_dockerfile()
+
+
+def make_static_dockerfile():
+    dockerfile = f"""
+{WARNING}
+
+{BUILD_DOCS}
+
+{BUILD_FRONTEND}
+
+FROM scratch
+COPY --from=0 /usr/src/app/docs/site /docs/site
+COPY --from=1 /usr/src/app/ui/frontend/build /ui/frontend/build
+"""
+
+    write_dockerfile("Dockerfile.static", dockerfile)
+
+
+def make_rhasspy_external_dockerfile():
+    final_build = FINAL_BULD.format(
+        build_image="python:3.7-slim",
+        venv_location="--from=0 /venv",
+        docs_location="tmp/static/docs/site",
+        frontend_location="tmp/static/ui/frontend/build",
+        extra="RUN pip3 install --no-cache-dir supervisor",
+        entrypoint='[ "supervisord", "--configuration", "/usr/src/app/setup/supervisord.conf" ]',
+    )
+
+    dockerfile = f"""
+{WARNING}
+
+{BUILD_VENV}
+
+{final_build}
+"""
+    write_dockerfile("Dockerfile.rhasspy-external", dockerfile)
+
+
+def make_gh_build_dockerfile():
+    final_build = FINAL_BULD.format(
+        build_image=f"rhasspy/rhasspy:{RHASSPY_VERSION}",
+        venv_location="--from=0 /venv",
+        docs_location="tmp/static/docs/site",
+        frontend_location="tmp/static/ui/frontend/build",
+        extra="",
+        entrypoint='[ "bash", "/usr/src/app/setup/setup.sh" ]',
+    )
+
+    dockerfile = f"""
+{WARNING}
+
+{BUILD_VENV}
+
+{final_build}
+"""
+    write_dockerfile("Dockerfile.gh-build", dockerfile)
+
+
+if __name__ == "__main__":
+    main()