fix: CI docker image push (#3476)

* fix ghcr app

* fix ghcr runtime push

* rename od_runtime to runtime
This commit is contained in:
Xingyao Wang
2024-08-20 04:53:28 +08:00
committed by GitHub
parent d1d2fb9aca
commit 8f0f764a85
12 changed files with 41 additions and 40 deletions

View File

@@ -69,7 +69,7 @@ jobs:
uses: actions/upload-artifact@v4
with:
name: ${{ matrix.image }}_image_${{ matrix.platform }}
path: /tmp/${{ matrix.image }}_$(echo "${{ steps.capture-tags.outputs.tags }}" | awk '{print $NF}')_${{ matrix.platform }}.tar
path: /tmp/${{ matrix.image }}_${{ steps.capture-tags.outputs.tags }}_${{ matrix.platform }}.tar
retention-days: 14
# Push the OpenHands and sandbox Docker images to the ghcr.io repository
@@ -99,7 +99,7 @@ jobs:
uses: actions/download-artifact@v4
with:
name: ${{ matrix.image }}_image_${{ matrix.platform }}
path: /tmp/${{ matrix.image }}_${{ steps.capture-tags.outputs.tags }}_${{ matrix.platform }}.tar
path: /tmp
- name: Load images and push to registry
run: |
mv /tmp/${{ matrix.platform }}/${{ matrix.image }}_${{ steps.capture-tags.outputs.tags }}_${{ matrix.platform }}.tar .

View File

@@ -31,7 +31,7 @@ jobs:
packages: write
strategy:
matrix:
image: ['od_runtime']
image: ['runtime']
base_image: ['nikolaik/python-nodejs:python3.11-nodejs22', 'python:3.11-bookworm', 'node:22-bookworm']
platform: ['amd64', 'arm64']
outputs:
@@ -139,7 +139,7 @@ jobs:
needs: prepare_test_image_tags
strategy:
matrix:
image: ['od_runtime']
image: ['runtime']
runtime_type: ['eventstream']
platform: ['amd64']
last_tag: ${{ fromJson(needs.prepare_test_image_tags.outputs.test_image_tags) }}
@@ -205,7 +205,7 @@ jobs:
strategy:
fail-fast: false
matrix:
image: ['od_runtime']
image: ['runtime']
runtime_type: ['eventstream']
platform: ['amd64']
last_tag: ${{ fromJson(needs.prepare_test_image_tags.outputs.test_image_tags) }}
@@ -270,7 +270,7 @@ jobs:
packages: write
strategy:
matrix:
image: ['od_runtime']
image: ['runtime']
runtime_type: ['eventstream']
platform: ['amd64', 'arm64']
last_tag: ${{ fromJson(needs.prepare_test_image_tags.outputs.test_image_tags) }}
@@ -297,17 +297,18 @@ jobs:
uses: actions/download-artifact@v4
with:
name: ${{ matrix.image }}_${{ matrix.last_tag }}_${{ matrix.platform }}
path: /tmp/${{ matrix.image }}_${{ matrix.last_tag }}_${{ matrix.platform }}.tar
path: /tmp/
- name: List downloaded files
run: |
ls -la /tmp/*
ls -la /tmp/${{ matrix.platform }}
- name: Load images and push to registry
run: |
image_file=$(find /tmp/${{ matrix.platform }} -name "${{ matrix.image }}_${{ matrix.last_tag }}_${{ matrix.platform }}.tar" | head -n 1)
image_file=$(find /tmp -name "${{ matrix.image }}_${{ matrix.last_tag }}_${{ matrix.platform }}.tar" | head -n 1)
if [ -z "$image_file" ]; then
echo "No matching image file found"
echo "No matching image file found for tag: ${{ matrix.last_tag }}"
exit 1
fi
echo "Loading image from file: $image_file"
if ! loaded_image=$(docker load -i "$image_file" | grep "Loaded image:" | head -n 1 | awk '{print $3}'); then
echo "Failed to load Docker image"
@@ -335,7 +336,7 @@ jobs:
tags: ${{ needs.ghcr_build_runtime.outputs.tags }}
strategy:
matrix:
image: ['od_runtime']
image: ['runtime']
permissions:
contents: read
packages: write

2
.gitignore vendored
View File

@@ -223,7 +223,7 @@ containers/agnostic_sandbox
image_build_logs
run_instance_logs
od_runtime_*.tar
runtime_*.tar
# docker build
containers/runtime/Dockerfile

View File

@@ -27,13 +27,13 @@ echo "Tags: ${tags[@]}"
if [[ "$image_name" == "openhands" ]]; then
dir="./containers/app"
elif [[ "$image_name" == "od_runtime" ]]; then
elif [[ "$image_name" == "runtime" ]]; then
dir="./containers/runtime"
else
dir="./containers/$image_name"
fi
if [[ (! -f "$dir/Dockerfile") && "$image_name" != "od_runtime" ]]; then
if [[ (! -f "$dir/Dockerfile") && "$image_name" != "runtime" ]]; then
# Allow runtime to be built without a Dockerfile
echo "No Dockerfile found"
exit 1

View File

@@ -1,7 +1,7 @@
DOCKER_REGISTRY=ghcr.io
DOCKER_ORG=all-hands-ai
DOCKER_BASE_DIR="./containers/runtime"
DOCKER_IMAGE=od_runtime
DOCKER_IMAGE=runtime
# These variables will be appended by the runtime_build.py script
# DOCKER_IMAGE_TAG=
# DOCKER_IMAGE_HASH_TAG=

View File

@@ -84,16 +84,16 @@ Check out [relavant code](https://github.com/All-Hands-AI/OpenHands/blob/main/op
OpenHands uses a dual-tagging system for its runtime images to balance reproducibility with flexibility:
1. Hash-based tag: `{target_image_repo}:{target_image_hash_tag}`
Example: `od_runtime:abc123def456`
Example: `runtime:abc123def456`
- This tag is based on the MD5 hash of the Docker build folder, which includes the source code (of runtime client and related dependencies) and Dockerfile.
- Identical hash tags guarantee that the images were built with exactly the same source code and Dockerfile.
- This ensures reproducibility: the same hash always means the same image contents.
2. Generic tag: `{target_image_repo}:{target_image_tag}`
Example: `od_runtime:od_v0.8.3_ubuntu_tag_22.04`
Example: `runtime:od_v0.8.3_ubuntu_tag_22.04`
- This tag follows the format: `od_runtime:od_v{OD_VERSION}_{BASE_IMAGE_NAME}_tag_{BASE_IMAGE_TAG}`
- This tag follows the format: `runtime:od_v{OD_VERSION}_{BASE_IMAGE_NAME}_tag_{BASE_IMAGE_TAG}`
- It represents the latest build for a particular base image and OpenHands version combination.
- This tag is updated whenever a new image is built from the same base image, even if the source code changes.
@@ -103,13 +103,13 @@ The hash-based tag ensures exact reproducibility, while the generic tag provides
1. Image Naming Convention:
- Hash-based tag: `{target_image_repo}:{target_image_hash_tag}`
Example: `od_runtime:abc123def456`
Example: `runtime:abc123def456`
- Generic tag: `{target_image_repo}:{target_image_tag}`
Example: `od_runtime:od_v0.8.3_ubuntu_tag_22.04`
Example: `runtime:od_v0.8.3_ubuntu_tag_22.04`
2. Build Process:
- a. Convert the base image name to an OD runtime image name.
Example: `ubuntu:22.04` -> `od_runtime:od_v0.8.3_ubuntu_tag_22.04`
Example: `ubuntu:22.04` -> `runtime:od_v0.8.3_ubuntu_tag_22.04`
- b. Generate a build context (Dockerfile and OpenHands source code) and calculate its hash.
- c. Check for an existing image with the calculated hash.
- d. If not found, check for a recent compatible image to use as a base.
@@ -119,9 +119,9 @@ The hash-based tag ensures exact reproducibility, while the generic tag provides
3. Image Reuse and Rebuilding Logic:
The system follows these steps to determine whether to build a new image or use an existing one from a user-provided (base) image (e.g., `ubuntu:22.04`):
a. If an image exists with the same hash (e.g., `od_runtime:abc123def456`), it will be reused as is.
a. If an image exists with the same hash (e.g., `runtime:abc123def456`), it will be reused as is.
b. If the exact hash is not found, the system will try to rebuild using the latest generic image (e.g., `od_runtime:od_v0.8.3_ubuntu_tag_22.04`) as a base. This saves time by leveraging existing dependencies.
b. If the exact hash is not found, the system will try to rebuild using the latest generic image (e.g., `runtime:od_v0.8.3_ubuntu_tag_22.04`) as a base. This saves time by leveraging existing dependencies.
c. If neither the hash-tagged nor the generic-tagged image is found, the system will build the image completely from scratch.
@@ -135,10 +135,10 @@ Here's a flowchart illustrating the build process:
```mermaid
flowchart TD
A[Start] --> B{Convert base image name}
B --> |ubuntu:22.04 -> od_runtime:od_v0.8.3_ubuntu_tag_22.04| C[Generate build context and hash]
B --> |ubuntu:22.04 -> runtime:od_v0.8.3_ubuntu_tag_22.04| C[Generate build context and hash]
C --> D{Check for existing image with hash}
D -->|Found od_runtime:abc123def456| E[Use existing image]
D -->|Not found| F{Check for od_runtime:od_v0.8.3_ubuntu_tag_22.04}
D -->|Found runtime:abc123def456| E[Use existing image]
D -->|Not found| F{Check for runtime:od_v0.8.3_ubuntu_tag_22.04}
F -->|Found| G[Rebuild based on recent image]
F -->|Not found| H[Build from scratch]
G --> I[Tag with hash and generic tags]
@@ -151,7 +151,7 @@ This approach ensures that:
1. Identical source code and Dockerfile always produce the same image (via hash-based tags).
2. The system can quickly rebuild images when minor changes occur (by leveraging recent compatible images).
3. The generic tag (e.g., `od_runtime:od_v0.8.3_ubuntu_tag_22.04`) always points to the latest build for a particular base image and OpenHands version combination.
3. The generic tag (e.g., `runtime:od_v0.8.3_ubuntu_tag_22.04`) always points to the latest build for a particular base image and OpenHands version combination.
By using this method, OpenHands maintains an efficient and flexible system for building and managing runtime images, adapting to both development needs and production requirements.

View File

@@ -52,7 +52,7 @@ def get_config(
container_image='xingyaoww/od-eval-logic-reasoning:v1.0',
enable_auto_lint=True,
use_host_network=False,
od_runtime_extra_deps='$OD_INTERPRETER_PATH -m pip install scitools-pyke',
runtime_extra_deps='$OD_INTERPRETER_PATH -m pip install scitools-pyke',
),
# do not mount workspace
workspace_base=None,

View File

@@ -104,7 +104,7 @@ def get_config(
container_image='xingyaoww/od-eval-mint:v1.0',
enable_auto_lint=True,
use_host_network=False,
od_runtime_extra_deps=f'$OD_INTERPRETER_PATH -m pip install {" ".join(MINT_DEPENDENCIES)}',
runtime_extra_deps=f'$OD_INTERPRETER_PATH -m pip install {" ".join(MINT_DEPENDENCIES)}',
),
# do not mount workspace
workspace_base=None,

View File

@@ -58,7 +58,7 @@ def get_config(
enable_auto_lint=True,
use_host_network=False,
browsergym_eval_env=env_id,
od_runtime_startup_env_vars={
runtime_startup_env_vars={
'BASE_URL': base_url,
'OPENAI_API_KEY': openai_api_key,
'SHOPPING': f'{base_url}:7770/',

View File

@@ -185,12 +185,12 @@ class SandboxConfig(metaclass=Singleton):
enable_auto_lint: Whether to enable auto-lint.
use_host_network: Whether to use the host network.
initialize_plugins: Whether to initialize plugins.
od_runtime_extra_deps: The extra dependencies to install in the runtime image (typically used for evaluation).
runtime_extra_deps: The extra dependencies to install in the runtime image (typically used for evaluation).
This will be rendered into the end of the Dockerfile that builds the runtime image.
It can contain any valid shell commands (e.g., pip install numpy).
The path to the interpreter is available as $OD_INTERPRETER_PATH,
which can be used to install dependencies for the OD-specific Python interpreter.
od_runtime_startup_env_vars: The environment variables to set at the launch of the runtime.
runtime_startup_env_vars: The environment variables to set at the launch of the runtime.
This is a dictionary of key-value pairs.
This is useful for setting environment variables that are needed by the runtime.
For example, for specifying the base url of website for browsergym evaluation.
@@ -207,8 +207,8 @@ class SandboxConfig(metaclass=Singleton):
)
use_host_network: bool = False
initialize_plugins: bool = True
od_runtime_extra_deps: str | None = None
od_runtime_startup_env_vars: dict[str, str] = field(default_factory=dict)
runtime_extra_deps: str | None = None
runtime_startup_env_vars: dict[str, str] = field(default_factory=dict)
browsergym_eval_env: str | None = None
def defaults_to_dict(self) -> dict:

View File

@@ -75,15 +75,15 @@ class EventStreamRuntime(Runtime):
logger.debug(f'EventStreamRuntime `{sid}` config:\n{self.config}')
async def ainit(self, env_vars: dict[str, str] | None = None):
if self.config.sandbox.od_runtime_extra_deps:
if self.config.sandbox.runtime_extra_deps:
logger.info(
f'Installing extra user-provided dependencies in the runtime image: {self.config.sandbox.od_runtime_extra_deps}'
f'Installing extra user-provided dependencies in the runtime image: {self.config.sandbox.runtime_extra_deps}'
)
self.container_image = build_runtime_image(
self.container_image,
self.runtime_builder,
extra_deps=self.config.sandbox.od_runtime_extra_deps,
extra_deps=self.config.sandbox.runtime_extra_deps,
)
self.container = await self._init_container(
self.sandbox_workspace_dir,

View File

@@ -14,7 +14,7 @@ from openhands.core.logger import openhands_logger as logger
from openhands.runtime.builder import DockerRuntimeBuilder, RuntimeBuilder
RUNTIME_IMAGE_REPO = os.getenv(
'OD_RUNTIME_RUNTIME_IMAGE_REPO', 'ghcr.io/openhands/od_runtime'
'OD_RUNTIME_RUNTIME_IMAGE_REPO', 'ghcr.io/openhands/runtime'
)
@@ -177,7 +177,7 @@ def get_runtime_image_repo_and_tag(base_image: str) -> tuple[str, str]:
if RUNTIME_IMAGE_REPO in base_image:
logger.info(
f'The provided image [{base_image}] is a already a valid od_runtime image.\n'
f'The provided image [{base_image}] is a already a valid runtime image.\n'
f'Will try to reuse it as is.'
)
@@ -235,7 +235,7 @@ def build_runtime_image(
# non-hash generic image name, it could contain *similar* dependencies
# but *might* not exactly match the state of the source code.
# It resembles the "latest" tag in the docker image naming convention for
# a particular {repo}:{tag} pair (e.g., ubuntu:latest -> od_runtime:ubuntu_tag_latest)
# a particular {repo}:{tag} pair (e.g., ubuntu:latest -> runtime:ubuntu_tag_latest)
# we will build from IT to save time if the `from_scratch_hash` is not found
generic_runtime_image_name = f'{runtime_image_repo}:{runtime_image_tag}'