fix: CI docker image push (#3476)

* fix ghcr app

* fix ghcr runtime push

* rename od_runtime to runtime
This commit is contained in:
Xingyao Wang
2024-08-20 04:53:28 +08:00
committed by GitHub
parent d1d2fb9aca
commit 8f0f764a85
12 changed files with 41 additions and 40 deletions

View File

@@ -69,7 +69,7 @@ jobs:
uses: actions/upload-artifact@v4 uses: actions/upload-artifact@v4
with: with:
name: ${{ matrix.image }}_image_${{ matrix.platform }} name: ${{ matrix.image }}_image_${{ matrix.platform }}
path: /tmp/${{ matrix.image }}_$(echo "${{ steps.capture-tags.outputs.tags }}" | awk '{print $NF}')_${{ matrix.platform }}.tar path: /tmp/${{ matrix.image }}_${{ steps.capture-tags.outputs.tags }}_${{ matrix.platform }}.tar
retention-days: 14 retention-days: 14
# Push the OpenHands and sandbox Docker images to the ghcr.io repository # Push the OpenHands and sandbox Docker images to the ghcr.io repository
@@ -99,7 +99,7 @@ jobs:
uses: actions/download-artifact@v4 uses: actions/download-artifact@v4
with: with:
name: ${{ matrix.image }}_image_${{ matrix.platform }} name: ${{ matrix.image }}_image_${{ matrix.platform }}
path: /tmp/${{ matrix.image }}_${{ steps.capture-tags.outputs.tags }}_${{ matrix.platform }}.tar path: /tmp
- name: Load images and push to registry - name: Load images and push to registry
run: | run: |
mv /tmp/${{ matrix.platform }}/${{ matrix.image }}_${{ steps.capture-tags.outputs.tags }}_${{ matrix.platform }}.tar . mv /tmp/${{ matrix.platform }}/${{ matrix.image }}_${{ steps.capture-tags.outputs.tags }}_${{ matrix.platform }}.tar .

View File

@@ -31,7 +31,7 @@ jobs:
packages: write packages: write
strategy: strategy:
matrix: matrix:
image: ['od_runtime'] image: ['runtime']
base_image: ['nikolaik/python-nodejs:python3.11-nodejs22', 'python:3.11-bookworm', 'node:22-bookworm'] base_image: ['nikolaik/python-nodejs:python3.11-nodejs22', 'python:3.11-bookworm', 'node:22-bookworm']
platform: ['amd64', 'arm64'] platform: ['amd64', 'arm64']
outputs: outputs:
@@ -139,7 +139,7 @@ jobs:
needs: prepare_test_image_tags needs: prepare_test_image_tags
strategy: strategy:
matrix: matrix:
image: ['od_runtime'] image: ['runtime']
runtime_type: ['eventstream'] runtime_type: ['eventstream']
platform: ['amd64'] platform: ['amd64']
last_tag: ${{ fromJson(needs.prepare_test_image_tags.outputs.test_image_tags) }} last_tag: ${{ fromJson(needs.prepare_test_image_tags.outputs.test_image_tags) }}
@@ -205,7 +205,7 @@ jobs:
strategy: strategy:
fail-fast: false fail-fast: false
matrix: matrix:
image: ['od_runtime'] image: ['runtime']
runtime_type: ['eventstream'] runtime_type: ['eventstream']
platform: ['amd64'] platform: ['amd64']
last_tag: ${{ fromJson(needs.prepare_test_image_tags.outputs.test_image_tags) }} last_tag: ${{ fromJson(needs.prepare_test_image_tags.outputs.test_image_tags) }}
@@ -270,7 +270,7 @@ jobs:
packages: write packages: write
strategy: strategy:
matrix: matrix:
image: ['od_runtime'] image: ['runtime']
runtime_type: ['eventstream'] runtime_type: ['eventstream']
platform: ['amd64', 'arm64'] platform: ['amd64', 'arm64']
last_tag: ${{ fromJson(needs.prepare_test_image_tags.outputs.test_image_tags) }} last_tag: ${{ fromJson(needs.prepare_test_image_tags.outputs.test_image_tags) }}
@@ -297,17 +297,18 @@ jobs:
uses: actions/download-artifact@v4 uses: actions/download-artifact@v4
with: with:
name: ${{ matrix.image }}_${{ matrix.last_tag }}_${{ matrix.platform }} name: ${{ matrix.image }}_${{ matrix.last_tag }}_${{ matrix.platform }}
path: /tmp/${{ matrix.image }}_${{ matrix.last_tag }}_${{ matrix.platform }}.tar path: /tmp/
- name: List downloaded files - name: List downloaded files
run: | run: |
ls -la /tmp/* ls -la /tmp/${{ matrix.platform }}
- name: Load images and push to registry - name: Load images and push to registry
run: | run: |
image_file=$(find /tmp/${{ matrix.platform }} -name "${{ matrix.image }}_${{ matrix.last_tag }}_${{ matrix.platform }}.tar" | head -n 1) image_file=$(find /tmp -name "${{ matrix.image }}_${{ matrix.last_tag }}_${{ matrix.platform }}.tar" | head -n 1)
if [ -z "$image_file" ]; then if [ -z "$image_file" ]; then
echo "No matching image file found" echo "No matching image file found for tag: ${{ matrix.last_tag }}"
exit 1 exit 1
fi fi
echo "Loading image from file: $image_file" echo "Loading image from file: $image_file"
if ! loaded_image=$(docker load -i "$image_file" | grep "Loaded image:" | head -n 1 | awk '{print $3}'); then if ! loaded_image=$(docker load -i "$image_file" | grep "Loaded image:" | head -n 1 | awk '{print $3}'); then
echo "Failed to load Docker image" echo "Failed to load Docker image"
@@ -335,7 +336,7 @@ jobs:
tags: ${{ needs.ghcr_build_runtime.outputs.tags }} tags: ${{ needs.ghcr_build_runtime.outputs.tags }}
strategy: strategy:
matrix: matrix:
image: ['od_runtime'] image: ['runtime']
permissions: permissions:
contents: read contents: read
packages: write packages: write

2
.gitignore vendored
View File

@@ -223,7 +223,7 @@ containers/agnostic_sandbox
image_build_logs image_build_logs
run_instance_logs run_instance_logs
od_runtime_*.tar runtime_*.tar
# docker build # docker build
containers/runtime/Dockerfile containers/runtime/Dockerfile

View File

@@ -27,13 +27,13 @@ echo "Tags: ${tags[@]}"
if [[ "$image_name" == "openhands" ]]; then if [[ "$image_name" == "openhands" ]]; then
dir="./containers/app" dir="./containers/app"
elif [[ "$image_name" == "od_runtime" ]]; then elif [[ "$image_name" == "runtime" ]]; then
dir="./containers/runtime" dir="./containers/runtime"
else else
dir="./containers/$image_name" dir="./containers/$image_name"
fi fi
if [[ (! -f "$dir/Dockerfile") && "$image_name" != "od_runtime" ]]; then if [[ (! -f "$dir/Dockerfile") && "$image_name" != "runtime" ]]; then
# Allow runtime to be built without a Dockerfile # Allow runtime to be built without a Dockerfile
echo "No Dockerfile found" echo "No Dockerfile found"
exit 1 exit 1

View File

@@ -1,7 +1,7 @@
DOCKER_REGISTRY=ghcr.io DOCKER_REGISTRY=ghcr.io
DOCKER_ORG=all-hands-ai DOCKER_ORG=all-hands-ai
DOCKER_BASE_DIR="./containers/runtime" DOCKER_BASE_DIR="./containers/runtime"
DOCKER_IMAGE=od_runtime DOCKER_IMAGE=runtime
# These variables will be appended by the runtime_build.py script # These variables will be appended by the runtime_build.py script
# DOCKER_IMAGE_TAG= # DOCKER_IMAGE_TAG=
# DOCKER_IMAGE_HASH_TAG= # DOCKER_IMAGE_HASH_TAG=

View File

@@ -84,16 +84,16 @@ Check out [relavant code](https://github.com/All-Hands-AI/OpenHands/blob/main/op
OpenHands uses a dual-tagging system for its runtime images to balance reproducibility with flexibility: OpenHands uses a dual-tagging system for its runtime images to balance reproducibility with flexibility:
1. Hash-based tag: `{target_image_repo}:{target_image_hash_tag}` 1. Hash-based tag: `{target_image_repo}:{target_image_hash_tag}`
Example: `od_runtime:abc123def456` Example: `runtime:abc123def456`
- This tag is based on the MD5 hash of the Docker build folder, which includes the source code (of runtime client and related dependencies) and Dockerfile. - This tag is based on the MD5 hash of the Docker build folder, which includes the source code (of runtime client and related dependencies) and Dockerfile.
- Identical hash tags guarantee that the images were built with exactly the same source code and Dockerfile. - Identical hash tags guarantee that the images were built with exactly the same source code and Dockerfile.
- This ensures reproducibility: the same hash always means the same image contents. - This ensures reproducibility: the same hash always means the same image contents.
2. Generic tag: `{target_image_repo}:{target_image_tag}` 2. Generic tag: `{target_image_repo}:{target_image_tag}`
Example: `od_runtime:od_v0.8.3_ubuntu_tag_22.04` Example: `runtime:od_v0.8.3_ubuntu_tag_22.04`
- This tag follows the format: `od_runtime:od_v{OD_VERSION}_{BASE_IMAGE_NAME}_tag_{BASE_IMAGE_TAG}` - This tag follows the format: `runtime:od_v{OD_VERSION}_{BASE_IMAGE_NAME}_tag_{BASE_IMAGE_TAG}`
- It represents the latest build for a particular base image and OpenHands version combination. - It represents the latest build for a particular base image and OpenHands version combination.
- This tag is updated whenever a new image is built from the same base image, even if the source code changes. - This tag is updated whenever a new image is built from the same base image, even if the source code changes.
@@ -103,13 +103,13 @@ The hash-based tag ensures exact reproducibility, while the generic tag provides
1. Image Naming Convention: 1. Image Naming Convention:
- Hash-based tag: `{target_image_repo}:{target_image_hash_tag}` - Hash-based tag: `{target_image_repo}:{target_image_hash_tag}`
Example: `od_runtime:abc123def456` Example: `runtime:abc123def456`
- Generic tag: `{target_image_repo}:{target_image_tag}` - Generic tag: `{target_image_repo}:{target_image_tag}`
Example: `od_runtime:od_v0.8.3_ubuntu_tag_22.04` Example: `runtime:od_v0.8.3_ubuntu_tag_22.04`
2. Build Process: 2. Build Process:
- a. Convert the base image name to an OD runtime image name. - a. Convert the base image name to an OD runtime image name.
Example: `ubuntu:22.04` -> `od_runtime:od_v0.8.3_ubuntu_tag_22.04` Example: `ubuntu:22.04` -> `runtime:od_v0.8.3_ubuntu_tag_22.04`
- b. Generate a build context (Dockerfile and OpenHands source code) and calculate its hash. - b. Generate a build context (Dockerfile and OpenHands source code) and calculate its hash.
- c. Check for an existing image with the calculated hash. - c. Check for an existing image with the calculated hash.
- d. If not found, check for a recent compatible image to use as a base. - d. If not found, check for a recent compatible image to use as a base.
@@ -119,9 +119,9 @@ The hash-based tag ensures exact reproducibility, while the generic tag provides
3. Image Reuse and Rebuilding Logic: 3. Image Reuse and Rebuilding Logic:
The system follows these steps to determine whether to build a new image or use an existing one from a user-provided (base) image (e.g., `ubuntu:22.04`): The system follows these steps to determine whether to build a new image or use an existing one from a user-provided (base) image (e.g., `ubuntu:22.04`):
a. If an image exists with the same hash (e.g., `od_runtime:abc123def456`), it will be reused as is. a. If an image exists with the same hash (e.g., `runtime:abc123def456`), it will be reused as is.
b. If the exact hash is not found, the system will try to rebuild using the latest generic image (e.g., `od_runtime:od_v0.8.3_ubuntu_tag_22.04`) as a base. This saves time by leveraging existing dependencies. b. If the exact hash is not found, the system will try to rebuild using the latest generic image (e.g., `runtime:od_v0.8.3_ubuntu_tag_22.04`) as a base. This saves time by leveraging existing dependencies.
c. If neither the hash-tagged nor the generic-tagged image is found, the system will build the image completely from scratch. c. If neither the hash-tagged nor the generic-tagged image is found, the system will build the image completely from scratch.
@@ -135,10 +135,10 @@ Here's a flowchart illustrating the build process:
```mermaid ```mermaid
flowchart TD flowchart TD
A[Start] --> B{Convert base image name} A[Start] --> B{Convert base image name}
B --> |ubuntu:22.04 -> od_runtime:od_v0.8.3_ubuntu_tag_22.04| C[Generate build context and hash] B --> |ubuntu:22.04 -> runtime:od_v0.8.3_ubuntu_tag_22.04| C[Generate build context and hash]
C --> D{Check for existing image with hash} C --> D{Check for existing image with hash}
D -->|Found od_runtime:abc123def456| E[Use existing image] D -->|Found runtime:abc123def456| E[Use existing image]
D -->|Not found| F{Check for od_runtime:od_v0.8.3_ubuntu_tag_22.04} D -->|Not found| F{Check for runtime:od_v0.8.3_ubuntu_tag_22.04}
F -->|Found| G[Rebuild based on recent image] F -->|Found| G[Rebuild based on recent image]
F -->|Not found| H[Build from scratch] F -->|Not found| H[Build from scratch]
G --> I[Tag with hash and generic tags] G --> I[Tag with hash and generic tags]
@@ -151,7 +151,7 @@ This approach ensures that:
1. Identical source code and Dockerfile always produce the same image (via hash-based tags). 1. Identical source code and Dockerfile always produce the same image (via hash-based tags).
2. The system can quickly rebuild images when minor changes occur (by leveraging recent compatible images). 2. The system can quickly rebuild images when minor changes occur (by leveraging recent compatible images).
3. The generic tag (e.g., `od_runtime:od_v0.8.3_ubuntu_tag_22.04`) always points to the latest build for a particular base image and OpenHands version combination. 3. The generic tag (e.g., `runtime:od_v0.8.3_ubuntu_tag_22.04`) always points to the latest build for a particular base image and OpenHands version combination.
By using this method, OpenHands maintains an efficient and flexible system for building and managing runtime images, adapting to both development needs and production requirements. By using this method, OpenHands maintains an efficient and flexible system for building and managing runtime images, adapting to both development needs and production requirements.

View File

@@ -52,7 +52,7 @@ def get_config(
container_image='xingyaoww/od-eval-logic-reasoning:v1.0', container_image='xingyaoww/od-eval-logic-reasoning:v1.0',
enable_auto_lint=True, enable_auto_lint=True,
use_host_network=False, use_host_network=False,
od_runtime_extra_deps='$OD_INTERPRETER_PATH -m pip install scitools-pyke', runtime_extra_deps='$OD_INTERPRETER_PATH -m pip install scitools-pyke',
), ),
# do not mount workspace # do not mount workspace
workspace_base=None, workspace_base=None,

View File

@@ -104,7 +104,7 @@ def get_config(
container_image='xingyaoww/od-eval-mint:v1.0', container_image='xingyaoww/od-eval-mint:v1.0',
enable_auto_lint=True, enable_auto_lint=True,
use_host_network=False, use_host_network=False,
od_runtime_extra_deps=f'$OD_INTERPRETER_PATH -m pip install {" ".join(MINT_DEPENDENCIES)}', runtime_extra_deps=f'$OD_INTERPRETER_PATH -m pip install {" ".join(MINT_DEPENDENCIES)}',
), ),
# do not mount workspace # do not mount workspace
workspace_base=None, workspace_base=None,

View File

@@ -58,7 +58,7 @@ def get_config(
enable_auto_lint=True, enable_auto_lint=True,
use_host_network=False, use_host_network=False,
browsergym_eval_env=env_id, browsergym_eval_env=env_id,
od_runtime_startup_env_vars={ runtime_startup_env_vars={
'BASE_URL': base_url, 'BASE_URL': base_url,
'OPENAI_API_KEY': openai_api_key, 'OPENAI_API_KEY': openai_api_key,
'SHOPPING': f'{base_url}:7770/', 'SHOPPING': f'{base_url}:7770/',

View File

@@ -185,12 +185,12 @@ class SandboxConfig(metaclass=Singleton):
enable_auto_lint: Whether to enable auto-lint. enable_auto_lint: Whether to enable auto-lint.
use_host_network: Whether to use the host network. use_host_network: Whether to use the host network.
initialize_plugins: Whether to initialize plugins. initialize_plugins: Whether to initialize plugins.
od_runtime_extra_deps: The extra dependencies to install in the runtime image (typically used for evaluation). runtime_extra_deps: The extra dependencies to install in the runtime image (typically used for evaluation).
This will be rendered into the end of the Dockerfile that builds the runtime image. This will be rendered into the end of the Dockerfile that builds the runtime image.
It can contain any valid shell commands (e.g., pip install numpy). It can contain any valid shell commands (e.g., pip install numpy).
The path to the interpreter is available as $OD_INTERPRETER_PATH, The path to the interpreter is available as $OD_INTERPRETER_PATH,
which can be used to install dependencies for the OD-specific Python interpreter. which can be used to install dependencies for the OD-specific Python interpreter.
od_runtime_startup_env_vars: The environment variables to set at the launch of the runtime. runtime_startup_env_vars: The environment variables to set at the launch of the runtime.
This is a dictionary of key-value pairs. This is a dictionary of key-value pairs.
This is useful for setting environment variables that are needed by the runtime. This is useful for setting environment variables that are needed by the runtime.
For example, for specifying the base url of website for browsergym evaluation. For example, for specifying the base url of website for browsergym evaluation.
@@ -207,8 +207,8 @@ class SandboxConfig(metaclass=Singleton):
) )
use_host_network: bool = False use_host_network: bool = False
initialize_plugins: bool = True initialize_plugins: bool = True
od_runtime_extra_deps: str | None = None runtime_extra_deps: str | None = None
od_runtime_startup_env_vars: dict[str, str] = field(default_factory=dict) runtime_startup_env_vars: dict[str, str] = field(default_factory=dict)
browsergym_eval_env: str | None = None browsergym_eval_env: str | None = None
def defaults_to_dict(self) -> dict: def defaults_to_dict(self) -> dict:

View File

@@ -75,15 +75,15 @@ class EventStreamRuntime(Runtime):
logger.debug(f'EventStreamRuntime `{sid}` config:\n{self.config}') logger.debug(f'EventStreamRuntime `{sid}` config:\n{self.config}')
async def ainit(self, env_vars: dict[str, str] | None = None): async def ainit(self, env_vars: dict[str, str] | None = None):
if self.config.sandbox.od_runtime_extra_deps: if self.config.sandbox.runtime_extra_deps:
logger.info( logger.info(
f'Installing extra user-provided dependencies in the runtime image: {self.config.sandbox.od_runtime_extra_deps}' f'Installing extra user-provided dependencies in the runtime image: {self.config.sandbox.runtime_extra_deps}'
) )
self.container_image = build_runtime_image( self.container_image = build_runtime_image(
self.container_image, self.container_image,
self.runtime_builder, self.runtime_builder,
extra_deps=self.config.sandbox.od_runtime_extra_deps, extra_deps=self.config.sandbox.runtime_extra_deps,
) )
self.container = await self._init_container( self.container = await self._init_container(
self.sandbox_workspace_dir, self.sandbox_workspace_dir,

View File

@@ -14,7 +14,7 @@ from openhands.core.logger import openhands_logger as logger
from openhands.runtime.builder import DockerRuntimeBuilder, RuntimeBuilder from openhands.runtime.builder import DockerRuntimeBuilder, RuntimeBuilder
RUNTIME_IMAGE_REPO = os.getenv( RUNTIME_IMAGE_REPO = os.getenv(
'OD_RUNTIME_RUNTIME_IMAGE_REPO', 'ghcr.io/openhands/od_runtime' 'OD_RUNTIME_RUNTIME_IMAGE_REPO', 'ghcr.io/openhands/runtime'
) )
@@ -177,7 +177,7 @@ def get_runtime_image_repo_and_tag(base_image: str) -> tuple[str, str]:
if RUNTIME_IMAGE_REPO in base_image: if RUNTIME_IMAGE_REPO in base_image:
logger.info( logger.info(
f'The provided image [{base_image}] is a already a valid od_runtime image.\n' f'The provided image [{base_image}] is a already a valid runtime image.\n'
f'Will try to reuse it as is.' f'Will try to reuse it as is.'
) )
@@ -235,7 +235,7 @@ def build_runtime_image(
# non-hash generic image name, it could contain *similar* dependencies # non-hash generic image name, it could contain *similar* dependencies
# but *might* not exactly match the state of the source code. # but *might* not exactly match the state of the source code.
# It resembles the "latest" tag in the docker image naming convention for # It resembles the "latest" tag in the docker image naming convention for
# a particular {repo}:{tag} pair (e.g., ubuntu:latest -> od_runtime:ubuntu_tag_latest) # a particular {repo}:{tag} pair (e.g., ubuntu:latest -> runtime:ubuntu_tag_latest)
# we will build from IT to save time if the `from_scratch_hash` is not found # we will build from IT to save time if the `from_scratch_hash` is not found
generic_runtime_image_name = f'{runtime_image_repo}:{runtime_image_tag}' generic_runtime_image_name = f'{runtime_image_repo}:{runtime_image_tag}'