mirror of
https://github.com/All-Hands-AI/OpenHands.git
synced 2024-08-29 01:18:33 +03:00
fix: CI docker image push (#3476)
* fix ghcr app * fix ghcr runtime push * rename od_runtime to runtime
This commit is contained in:
4
.github/workflows/ghcr_app.yml
vendored
4
.github/workflows/ghcr_app.yml
vendored
@@ -69,7 +69,7 @@ jobs:
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: ${{ matrix.image }}_image_${{ matrix.platform }}
|
||||
path: /tmp/${{ matrix.image }}_$(echo "${{ steps.capture-tags.outputs.tags }}" | awk '{print $NF}')_${{ matrix.platform }}.tar
|
||||
path: /tmp/${{ matrix.image }}_${{ steps.capture-tags.outputs.tags }}_${{ matrix.platform }}.tar
|
||||
retention-days: 14
|
||||
|
||||
# Push the OpenHands and sandbox Docker images to the ghcr.io repository
|
||||
@@ -99,7 +99,7 @@ jobs:
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: ${{ matrix.image }}_image_${{ matrix.platform }}
|
||||
path: /tmp/${{ matrix.image }}_${{ steps.capture-tags.outputs.tags }}_${{ matrix.platform }}.tar
|
||||
path: /tmp
|
||||
- name: Load images and push to registry
|
||||
run: |
|
||||
mv /tmp/${{ matrix.platform }}/${{ matrix.image }}_${{ steps.capture-tags.outputs.tags }}_${{ matrix.platform }}.tar .
|
||||
|
||||
19
.github/workflows/ghcr_runtime.yml
vendored
19
.github/workflows/ghcr_runtime.yml
vendored
@@ -31,7 +31,7 @@ jobs:
|
||||
packages: write
|
||||
strategy:
|
||||
matrix:
|
||||
image: ['od_runtime']
|
||||
image: ['runtime']
|
||||
base_image: ['nikolaik/python-nodejs:python3.11-nodejs22', 'python:3.11-bookworm', 'node:22-bookworm']
|
||||
platform: ['amd64', 'arm64']
|
||||
outputs:
|
||||
@@ -139,7 +139,7 @@ jobs:
|
||||
needs: prepare_test_image_tags
|
||||
strategy:
|
||||
matrix:
|
||||
image: ['od_runtime']
|
||||
image: ['runtime']
|
||||
runtime_type: ['eventstream']
|
||||
platform: ['amd64']
|
||||
last_tag: ${{ fromJson(needs.prepare_test_image_tags.outputs.test_image_tags) }}
|
||||
@@ -205,7 +205,7 @@ jobs:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
image: ['od_runtime']
|
||||
image: ['runtime']
|
||||
runtime_type: ['eventstream']
|
||||
platform: ['amd64']
|
||||
last_tag: ${{ fromJson(needs.prepare_test_image_tags.outputs.test_image_tags) }}
|
||||
@@ -270,7 +270,7 @@ jobs:
|
||||
packages: write
|
||||
strategy:
|
||||
matrix:
|
||||
image: ['od_runtime']
|
||||
image: ['runtime']
|
||||
runtime_type: ['eventstream']
|
||||
platform: ['amd64', 'arm64']
|
||||
last_tag: ${{ fromJson(needs.prepare_test_image_tags.outputs.test_image_tags) }}
|
||||
@@ -297,17 +297,18 @@ jobs:
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: ${{ matrix.image }}_${{ matrix.last_tag }}_${{ matrix.platform }}
|
||||
path: /tmp/${{ matrix.image }}_${{ matrix.last_tag }}_${{ matrix.platform }}.tar
|
||||
path: /tmp/
|
||||
- name: List downloaded files
|
||||
run: |
|
||||
ls -la /tmp/*
|
||||
ls -la /tmp/${{ matrix.platform }}
|
||||
- name: Load images and push to registry
|
||||
run: |
|
||||
image_file=$(find /tmp/${{ matrix.platform }} -name "${{ matrix.image }}_${{ matrix.last_tag }}_${{ matrix.platform }}.tar" | head -n 1)
|
||||
image_file=$(find /tmp -name "${{ matrix.image }}_${{ matrix.last_tag }}_${{ matrix.platform }}.tar" | head -n 1)
|
||||
if [ -z "$image_file" ]; then
|
||||
echo "No matching image file found"
|
||||
echo "No matching image file found for tag: ${{ matrix.last_tag }}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Loading image from file: $image_file"
|
||||
if ! loaded_image=$(docker load -i "$image_file" | grep "Loaded image:" | head -n 1 | awk '{print $3}'); then
|
||||
echo "Failed to load Docker image"
|
||||
@@ -335,7 +336,7 @@ jobs:
|
||||
tags: ${{ needs.ghcr_build_runtime.outputs.tags }}
|
||||
strategy:
|
||||
matrix:
|
||||
image: ['od_runtime']
|
||||
image: ['runtime']
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
|
||||
2
.gitignore
vendored
2
.gitignore
vendored
@@ -223,7 +223,7 @@ containers/agnostic_sandbox
|
||||
image_build_logs
|
||||
run_instance_logs
|
||||
|
||||
od_runtime_*.tar
|
||||
runtime_*.tar
|
||||
|
||||
# docker build
|
||||
containers/runtime/Dockerfile
|
||||
|
||||
@@ -27,13 +27,13 @@ echo "Tags: ${tags[@]}"
|
||||
|
||||
if [[ "$image_name" == "openhands" ]]; then
|
||||
dir="./containers/app"
|
||||
elif [[ "$image_name" == "od_runtime" ]]; then
|
||||
elif [[ "$image_name" == "runtime" ]]; then
|
||||
dir="./containers/runtime"
|
||||
else
|
||||
dir="./containers/$image_name"
|
||||
fi
|
||||
|
||||
if [[ (! -f "$dir/Dockerfile") && "$image_name" != "od_runtime" ]]; then
|
||||
if [[ (! -f "$dir/Dockerfile") && "$image_name" != "runtime" ]]; then
|
||||
# Allow runtime to be built without a Dockerfile
|
||||
echo "No Dockerfile found"
|
||||
exit 1
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
DOCKER_REGISTRY=ghcr.io
|
||||
DOCKER_ORG=all-hands-ai
|
||||
DOCKER_BASE_DIR="./containers/runtime"
|
||||
DOCKER_IMAGE=od_runtime
|
||||
DOCKER_IMAGE=runtime
|
||||
# These variables will be appended by the runtime_build.py script
|
||||
# DOCKER_IMAGE_TAG=
|
||||
# DOCKER_IMAGE_HASH_TAG=
|
||||
|
||||
@@ -84,16 +84,16 @@ Check out [relavant code](https://github.com/All-Hands-AI/OpenHands/blob/main/op
|
||||
OpenHands uses a dual-tagging system for its runtime images to balance reproducibility with flexibility:
|
||||
|
||||
1. Hash-based tag: `{target_image_repo}:{target_image_hash_tag}`
|
||||
Example: `od_runtime:abc123def456`
|
||||
Example: `runtime:abc123def456`
|
||||
|
||||
- This tag is based on the MD5 hash of the Docker build folder, which includes the source code (of runtime client and related dependencies) and Dockerfile.
|
||||
- Identical hash tags guarantee that the images were built with exactly the same source code and Dockerfile.
|
||||
- This ensures reproducibility: the same hash always means the same image contents.
|
||||
|
||||
2. Generic tag: `{target_image_repo}:{target_image_tag}`
|
||||
Example: `od_runtime:od_v0.8.3_ubuntu_tag_22.04`
|
||||
Example: `runtime:od_v0.8.3_ubuntu_tag_22.04`
|
||||
|
||||
- This tag follows the format: `od_runtime:od_v{OD_VERSION}_{BASE_IMAGE_NAME}_tag_{BASE_IMAGE_TAG}`
|
||||
- This tag follows the format: `runtime:od_v{OD_VERSION}_{BASE_IMAGE_NAME}_tag_{BASE_IMAGE_TAG}`
|
||||
- It represents the latest build for a particular base image and OpenHands version combination.
|
||||
- This tag is updated whenever a new image is built from the same base image, even if the source code changes.
|
||||
|
||||
@@ -103,13 +103,13 @@ The hash-based tag ensures exact reproducibility, while the generic tag provides
|
||||
|
||||
1. Image Naming Convention:
|
||||
- Hash-based tag: `{target_image_repo}:{target_image_hash_tag}`
|
||||
Example: `od_runtime:abc123def456`
|
||||
Example: `runtime:abc123def456`
|
||||
- Generic tag: `{target_image_repo}:{target_image_tag}`
|
||||
Example: `od_runtime:od_v0.8.3_ubuntu_tag_22.04`
|
||||
Example: `runtime:od_v0.8.3_ubuntu_tag_22.04`
|
||||
|
||||
2. Build Process:
|
||||
- a. Convert the base image name to an OD runtime image name.
|
||||
Example: `ubuntu:22.04` -> `od_runtime:od_v0.8.3_ubuntu_tag_22.04`
|
||||
Example: `ubuntu:22.04` -> `runtime:od_v0.8.3_ubuntu_tag_22.04`
|
||||
- b. Generate a build context (Dockerfile and OpenHands source code) and calculate its hash.
|
||||
- c. Check for an existing image with the calculated hash.
|
||||
- d. If not found, check for a recent compatible image to use as a base.
|
||||
@@ -119,9 +119,9 @@ The hash-based tag ensures exact reproducibility, while the generic tag provides
|
||||
3. Image Reuse and Rebuilding Logic:
|
||||
The system follows these steps to determine whether to build a new image or use an existing one from a user-provided (base) image (e.g., `ubuntu:22.04`):
|
||||
|
||||
a. If an image exists with the same hash (e.g., `od_runtime:abc123def456`), it will be reused as is.
|
||||
a. If an image exists with the same hash (e.g., `runtime:abc123def456`), it will be reused as is.
|
||||
|
||||
b. If the exact hash is not found, the system will try to rebuild using the latest generic image (e.g., `od_runtime:od_v0.8.3_ubuntu_tag_22.04`) as a base. This saves time by leveraging existing dependencies.
|
||||
b. If the exact hash is not found, the system will try to rebuild using the latest generic image (e.g., `runtime:od_v0.8.3_ubuntu_tag_22.04`) as a base. This saves time by leveraging existing dependencies.
|
||||
|
||||
c. If neither the hash-tagged nor the generic-tagged image is found, the system will build the image completely from scratch.
|
||||
|
||||
@@ -135,10 +135,10 @@ Here's a flowchart illustrating the build process:
|
||||
```mermaid
|
||||
flowchart TD
|
||||
A[Start] --> B{Convert base image name}
|
||||
B --> |ubuntu:22.04 -> od_runtime:od_v0.8.3_ubuntu_tag_22.04| C[Generate build context and hash]
|
||||
B --> |ubuntu:22.04 -> runtime:od_v0.8.3_ubuntu_tag_22.04| C[Generate build context and hash]
|
||||
C --> D{Check for existing image with hash}
|
||||
D -->|Found od_runtime:abc123def456| E[Use existing image]
|
||||
D -->|Not found| F{Check for od_runtime:od_v0.8.3_ubuntu_tag_22.04}
|
||||
D -->|Found runtime:abc123def456| E[Use existing image]
|
||||
D -->|Not found| F{Check for runtime:od_v0.8.3_ubuntu_tag_22.04}
|
||||
F -->|Found| G[Rebuild based on recent image]
|
||||
F -->|Not found| H[Build from scratch]
|
||||
G --> I[Tag with hash and generic tags]
|
||||
@@ -151,7 +151,7 @@ This approach ensures that:
|
||||
|
||||
1. Identical source code and Dockerfile always produce the same image (via hash-based tags).
|
||||
2. The system can quickly rebuild images when minor changes occur (by leveraging recent compatible images).
|
||||
3. The generic tag (e.g., `od_runtime:od_v0.8.3_ubuntu_tag_22.04`) always points to the latest build for a particular base image and OpenHands version combination.
|
||||
3. The generic tag (e.g., `runtime:od_v0.8.3_ubuntu_tag_22.04`) always points to the latest build for a particular base image and OpenHands version combination.
|
||||
|
||||
By using this method, OpenHands maintains an efficient and flexible system for building and managing runtime images, adapting to both development needs and production requirements.
|
||||
|
||||
|
||||
@@ -52,7 +52,7 @@ def get_config(
|
||||
container_image='xingyaoww/od-eval-logic-reasoning:v1.0',
|
||||
enable_auto_lint=True,
|
||||
use_host_network=False,
|
||||
od_runtime_extra_deps='$OD_INTERPRETER_PATH -m pip install scitools-pyke',
|
||||
runtime_extra_deps='$OD_INTERPRETER_PATH -m pip install scitools-pyke',
|
||||
),
|
||||
# do not mount workspace
|
||||
workspace_base=None,
|
||||
|
||||
@@ -104,7 +104,7 @@ def get_config(
|
||||
container_image='xingyaoww/od-eval-mint:v1.0',
|
||||
enable_auto_lint=True,
|
||||
use_host_network=False,
|
||||
od_runtime_extra_deps=f'$OD_INTERPRETER_PATH -m pip install {" ".join(MINT_DEPENDENCIES)}',
|
||||
runtime_extra_deps=f'$OD_INTERPRETER_PATH -m pip install {" ".join(MINT_DEPENDENCIES)}',
|
||||
),
|
||||
# do not mount workspace
|
||||
workspace_base=None,
|
||||
|
||||
@@ -58,7 +58,7 @@ def get_config(
|
||||
enable_auto_lint=True,
|
||||
use_host_network=False,
|
||||
browsergym_eval_env=env_id,
|
||||
od_runtime_startup_env_vars={
|
||||
runtime_startup_env_vars={
|
||||
'BASE_URL': base_url,
|
||||
'OPENAI_API_KEY': openai_api_key,
|
||||
'SHOPPING': f'{base_url}:7770/',
|
||||
|
||||
@@ -185,12 +185,12 @@ class SandboxConfig(metaclass=Singleton):
|
||||
enable_auto_lint: Whether to enable auto-lint.
|
||||
use_host_network: Whether to use the host network.
|
||||
initialize_plugins: Whether to initialize plugins.
|
||||
od_runtime_extra_deps: The extra dependencies to install in the runtime image (typically used for evaluation).
|
||||
runtime_extra_deps: The extra dependencies to install in the runtime image (typically used for evaluation).
|
||||
This will be rendered into the end of the Dockerfile that builds the runtime image.
|
||||
It can contain any valid shell commands (e.g., pip install numpy).
|
||||
The path to the interpreter is available as $OD_INTERPRETER_PATH,
|
||||
which can be used to install dependencies for the OD-specific Python interpreter.
|
||||
od_runtime_startup_env_vars: The environment variables to set at the launch of the runtime.
|
||||
runtime_startup_env_vars: The environment variables to set at the launch of the runtime.
|
||||
This is a dictionary of key-value pairs.
|
||||
This is useful for setting environment variables that are needed by the runtime.
|
||||
For example, for specifying the base url of website for browsergym evaluation.
|
||||
@@ -207,8 +207,8 @@ class SandboxConfig(metaclass=Singleton):
|
||||
)
|
||||
use_host_network: bool = False
|
||||
initialize_plugins: bool = True
|
||||
od_runtime_extra_deps: str | None = None
|
||||
od_runtime_startup_env_vars: dict[str, str] = field(default_factory=dict)
|
||||
runtime_extra_deps: str | None = None
|
||||
runtime_startup_env_vars: dict[str, str] = field(default_factory=dict)
|
||||
browsergym_eval_env: str | None = None
|
||||
|
||||
def defaults_to_dict(self) -> dict:
|
||||
|
||||
@@ -75,15 +75,15 @@ class EventStreamRuntime(Runtime):
|
||||
logger.debug(f'EventStreamRuntime `{sid}` config:\n{self.config}')
|
||||
|
||||
async def ainit(self, env_vars: dict[str, str] | None = None):
|
||||
if self.config.sandbox.od_runtime_extra_deps:
|
||||
if self.config.sandbox.runtime_extra_deps:
|
||||
logger.info(
|
||||
f'Installing extra user-provided dependencies in the runtime image: {self.config.sandbox.od_runtime_extra_deps}'
|
||||
f'Installing extra user-provided dependencies in the runtime image: {self.config.sandbox.runtime_extra_deps}'
|
||||
)
|
||||
|
||||
self.container_image = build_runtime_image(
|
||||
self.container_image,
|
||||
self.runtime_builder,
|
||||
extra_deps=self.config.sandbox.od_runtime_extra_deps,
|
||||
extra_deps=self.config.sandbox.runtime_extra_deps,
|
||||
)
|
||||
self.container = await self._init_container(
|
||||
self.sandbox_workspace_dir,
|
||||
|
||||
@@ -14,7 +14,7 @@ from openhands.core.logger import openhands_logger as logger
|
||||
from openhands.runtime.builder import DockerRuntimeBuilder, RuntimeBuilder
|
||||
|
||||
RUNTIME_IMAGE_REPO = os.getenv(
|
||||
'OD_RUNTIME_RUNTIME_IMAGE_REPO', 'ghcr.io/openhands/od_runtime'
|
||||
'OD_RUNTIME_RUNTIME_IMAGE_REPO', 'ghcr.io/openhands/runtime'
|
||||
)
|
||||
|
||||
|
||||
@@ -177,7 +177,7 @@ def get_runtime_image_repo_and_tag(base_image: str) -> tuple[str, str]:
|
||||
|
||||
if RUNTIME_IMAGE_REPO in base_image:
|
||||
logger.info(
|
||||
f'The provided image [{base_image}] is a already a valid od_runtime image.\n'
|
||||
f'The provided image [{base_image}] is a already a valid runtime image.\n'
|
||||
f'Will try to reuse it as is.'
|
||||
)
|
||||
|
||||
@@ -235,7 +235,7 @@ def build_runtime_image(
|
||||
# non-hash generic image name, it could contain *similar* dependencies
|
||||
# but *might* not exactly match the state of the source code.
|
||||
# It resembles the "latest" tag in the docker image naming convention for
|
||||
# a particular {repo}:{tag} pair (e.g., ubuntu:latest -> od_runtime:ubuntu_tag_latest)
|
||||
# a particular {repo}:{tag} pair (e.g., ubuntu:latest -> runtime:ubuntu_tag_latest)
|
||||
# we will build from IT to save time if the `from_scratch_hash` is not found
|
||||
generic_runtime_image_name = f'{runtime_image_repo}:{runtime_image_tag}'
|
||||
|
||||
|
||||
Reference in New Issue
Block a user