alihan/nv-ingest
1
0
Fork 0
mirror of https://github.com/NVIDIA/nv-ingest.git synced 2025-01-05 18:58:13 +03:00
Code Issues Packages Projects Releases Wiki Activity

Add caching for conda packages to Dockerfile to speed up successive builds. (#289)

Browse Source
This commit is contained in:
Devin Robison
2024-12-16 15:22:55 -07:00
committed by GitHub
parent b4aff512c4
commit a338a0297c
1 changed files with 25 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
Dockerfile
View File

@@ -30,12 +30,15 @@ RUN wget -O Miniforge3.sh "https://github.com/conda-forge/miniforge/releases/lat
ENV PATH=/opt/conda/bin:$PATH
# Install Mamba, a faster alternative to conda, within the base environment
RUN conda install -y mamba -n base -c conda-forge
RUN --mount=type=cache,target=/opt/conda/pkgs \
--mount=type=cache,target=/root/.cache/pip \
conda install -y mamba conda-build==24.5.1 -n base -c conda-forge
COPY conda/environments/nv_ingest_environment.yml /workspace/nv_ingest_environment.yml
# Create nv_ingest base environment
RUN mamba env create -f /workspace/nv_ingest_environment.yml \
&& conda clean --all --yes
RUN --mount=type=cache,target=/opt/conda/pkgs \
--mount=type=cache,target=/root/.cache/pip \
mamba env create -f /workspace/nv_ingest_environment.yml
# Set default shell to bash
SHELL ["/bin/bash", "-c"]
@@ -44,9 +47,10 @@ SHELL ["/bin/bash", "-c"]
RUN echo "source activate nv_ingest_runtime" >> ~/.bashrc
# Install Tini via conda from the conda-forge channel
RUN source activate nv_ingest_runtime \
&& mamba install -y -c conda-forge tini \
&& conda clean --all --yes
RUN --mount=type=cache,target=/opt/conda/pkgs \
--mount=type=cache,target=/root/.cache/pip \
source activate nv_ingest_runtime \
&& mamba install -y -c conda-forge tini
# Set the working directory in the container
WORKDIR /workspace
@@ -64,15 +68,15 @@ ENV HAYSTACK_TELEMETRY_ENABLED=False
# Ensure the NV_INGEST_VERSION is PEP 440 compatible
RUN if [ -z "${VERSION}" ]; then \
export VERSION="$(date +'%Y.%m.%d')"; \
export VERSION="$(date +'%Y.%m.%d')"; \
fi; \
if [ "${RELEASE_TYPE}" = "dev" ]; then \
export NV_INGEST_VERSION_OVERRIDE="${VERSION}.dev${VERSION_REV}"; \
export NV_INGEST_VERSION_OVERRIDE="${VERSION}.dev${VERSION_REV}"; \
elif [ "${RELEASE_TYPE}" = "release" ]; then \
export NV_INGEST_VERSION_OVERRIDE="${VERSION}.post${VERSION_REV}"; \
export NV_INGEST_VERSION_OVERRIDE="${VERSION}.post${VERSION_REV}"; \
else \
echo "Invalid RELEASE_TYPE: ${RELEASE_TYPE}"; \
exit 1; \
echo "Invalid RELEASE_TYPE: ${RELEASE_TYPE}"; \
exit 1; \
fi
ENV NV_INGEST_RELEASE_TYPE=${RELEASE_TYPE}
@@ -87,12 +91,16 @@ COPY client client
COPY src/nv_ingest src/nv_ingest
RUN rm -rf ./src/nv_ingest/dist ./client/dist
# Run the build_pip_packages.sh script with the specified build type and library
RUN chmod +x ./ci/scripts/build_pip_packages.sh \
# Add pip cache path to match conda's package cache
RUN --mount=type=cache,target=/opt/conda/pkgs \
--mount=type=cache,target=/root/.cache/pip \
chmod +x ./ci/scripts/build_pip_packages.sh \
&& ./ci/scripts/build_pip_packages.sh --type ${RELEASE_TYPE} --lib client \
&& ./ci/scripts/build_pip_packages.sh --type ${RELEASE_TYPE} --lib service
RUN source activate nv_ingest_runtime \
RUN --mount=type=cache,target=/opt/conda/pkgs\
--mount=type=cache,target=/root/.cache/pip \
source activate nv_ingest_runtime \
&& pip install ./dist/*.whl \
&& pip install ./client/dist/*.whl
@@ -110,7 +118,9 @@ ENTRYPOINT ["/opt/conda/envs/nv_ingest_runtime/bin/tini", "--", "/workspace/dock
FROM nv_ingest_install AS development
RUN source activate nv_ingest && \
RUN source activate nv_ingest_runtime && \
--mount=type=cache,target=/opt/conda/pkgs \
--mount=type=cache,target=/root/.cache/pip \
pip install -e ./client
CMD ["/bin/bash"]