-
Notifications
You must be signed in to change notification settings - Fork 297
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
add flytekit base image with flyteidl-rust
Signed-off-by: Austin Liu <austin362667@gmail.com>
- Loading branch information
1 parent
308a4fb
commit 1d0bef5
Showing
1 changed file
with
88 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,88 @@ | ||
# This Dockerfile aims to build and install the `flyteidl-rust` Python package locally. | ||
|
||
# This Dockerfile is here to help with end-to-end testing | ||
# From flytekit | ||
# $ docker build -f Dockerfile.rust --build-arg PYTHON_VERSION=3.12 -t localhost:30000/flytekittest:someversion . | ||
# $ docker push localhost:30000/flytekittest:someversion | ||
# From your test user code | ||
# $ pyflyte run --remote --image localhost:30000/flytekittest:someversion | ||
|
||
ARG PYTHON_VERSION | ||
|
||
# Stage 1: Build environment: this base will be tossed out after stage 1 finished | ||
FROM python:${PYTHON_VERSION}-slim-bookworm as builder | ||
|
||
# Install necessary dependencies | ||
RUN apt-get update && apt-get install build-essential vim libmagic1 git curl -y | ||
|
||
# Install Rust | ||
RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y | ||
|
||
# Set environment variables for Rust | ||
ENV PATH="/root/.cargo/bin:${PATH}" | ||
|
||
# Install maturin to build Python bindings with PyO3 | ||
RUN pip install maturin | ||
|
||
RUN git clone -b flyrs https://github.com/austin362667/flyte.git | ||
|
||
# Build the Python bindings | ||
RUN --mount=type=cache,target=/usr/local/cargo/registry maturin build --release --out dist -m flyte/flyteidl/Cargo.toml | ||
|
||
|
||
# Stage 2: Runtime environment: The fresh and final base image | ||
FROM python:${PYTHON_VERSION}-slim-bookworm | ||
|
||
# Scoop out our fully-built wheel file from the `builder` stage | ||
COPY --from=builder dist/*.whl . | ||
|
||
# Install the built `flyteidl-rust` Python package locally | ||
RUN pip install *.whl | ||
|
||
|
||
MAINTAINER Flyte Team <users@flyte.org> | ||
LABEL org.opencontainers.image.source=https://github.com/flyteorg/flytekit | ||
|
||
WORKDIR /root | ||
ENV FLYTE_SDK_RICH_TRACEBACKS 0 | ||
|
||
# Flytekit version of flytekit to be installed in the image | ||
ARG PSEUDO_VERSION | ||
|
||
# Note: Pod tasks should be exposed in the default image | ||
# Note: Some packages will create config files under /home by default, so we need to make sure it's writable | ||
# Note: There are use cases that require reading and writing files under /tmp, so we need to change its permissions. | ||
|
||
# Run a series of commands to set up the environment: | ||
# 1. Update and install dependencies. | ||
# 2. Install Flytekit and its plugins. | ||
# 3. Clean up the apt cache to reduce image size. Reference: https://gist.github.com/marvell/7c812736565928e602c4 | ||
# 4. Create a non-root user 'flytekit' and set appropriate permissions for directories. | ||
RUN apt-get update && apt-get install build-essential vim libmagic1 git -y \ | ||
&& pip install uv | ||
|
||
COPY . /flytekit | ||
|
||
# Use a future version of SETUPTOOLS_SCM_PRETEND_VERSION_FOR_FLYTEIDL such that uv resolution works. | ||
RUN SETUPTOOLS_SCM_PRETEND_VERSION_FOR_FLYTEKIT=$PSEUDO_VERSION \ | ||
SETUPTOOLS_SCM_PRETEND_VERSION_FOR_FLYTEIDL=3.0.0dev0 \ | ||
uv pip install --system --no-cache-dir -U \ | ||
"git+https://github.com/flyteorg/flyte.git@master#subdirectory=flyteidl" \ | ||
-e /flytekit \ | ||
-e /flytekit/plugins/flytekit-k8s-pod \ | ||
-e /flytekit/plugins/flytekit-deck-standard \ | ||
-e /flytekit/plugins/flytekit-flyteinteractive \ | ||
scikit-learn \ | ||
&& apt-get clean autoclean \ | ||
&& apt-get autoremove --yes \ | ||
&& rm -rf /var/lib/{apt,dpkg,cache,log}/ \ | ||
&& useradd -u 1000 flytekit \ | ||
&& chown flytekit: /root \ | ||
&& chown flytekit: /home \ | ||
&& : | ||
|
||
|
||
ENV PYTHONPATH "/flytekit:/flytekit/plugins/flytekit-k8s-pod:/flytekit/plugins/flytekit-deck-standard:" | ||
|
||
# Switch to the 'flytekit' user for better security. | ||
USER flytekit |