Skip to content

Commit

Permalink
add flytekit base image with flyteidl-rust
Browse files Browse the repository at this point in the history
Signed-off-by: Austin Liu <austin362667@gmail.com>
  • Loading branch information
austin362667 committed Aug 11, 2024
1 parent 308a4fb commit 1d0bef5
Showing 1 changed file with 88 additions and 0 deletions.
88 changes: 88 additions & 0 deletions Dockerfile.rust
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
# This Dockerfile aims to build and install the `flyteidl-rust` Python package locally.

# This Dockerfile is here to help with end-to-end testing
# From flytekit
# $ docker build -f Dockerfile.rust --build-arg PYTHON_VERSION=3.12 -t localhost:30000/flytekittest:someversion .
# $ docker push localhost:30000/flytekittest:someversion
# From your test user code
# $ pyflyte run --remote --image localhost:30000/flytekittest:someversion

ARG PYTHON_VERSION

# Stage 1: Build environment: this base will be tossed out after stage 1 finished
FROM python:${PYTHON_VERSION}-slim-bookworm as builder

# Install necessary dependencies
RUN apt-get update && apt-get install build-essential vim libmagic1 git curl -y

# Install Rust
RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y

# Set environment variables for Rust
ENV PATH="/root/.cargo/bin:${PATH}"

# Install maturin to build Python bindings with PyO3
RUN pip install maturin

RUN git clone -b flyrs https://github.com/austin362667/flyte.git

# Build the Python bindings
RUN --mount=type=cache,target=/usr/local/cargo/registry maturin build --release --out dist -m flyte/flyteidl/Cargo.toml


# Stage 2: Runtime environment: The fresh and final base image
FROM python:${PYTHON_VERSION}-slim-bookworm

# Scoop out our fully-built wheel file from the `builder` stage
COPY --from=builder dist/*.whl .

# Install the built `flyteidl-rust` Python package locally
RUN pip install *.whl


MAINTAINER Flyte Team <users@flyte.org>
LABEL org.opencontainers.image.source=https://github.com/flyteorg/flytekit

WORKDIR /root
ENV FLYTE_SDK_RICH_TRACEBACKS 0

# Flytekit version of flytekit to be installed in the image
ARG PSEUDO_VERSION

# Note: Pod tasks should be exposed in the default image
# Note: Some packages will create config files under /home by default, so we need to make sure it's writable
# Note: There are use cases that require reading and writing files under /tmp, so we need to change its permissions.

# Run a series of commands to set up the environment:
# 1. Update and install dependencies.
# 2. Install Flytekit and its plugins.
# 3. Clean up the apt cache to reduce image size. Reference: https://gist.github.com/marvell/7c812736565928e602c4
# 4. Create a non-root user 'flytekit' and set appropriate permissions for directories.
RUN apt-get update && apt-get install build-essential vim libmagic1 git -y \
&& pip install uv

COPY . /flytekit

# Use a future version of SETUPTOOLS_SCM_PRETEND_VERSION_FOR_FLYTEIDL such that uv resolution works.
RUN SETUPTOOLS_SCM_PRETEND_VERSION_FOR_FLYTEKIT=$PSEUDO_VERSION \
SETUPTOOLS_SCM_PRETEND_VERSION_FOR_FLYTEIDL=3.0.0dev0 \
uv pip install --system --no-cache-dir -U \
"git+https://github.com/flyteorg/flyte.git@master#subdirectory=flyteidl" \
-e /flytekit \
-e /flytekit/plugins/flytekit-k8s-pod \
-e /flytekit/plugins/flytekit-deck-standard \
-e /flytekit/plugins/flytekit-flyteinteractive \
scikit-learn \
&& apt-get clean autoclean \
&& apt-get autoremove --yes \
&& rm -rf /var/lib/{apt,dpkg,cache,log}/ \
&& useradd -u 1000 flytekit \
&& chown flytekit: /root \
&& chown flytekit: /home \
&& :


ENV PYTHONPATH "/flytekit:/flytekit/plugins/flytekit-k8s-pod:/flytekit/plugins/flytekit-deck-standard:"

# Switch to the 'flytekit' user for better security.
USER flytekit

0 comments on commit 1d0bef5

Please sign in to comment.