Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add docker commands in ImageSpec #2676

Open
wants to merge 29 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 22 commits
Commits
Show all changes
29 commits
Select commit Hold shift + click to select a range
662f15e
feat: add docker commands in ImageSpec
mao3267 Aug 9, 2024
4f670dc
use private-key (#2645)
Future-Outlier Aug 2, 2024
b4e7ece
Don't call remote when --help in remote-X (#2642)
wild-endeavor Aug 2, 2024
a22792f
Bump grpc receive message size (#2640)
wild-endeavor Aug 3, 2024
fbba823
Raise an exception when filters' value isn't a list. (#2576)
arbaobao Aug 5, 2024
b946674
Update error message for TypeTransformerFailedError (#2648)
wayner0628 Aug 5, 2024
ffc030e
[Error Message] Dataclasses Mismatched Type (#2650)
Future-Outlier Aug 6, 2024
ac8563d
Added warning for command list and shell true (#2653)
pryce-turner Aug 6, 2024
b59346a
In `FlyteRemote.upload_file`, pass the file object directly rather th…
redartera Aug 6, 2024
f11f397
Modify test_array_node.py to support running in python 3.8 (#2652)
eapolinario Aug 6, 2024
901fe4f
Handle common cases of mutable default arguments explicitly (#2651)
eapolinario Aug 6, 2024
9e0dbf5
Allow a hash method to be present for numpy arrays (#2649)
demmerichs Aug 6, 2024
140b58f
return exceptions when gathering (#2657)
wild-endeavor Aug 7, 2024
78a6219
Correct FlyteFile docstring (#2658)
ppiegaze Aug 8, 2024
8d229d9
Remove pip cache after install (#2662)
thomasjpfan Aug 8, 2024
59b5932
Adds validation to image_spec for list of strings (#2655)
thomasjpfan Aug 8, 2024
be3b145
Make elastic timeout configurable for HorovovJob. (#2631)
supercharleszhu Aug 8, 2024
9ea6b23
Fix overriding of loader_args task resolver in papermill plugin (#2660)
eapolinario Aug 8, 2024
e83c240
Catch all exceptions when rendering python dependencies (#2664)
thomasjpfan Aug 8, 2024
b3dbc08
Don't check the retrun statement for reference_launch_plan (#2665)
pingsutw Aug 8, 2024
72319a2
Bump flyteidl to 1.13.1 (#2666)
mao3267 Aug 9, 2024
5a72f98
Merge branch 'master' of https://github.com/mao3267/flytekit into add…
mao3267 Aug 11, 2024
8446503
fix: raise exception for docker commands in envd builder
mao3267 Aug 13, 2024
0a85ccc
test: add unit tests for docker commands
mao3267 Aug 13, 2024
f18c283
fix: modify error message
mao3267 Aug 16, 2024
f073069
test: raise exception for docker commands in envd builder
mao3267 Aug 16, 2024
f2c3679
test: ImageSpec with docker commands
mao3267 Aug 16, 2024
767ef0f
Merge branch 'master' of https://github.com/mao3267/flytekit into add…
mao3267 Aug 16, 2024
cca874b
fix: add docker commands to parameters_str_list
mao3267 Aug 17, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions flytekit/image_spec/default_builder.py
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,7 @@

$COPY_COMMAND_RUNTIME
RUN $RUN_COMMANDS
$DOCKER_COMMANDS

WORKDIR /root
SHELL ["/bin/bash", "-c"]
Expand Down Expand Up @@ -232,6 +233,11 @@ def create_docker_context(image_spec: ImageSpec, tmp_dir: Path):
else:
run_commands = ""

if image_spec.docker_commands:
pingsutw marked this conversation as resolved.
Show resolved Hide resolved
docker_commands = "\n".join(image_spec.docker_commands)
else:
docker_commands = ""

docker_content = DOCKER_FILE_TEMPLATE.substitute(
PYTHON_VERSION=python_version,
UV_PYTHON_INSTALL_COMMAND=uv_python_install_command,
Expand All @@ -244,6 +250,7 @@ def create_docker_context(image_spec: ImageSpec, tmp_dir: Path):
COPY_COMMAND_RUNTIME=copy_command_runtime,
ENTRYPOINT=entrypoint,
RUN_COMMANDS=run_commands,
DOCKER_COMMANDS=docker_commands,
)

dockerfile_path = tmp_dir / "Dockerfile"
Expand Down Expand Up @@ -272,6 +279,7 @@ class DefaultImageBuilder(ImageSpecBuilder):
"pip_index",
# "registry_config",
"commands",
"docker_commands",
}

def build_image(self, image_spec: ImageSpec) -> str:
Expand Down
17 changes: 17 additions & 0 deletions flytekit/image_spec/image_spec.py
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ class ImageSpec:
commands: Command to run during the building process
tag_format: Custom string format for image tag. The ImageSpec hash passed in as `spec_hash`. For example,
to add a "dev" suffix to the image tag, set `tag_format="{spec_hash}-dev"`
docker_commands: List of docker commands to run during the building process
"""

name: str = "flytekit"
Expand All @@ -72,6 +73,7 @@ class ImageSpec:
entrypoint: Optional[List[str]] = None
commands: Optional[List[str]] = None
tag_format: Optional[str] = None
docker_commands: Optional[List[str]] = None
pingsutw marked this conversation as resolved.
Show resolved Hide resolved

def __post_init__(self):
self.name = self.name.lower()
Expand Down Expand Up @@ -227,6 +229,21 @@ def with_apt_packages(self, apt_packages: Union[str, List[str]]) -> "ImageSpec":

return new_image_spec

def with_docker_commands(self, docker_commands: Union[str, List[str]]) -> "ImageSpec":
"""
Builder that returns a new image spec with additional list of docker commands that will be executed during the building process.
"""
new_image_spec = copy.deepcopy(self)
if new_image_spec.docker_commands is None:
new_image_spec.docker_commands = []

if isinstance(docker_commands, List):
new_image_spec.docker_commands.extend(docker_commands)
else:
new_image_spec.docker_commands.append(docker_commands)

return new_image_spec

def force_push(self) -> "ImageSpec":
"""
Builder that returns a new image spec with force push enabled.
Expand Down
Loading