Skip to content

Commit

Permalink
ci: Add Python 3.9 to build matrix (flyteorg#2622)
Browse files Browse the repository at this point in the history
Signed-off-by: Kevin Su <pingsutw@apache.org>
Signed-off-by: Eduardo Apolinario <eapolinario@users.noreply.github.com>
Signed-off-by: Future-Outlier <eric901201@gmail.com>
Co-authored-by: Eduardo Apolinario <eapolinario@users.noreply.github.com>
Co-authored-by: Future-Outlier <eric901201@gmail.com>
Signed-off-by: mao3267 <chenvincent610@gmail.com>
  • Loading branch information
3 people authored and mao3267 committed Aug 2, 2024
1 parent 20c8250 commit 2ef9b08
Show file tree
Hide file tree
Showing 20 changed files with 63 additions and 55 deletions.
10 changes: 9 additions & 1 deletion .github/workflows/pythonbuild.yml
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ jobs:
if [[ ${{ github.event_name }} == "schedule" ]]; then
echo "python_versions=[\"3.8\",\"3.9\",\"3.10\",\"3.11\",\"3.12\"]" >> $GITHUB_ENV
else
echo "python_versions=[\"3.12\"]" >> $GITHUB_ENV
echo "python_versions=[\"3.9\", \"3.12\"]" >> $GITHUB_ENV
fi
build:
Expand Down Expand Up @@ -128,6 +128,8 @@ jobs:
pandas: "pandas<2.0.0"
- numpy: "numpy<2.0.0"
pandas: "pandas>=2.0.0"
- numpy: "numpy>=2.0.0"
python-version: "3.8"

steps:
- uses: actions/checkout@v4
Expand Down Expand Up @@ -248,6 +250,8 @@ jobs:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- name: 'Clear action cache'
uses: ./.github/actions/clear-action-cache # sandbox has disk pressure, so we need to clear the cache to get more disk space.
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
Expand Down Expand Up @@ -354,6 +358,10 @@ jobs:
- flytekit-vaex
- flytekit-whylogs
exclude:
- python-version: 3.8
plugin-names: "flytekit-aws-sagemaker"
- python-version: 3.9
plugin-names: "flytekit-aws-sagemaker"
# flytekit-modin depends on ray which does not have a 3.11 wheel yet.
# Issue tracked in https://github.com/ray-project/ray/issues/27881
- python-version: 3.11
Expand Down
1 change: 1 addition & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -119,5 +119,6 @@ build-dev: export PLATFORM ?= linux/arm64
build-dev: export REGISTRY ?= localhost:30000
build-dev: export PYTHON_VERSION ?= 3.12
build-dev: export PSEUDO_VERSION ?= $(shell python -m setuptools_scm)
build-dev: export TAG ?= dev
build-dev:
docker build --platform ${PLATFORM} --push . -f Dockerfile.dev -t ${REGISTRY}/flytekit:${TAG} --build-arg PYTHON_VERSION=${PYTHON_VERSION} --build-arg PSEUDO_VERSION=${PSEUDO_VERSION}
2 changes: 1 addition & 1 deletion dev-requirements.in
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ setuptools_scm
pytest-icdiff

# Tensorflow is not available for python 3.12 yet: https://github.com/tensorflow/tensorflow/issues/62003
tensorflow; python_version<'3.12'
tensorflow<=2.15.1; python_version<'3.12'
# Newer versions of torch bring in nvidia dependencies that are not present in windows, so
# we put this constraint while we do not have per-environment requirements files
torch<=1.12.1; python_version<'3.11'
Expand Down
4 changes: 2 additions & 2 deletions flytekit/core/array_node_map_task.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,10 @@
import logging
import math
import os # TODO: use flytekit logger
import typing
from contextlib import contextmanager
from typing import Any, Dict, List, Optional, Set, Union, cast

import typing_extensions
from flyteidl.core import tasks_pb2

from flytekit.configuration import SerializationSettings
Expand Down Expand Up @@ -72,7 +72,7 @@ def __init__(
transformer = TypeEngine.get_transformer(v)
if isinstance(transformer, FlytePickleTransformer):
if is_annotated(v):
for annotation in typing.get_args(v)[1:]:
for annotation in typing_extensions.get_args(v)[1:]:
if isinstance(annotation, pickle.BatchSize):
raise ValueError("Choosing a BatchSize for map tasks inputs is not supported.")

Expand Down
1 change: 1 addition & 0 deletions plugins/flytekit-airflow/tests/test_agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,7 @@ async def test_airflow_agent():
"This is deprecated!",
True,
"A",
None
)

interfaces = interface_models.TypedInterface(inputs={}, outputs={})
Expand Down
4 changes: 2 additions & 2 deletions plugins/flytekit-kf-pytorch/flytekitplugins/kfpytorch/task.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
import flytekit
from flytekit import PythonFunctionTask, Resources, lazy_module
from flytekit.configuration import SerializationSettings
from flytekit.core.context_manager import FlyteContextManager, OutputMetadata
from flytekit.core.context_manager import OutputMetadata
from flytekit.core.pod_template import PodTemplate
from flytekit.core.resources import convert_resources_to_resource_model
from flytekit.exceptions.user import FlyteRecoverableException
Expand Down Expand Up @@ -465,7 +465,7 @@ def fn_partial():
# Rank 0 returns the result of the task function
if 0 in out:
# For rank 0, we transfer the decks created in the worker process to the parent process
ctx = FlyteContextManager.current_context()
ctx = flytekit.current_context()
for deck in out[0].decks:
if not isinstance(deck, flytekit.deck.deck.TimeLineDeck):
ctx.decks.append(deck)
Expand Down
2 changes: 1 addition & 1 deletion plugins/flytekit-mlflow/tests/test_mlflow_tracking.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,4 +29,4 @@ def train_model(epochs: int):

def test_local_exec():
train_model(epochs=1)
assert len(flytekit.current_context().decks) == 5 # mlflow metrics, params, timeline, input, and output
assert len(flytekit.current_context().decks) == 7 # mlflow metrics, params, timeline, input, and output, source code, dependencies
5 changes: 0 additions & 5 deletions plugins/flytekit-ray/flytekitplugins/ray/task.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,6 @@ class RayJobConfig:
address: typing.Optional[str] = None
shutdown_after_job_finishes: bool = False
ttl_seconds_after_finished: typing.Optional[int] = None
excludes_working_dir: typing.Optional[typing.List[str]] = None


class RayFunctionTask(PythonFunctionTask):
Expand Down Expand Up @@ -76,10 +75,6 @@ def pre_execute(self, user_params: ExecutionParameters) -> ExecutionParameters:
"excludes": ["script_mode.tar.gz", "fast*.tar.gz"],
}

cfg = self._task_config
if cfg.excludes_working_dir:
init_params["runtime_env"]["excludes"].extend(cfg.excludes_working_dir)

ray.init(**init_params)
return user_params

Expand Down
2 changes: 1 addition & 1 deletion plugins/flytekit-ray/tests/test_ray.py
Original file line number Diff line number Diff line change
Expand Up @@ -72,4 +72,4 @@ def t1(a: int) -> str:
]

assert t1(a=3) == "5"
assert not ray.is_initialized()
assert ray.is_initialized()
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import typing
from functools import partial

from flytekit import map_task, task, workflow
Expand All @@ -9,6 +10,6 @@ def fn(x: int, y: int) -> int:


@workflow
def workflow_with_maptask(data: list[int], y: int) -> list[int]:
def workflow_with_maptask(data: typing.List[int], y: int) -> typing.List[int]:
partial_fn = partial(fn, y=y)
return map_task(partial_fn)(x=data)
7 changes: 4 additions & 3 deletions tests/flytekit/unit/cli/pyflyte/test_run.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
)
from flytekit.interaction.click_types import DirParamType, FileParamType
from flytekit.remote import FlyteRemote
from typing import Iterator
from typing import Iterator, List
from flytekit.types.iterator import JSON
from flytekit import workflow

Expand Down Expand Up @@ -276,6 +276,7 @@ def test_union_type_with_invalid_input():
assert result.exit_code == 2


@pytest.mark.skipif(sys.version_info < (3, 9), reason="listing entities requires python>=3.9")
@pytest.mark.parametrize(
"workflow_file",
[
Expand Down Expand Up @@ -521,11 +522,11 @@ def tk(x: Iterator[JSON] = jsons()) -> Iterator[JSON]:
return t1(x=x)

@task
def t2(x: list[int]) -> list[int]:
def t2(x: List[int]) -> List[int]:
return x

@workflow
def tk_list(x: list[int] = [1, 2, 3]) -> list[int]:
def tk_list(x: List[int] = [1, 2, 3]) -> List[int]:
return t2(x=x)

@task
Expand Down
3 changes: 2 additions & 1 deletion tests/flytekit/unit/core/test_array_node_map_task.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
import typing
from collections import OrderedDict
from typing import List
from typing_extensions import Annotated

import pytest

Expand Down Expand Up @@ -78,7 +79,7 @@ def say_hello(name: str) -> str:

def test_map_task_with_pickle():
@task
def say_hello(name: typing.Annotated[typing.Any, BatchSize(10)]) -> str:
def say_hello(name: Annotated[typing.Any, BatchSize(10)]) -> str:
return f"hello {name}!"

with pytest.raises(ValueError, match="Choosing a BatchSize for map tasks inputs is not supported."):
Expand Down
3 changes: 2 additions & 1 deletion tests/flytekit/unit/core/test_dataclass.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,8 @@
import sys
import tempfile
from dataclasses import dataclass
from typing import Annotated, List, Dict, Optional
from typing import List, Dict, Optional
from typing_extensions import Annotated
from flytekit.types.schema import FlyteSchema
from flytekit.core.type_engine import TypeEngine
from flytekit.core.context_manager import FlyteContextManager
Expand Down
22 changes: 11 additions & 11 deletions tests/flytekit/unit/core/test_serialization.py
Original file line number Diff line number Diff line change
Expand Up @@ -525,7 +525,7 @@ def wf_with_input() -> int:
return t1(a=input_val)

@workflow
def wf_with_sub_wf() -> tuple[int, int]:
def wf_with_sub_wf() -> typing.Tuple[int, int]:
return (wf_no_input(), wf_with_input())

wf_no_input_spec = get_serializable(OrderedDict(), serialization_settings, wf_no_input)
Expand Down Expand Up @@ -564,7 +564,7 @@ def wf_with_input() -> str:
return t1(a=input_val)

@workflow
def wf_with_sub_wf() -> tuple[str, str]:
def wf_with_sub_wf() -> typing.Tuple[str, str]:
return (wf_no_input(), wf_with_input())

wf_no_input_spec = get_serializable(OrderedDict(), serialization_settings, wf_no_input)
Expand Down Expand Up @@ -603,7 +603,7 @@ def wf_with_input() -> typing.Optional[int]:
return t1(a=input_val)

@workflow
def wf_with_sub_wf() -> tuple[typing.Optional[int], typing.Optional[int]]:
def wf_with_sub_wf() -> typing.Tuple[typing.Optional[int], typing.Optional[int]]:
return (wf_no_input(), wf_with_input())

wf_no_input_spec = get_serializable(OrderedDict(), serialization_settings, wf_no_input)
Expand Down Expand Up @@ -673,7 +673,7 @@ def wf_with_input() -> typing.Optional[int]:
return t1(a=input_val)

@workflow
def wf_with_sub_wf() -> tuple[typing.Optional[int], typing.Optional[int]]:
def wf_with_sub_wf() -> typing.Tuple[typing.Optional[int], typing.Optional[int]]:
return (wf_no_input(), wf_with_input())

wf_no_input_spec = get_serializable(OrderedDict(), serialization_settings, wf_no_input)
Expand Down Expand Up @@ -764,15 +764,15 @@ def test_default_args_task_list_type():
input_val = [1, 2, 3]

@task
def t1(a: list[int] = []) -> list[int]:
def t1(a: typing.List[int] = []) -> typing.List[int]:
return a

@workflow
def wf_no_input() -> list[int]:
def wf_no_input() -> typing.List[int]:
return t1()

@workflow
def wf_with_input() -> list[int]:
def wf_with_input() -> typing.List[int]:
return t1(a=input_val)

with pytest.raises(FlyteAssertion, match="Cannot use non-hashable object as default argument"):
Expand All @@ -799,15 +799,15 @@ def test_default_args_task_dict_type():
input_val = {"a": 1, "b": 2}

@task
def t1(a: dict[str, int] = {}) -> dict[str, int]:
def t1(a: typing.Dict[str, int] = {}) -> typing.Dict[str, int]:
return a

@workflow
def wf_no_input() -> dict[str, int]:
def wf_no_input() -> typing.Dict[str, int]:
return t1()

@workflow
def wf_with_input() -> dict[str, int]:
def wf_with_input() -> typing.Dict[str, int]:
return t1(a=input_val)

with pytest.raises(FlyteAssertion, match="Cannot use non-hashable object as default argument"):
Expand Down Expand Up @@ -846,7 +846,7 @@ def wf_with_input() -> typing.Optional[typing.List[int]]:
return t1(a=input_val)

@workflow
def wf_with_sub_wf() -> tuple[typing.Optional[typing.List[int]], typing.Optional[typing.List[int]]]:
def wf_with_sub_wf() -> typing.Tuple[typing.Optional[typing.List[int]], typing.Optional[typing.List[int]]]:
return (wf_no_input(), wf_with_input())

wf_no_input_spec = get_serializable(OrderedDict(), serialization_settings, wf_no_input)
Expand Down
21 changes: 16 additions & 5 deletions tests/flytekit/unit/core/test_type_engine.py
Original file line number Diff line number Diff line change
Expand Up @@ -2389,17 +2389,28 @@ class Result(DataClassJsonMixin):
schema: TestSchema # type: ignore


@pytest.mark.parametrize(
"t",
[
def get_unsupported_complex_literals_tests():
if sys.version_info < (3, 9):
return [
typing_extensions.Annotated[typing.Dict[int, str], FlyteAnnotation({"foo": "bar"})],
typing_extensions.Annotated[typing.Dict[str, str], FlyteAnnotation({"foo": "bar"})],
typing_extensions.Annotated[Color, FlyteAnnotation({"foo": "bar"})],
typing_extensions.Annotated[Result, FlyteAnnotation({"foo": "bar"})],
]
return [
typing_extensions.Annotated[dict, FlyteAnnotation({"foo": "bar"})],
typing_extensions.Annotated[dict[int, str], FlyteAnnotation({"foo": "bar"})],
typing_extensions.Annotated[typing.Dict[int, str], FlyteAnnotation({"foo": "bar"})],
typing_extensions.Annotated[dict[str, str], FlyteAnnotation({"foo": "bar"})],
typing_extensions.Annotated[typing.Dict[str, str], FlyteAnnotation({"foo": "bar"})],
typing_extensions.Annotated[Color, FlyteAnnotation({"foo": "bar"})],
typing_extensions.Annotated[Result, FlyteAnnotation({"foo": "bar"})],
],
]


@pytest.mark.parametrize(
"t",
get_unsupported_complex_literals_tests(),
)
def test_unsupported_complex_literals(t):
with pytest.raises(ValueError):
Expand Down Expand Up @@ -3006,7 +3017,7 @@ def test_dataclass_encoder_and_decoder_registry():
class Datum:
x: int
y: str
z: dict[int, int]
z: typing.Dict[int, int]
w: List[int]

@task
Expand Down
1 change: 1 addition & 0 deletions tests/flytekit/unit/core/test_type_hints.py
Original file line number Diff line number Diff line change
Expand Up @@ -1526,6 +1526,7 @@ def t2() -> dict:
assert output_lm.literals["o0"].scalar.generic == expected_struct


@pytest.mark.skipif(sys.version_info < (3, 9), reason="Use of dict hints is only supported in Python 3.9+")
def test_guess_dict4():
@dataclass
class Foo(DataClassJsonMixin):
Expand Down
17 changes: 2 additions & 15 deletions tests/flytekit/unit/extend/test_agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
GetTaskRequest,
ListAgentsRequest,
ListAgentsResponse,
TaskCategory,
TaskCategory, DeleteTaskResponse,
)
from flyteidl.core.execution_pb2 import TaskExecution, TaskLog
from flyteidl.core.identifier_pb2 import ResourceType
Expand Down Expand Up @@ -223,7 +223,7 @@ async def test_async_agent_service(agent, consume_metadata):
res = await service.GetTask(GetTaskRequest(task_category=task_category, resource_meta=metadata_bytes), ctx)
assert res.resource.phase == TaskExecution.SUCCEEDED
res = await service.DeleteTask(DeleteTaskRequest(task_category=task_category, resource_meta=metadata_bytes), ctx)
assert res is None
assert res == DeleteTaskResponse()

agent_metadata = AgentRegistry.get_agent_metadata(agent.name)
assert agent_metadata.supported_task_types[0] == agent.task_category.name
Expand Down Expand Up @@ -399,16 +399,3 @@ def sample_agents():
name="ChatGPT Agent", is_sync=True, supported_task_categories=[TaskCategory(name="chatgpt", version=0)]
)
return [async_agent, sync_agent]


@patch("flytekit.clis.sdk_in_container.serve.click.secho")
@patch("flytekit.extend.backend.base_agent.AgentRegistry.list_agents")
def test_print_agents_metadata_output(list_agents_mock, mock_secho, sample_agents):
list_agents_mock.return_value = sample_agents
print_agents_metadata()
expected_calls = [
(("Starting Sensor that supports task categories ['sensor']",), {"fg": "blue"}),
(("Starting ChatGPT Agent that supports task categories ['chatgpt']",), {"fg": "blue"}),
]
mock_secho.assert_has_calls(expected_calls, any_order=True)
assert mock_secho.call_count == len(expected_calls)
2 changes: 1 addition & 1 deletion tests/flytekit/unit/extras/tasks/test_shell.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ def test_shell_task_access_to_result():
t()

assert t.result.returncode == 0
assert t.result.output == "Hello World!" # ShellTask strips carriage returns
assert "Hello World!" in t.result.output # ShellTask strips carriage returns
assert t.result.error == ""


Expand Down
4 changes: 2 additions & 2 deletions tests/flytekit/unit/interaction/test_click_types.py
Original file line number Diff line number Diff line change
Expand Up @@ -260,8 +260,8 @@ def test_dataclass_type():
class Datum:
x: int
y: str
z: dict[int, str]
w: list[int]
z: typing.Dict[int, str]
w: typing.List[int]

t = JsonParamType(Datum)
value = '{ "x": 1, "y": "2", "z": { "1": "one", "2": "two" }, "w": [1, 2, 3] }'
Expand Down
Loading

0 comments on commit 2ef9b08

Please sign in to comment.