[Flyte Decks] support ydata-profiling in python 3.12 (#2766)

* [Flyte Decks] support ydata-profiling in python 3.12 Signed-off-by: Future-Outlier <eric901201@gmail.com> * remove exclude deck standard python3.12 ci Signed-off-by: Future-Outlier <eric901201@gmail.com> * make plugin soft dependencies Signed-off-by: Future-Outlier <eric901201@gmail.com> * add dev-requirements.in Signed-off-by: Future-Outlier <eric901201@gmail.com> * nit Signed-off-by: Future-Outlier <eric901201@gmail.com> * better README with dependenc Signed-off-by: Future-Outlier <eric901201@gmail.com> * add other dependency in dev-requirements.in, this will help setup-global-uv Signed-off-by: Future-Outlier <eric901201@gmail.com> * Trigger CI Signed-off-by: Future-Outlier <eric901201@gmail.com> * Trigger CI Signed-off-by: Future-Outlier <eric901201@gmail.com> * Update dependenct Signed-off-by: Future-Outlier <eric901201@gmail.com> * new dockerfile dev Signed-off-by: Future-Outlier <eric901201@gmail.com> * new dockerfile Signed-off-by: Future-Outlier <eric901201@gmail.com> * new dockerfile Signed-off-by: Future-Outlier <eric901201@gmail.com> * revert back Signed-off-by: Future-Outlier <eric901201@gmail.com> * new dev image Signed-off-by: Future-Outlier <eric901201@gmail.com> --------- Signed-off-by: Future-Outlier <eric901201@gmail.com>
flyteorg · Sep 25, 2024 · b6f30e6 · b6f30e6
1 parent 9a9f5b2
commit b6f30e6
Show file tree

Hide file tree

Showing 9 changed files with 191 additions and 31 deletions.
diff --git a/.github/workflows/pythonbuild.yml b/.github/workflows/pythonbuild.yml
@@ -396,9 +396,6 @@ jobs:
           # apache-beam, one of flytekit-airflow dependencies, does not support python 3.12: https://github.com/apache/beam/issues/29149
           - python-version: 3.12
             plugin-names: "flytekit-airflow"
-          # ydata-profiling, a dependency of flytekit-deck-standard, does not support python 3.12: https://github.com/ydataai/ydata-profiling/issues/1510
-          - python-version: 3.12
-            plugin-names: "flytekit-deck-standard"
           # Tensorflow is a dependency of flytekit-mlflow tests and that is not supported yet: https://github.com/tensorflow/tensorflow/issues/62003
           - python-version: 3.12
             plugin-names: "flytekit-mlflow"

diff --git a/Dockerfile.dev b/Dockerfile.dev
@@ -40,7 +40,13 @@ RUN SETUPTOOLS_SCM_PRETEND_VERSION_FOR_FLYTEKIT=$PSEUDO_VERSION \
             -e /flytekit \
             -e /flytekit/plugins/flytekit-deck-standard \
             -e /flytekit/plugins/flytekit-flyteinteractive \
+            markdown \
+            pandas \
+            pillow \
+            plotly \
+            pygments \
             scikit-learn \
+            ydata-profiling \
     && apt-get clean autoclean \
     && apt-get autoremove --yes \
     && rm -rf /var/lib/{apt,dpkg,cache,log}/ \

diff --git a/dev-requirements.in b/dev-requirements.in
@@ -48,11 +48,15 @@ types-decorator
 types-mock
 autoflake
 
+markdown
 pillow
 numpy
 pandas
+plotly
 pyarrow
+pygments
 scikit-learn
+ydata-profiling
 types-requests
 prometheus-client
 

diff --git a/dev-requirements.txt b/dev-requirements.txt
@@ -20,6 +20,8 @@ aioitertools==0.11.0
     # via aiobotocore
 aiosignal==1.3.1
     # via aiohttp
+annotated-types==0.7.0
+    # via pydantic
 asn1crypto==1.5.1
     # via snowflake-connector-python
 asttokens==2.4.1
@@ -29,6 +31,7 @@ attrs==23.2.0
     #   aiohttp
     #   hypothesis
     #   jsonlines
+    #   visions
 autoflake==2.3.1
     # via -r dev-requirements.in
 azure-core==1.30.1
@@ -70,6 +73,8 @@ cloudpickle==3.0.0
     # via flytekit
 codespell==2.3.0
     # via -r dev-requirements.in
+contourpy==1.3.0
+    # via matplotlib
 coverage[toml]==7.5.3
     # via
     #   -r dev-requirements.in
@@ -83,8 +88,11 @@ cryptography==43.0.1
     #   msal
     #   pyjwt
     #   pyopenssl
-    #   secretstorage
     #   snowflake-connector-python
+cycler==0.12.1
+    # via matplotlib
+dacite==1.8.1
+    # via ydata-profiling
 dataclasses-json==0.5.9
     # via flytekit
 decorator==5.1.1
@@ -111,6 +119,8 @@ flyteidl @ git+https://github.com/flyteorg/flyte.git@master#subdirectory=flyteid
     # via
     #   -r dev-requirements.in
     #   flytekit
+fonttools==4.54.1
+    # via matplotlib
 frozenlist==1.4.1
     # via
     #   aiohttp
@@ -175,6 +185,8 @@ grpcio-status==1.62.2
     # via
     #   flytekit
     #   google-api-core
+htmlmin==0.1.12
+    # via ydata-profiling
 hypothesis==6.103.0
     # via -r dev-requirements.in
 icdiff==2.0.7
@@ -186,6 +198,10 @@ idna==3.7
     #   requests
     #   snowflake-connector-python
     #   yarl
+imagehash==4.3.1
+    # via
+    #   visions
+    #   ydata-profiling
 importlib-metadata==7.1.0
     # via flytekit
 iniconfig==2.0.0
@@ -206,16 +222,15 @@ jaraco-functools==4.0.1
     # via keyring
 jedi==0.19.1
     # via ipython
-jeepney==0.8.0
-    # via
-    #   keyring
-    #   secretstorage
+jinja2==3.1.4
+    # via ydata-profiling
 jmespath==1.0.1
     # via botocore
 joblib==1.4.2
     # via
     #   -r dev-requirements.in
     #   flytekit
+    #   phik
     #   scikit-learn
 jsonlines==4.0.0
     # via flytekit
@@ -225,12 +240,20 @@ keyring==25.2.1
     # via flytekit
 keyrings-alt==5.0.1
     # via -r dev-requirements.in
+kiwisolver==1.4.7
+    # via matplotlib
 kubernetes==29.0.0
     # via -r dev-requirements.in
+llvmlite==0.43.0
+    # via numba
+markdown==3.7
+    # via -r dev-requirements.in
 markdown-it-py==3.0.0
     # via
     #   flytekit
     #   rich
+markupsafe==2.1.5
+    # via jinja2
 marshmallow==3.21.2
     # via
     #   dataclasses-json
@@ -244,6 +267,12 @@ marshmallow-jsonschema==0.13.0
     # via flytekit
 mashumaro==3.13
     # via flytekit
+matplotlib==3.9.2
+    # via
+    #   phik
+    #   seaborn
+    #   wordcloud
+    #   ydata-profiling
 matplotlib-inline==0.1.7
     # via ipython
 mdurl==0.1.2
@@ -265,21 +294,41 @@ multidict==6.0.5
     # via
     #   aiohttp
     #   yarl
+multimethod==1.12
+    # via
+    #   visions
+    #   ydata-profiling
 mypy==1.6.1
     # via -r dev-requirements.in
 mypy-extensions==1.0.0
     # via
     #   mypy
     #   typing-inspect
+networkx==3.3
+    # via visions
 nodeenv==1.9.0
     # via pre-commit
+numba==0.60.0
+    # via ydata-profiling
 numpy==1.26.4
     # via
     #   -r dev-requirements.in
+    #   contourpy
+    #   imagehash
+    #   matplotlib
+    #   numba
     #   pandas
+    #   patsy
+    #   phik
     #   pyarrow
+    #   pywavelets
     #   scikit-learn
     #   scipy
+    #   seaborn
+    #   statsmodels
+    #   visions
+    #   wordcloud
+    #   ydata-profiling
 oauthlib==3.2.2
     # via
     #   kubernetes
@@ -291,22 +340,42 @@ packaging==24.0
     #   docker
     #   google-cloud-bigquery
     #   marshmallow
+    #   matplotlib
     #   msal-extensions
+    #   plotly
     #   pytest
     #   setuptools-scm
     #   snowflake-connector-python
+    #   statsmodels
 pandas==2.2.2
-    # via -r dev-requirements.in
+    # via
+    #   -r dev-requirements.in
+    #   phik
+    #   seaborn
+    #   statsmodels
+    #   visions
+    #   ydata-profiling
 parso==0.8.4
     # via jedi
+patsy==0.5.6
+    # via statsmodels
 pexpect==4.9.0
     # via ipython
+phik==0.12.4
+    # via ydata-profiling
 pillow==10.3.0
-    # via -r dev-requirements.in
+    # via
+    #   -r dev-requirements.in
+    #   imagehash
+    #   matplotlib
+    #   visions
+    #   wordcloud
 platformdirs==4.2.2
     # via
     #   snowflake-connector-python
     #   virtualenv
+plotly==5.24.1
+    # via -r dev-requirements.in
 pluggy==1.5.0
     # via pytest
 portalocker==2.8.2
@@ -350,10 +419,15 @@ pyasn1-modules==0.4.0
     # via google-auth
 pycparser==2.22
     # via cffi
+pydantic==2.9.2
+    # via ydata-profiling
+pydantic-core==2.23.4
+    # via pydantic
 pyflakes==3.2.0
     # via autoflake
 pygments==2.18.0
     # via
+    #   -r dev-requirements.in
     #   flytekit
     #   ipython
     #   rich
@@ -363,6 +437,8 @@ pyjwt[crypto]==2.8.0
     #   snowflake-connector-python
 pyopenssl==24.2.1
     # via snowflake-connector-python
+pyparsing==3.1.4
+    # via matplotlib
 pytest==8.2.1
     # via
     #   -r dev-requirements.in
@@ -390,6 +466,7 @@ python-dateutil==2.9.0.post0
     #   croniter
     #   google-cloud-bigquery
     #   kubernetes
+    #   matplotlib
     #   pandas
 python-json-logger==2.0.7
     # via flytekit
@@ -402,11 +479,14 @@ pytz==2024.1
     #   croniter
     #   pandas
     #   snowflake-connector-python
+pywavelets==1.7.0
+    # via imagehash
 pyyaml==6.0.1
     # via
     #   flytekit
     #   kubernetes
     #   pre-commit
+    #   ydata-profiling
 requests==2.32.3
     # via
     #   azure-core
@@ -421,6 +501,7 @@ requests==2.32.3
     #   msal
     #   requests-oauthlib
     #   snowflake-connector-python
+    #   ydata-profiling
 requests-oauthlib==2.0.0
     # via
     #   google-auth-oauthlib
@@ -438,9 +519,14 @@ s3fs==2024.5.0
 scikit-learn==1.5.0
     # via -r dev-requirements.in
 scipy==1.13.1
-    # via scikit-learn
-secretstorage==3.3.3
-    # via keyring
+    # via
+    #   imagehash
+    #   phik
+    #   scikit-learn
+    #   statsmodels
+    #   ydata-profiling
+seaborn==0.13.2
+    # via ydata-profiling
 setuptools-scm==8.1.0
     # via -r dev-requirements.in
 six==1.16.0
@@ -449,6 +535,7 @@ six==1.16.0
     #   azure-core
     #   isodate
     #   kubernetes
+    #   patsy
     #   python-dateutil
 snowflake-connector-python==3.12.1
     # via -r dev-requirements.in
@@ -460,14 +547,22 @@ stack-data==0.6.3
     # via ipython
 statsd==3.3.0
     # via flytekit
+statsmodels==0.14.3
+    # via ydata-profiling
+tenacity==9.0.0
+    # via plotly
 threadpoolctl==3.5.0
     # via scikit-learn
 tomlkit==0.13.2
     # via snowflake-connector-python
+tqdm==4.66.5
+    # via ydata-profiling
 traitlets==5.14.3
     # via
     #   ipython
     #   matplotlib-inline
+typeguard==4.3.0
+    # via ydata-profiling
 types-croniter==2.0.0.20240423
     # via -r dev-requirements.in
 types-decorator==5.1.8.20240310
@@ -485,8 +580,11 @@ typing-extensions==4.12.0
     #   flytekit
     #   mashumaro
     #   mypy
+    #   pydantic
+    #   pydantic-core
     #   rich-click
     #   snowflake-connector-python
+    #   typeguard
     #   typing-inspect
 typing-inspect==0.9.0
     # via dataclasses-json
@@ -502,16 +600,22 @@ urllib3==2.2.1
     #   types-requests
 virtualenv==20.26.2
     # via pre-commit
+visions[type-image-path]==0.7.6
+    # via ydata-profiling
 wcwidth==0.2.13
     # via prompt-toolkit
 websocket-client==1.8.0
     # via
     #   docker
     #   kubernetes
+wordcloud==1.9.3
+    # via ydata-profiling
 wrapt==1.16.0
     # via aiobotocore
 yarl==1.9.4
     # via aiohttp
+ydata-profiling==4.10.0
+    # via -r dev-requirements.in
 zipp==3.19.1
     # via importlib-metadata