Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

SDK/Lightweight - Added python version compatibility checks #1524

Merged
merged 2 commits into from
Jun 23, 2019
Merged

SDK/Lightweight - Added python version compatibility checks #1524

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
660419a
SDK - Refactored the code in kfp.components._python_op._capture_funct…
Ark-kun Jun 14, 2019
eb6b9b5
SDK/Lightweight - Added python version compatibility checks
Ark-kun Jun 14, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
69 changes: 41 additions & 28 deletions sdk/python/kfp/components/_python_op.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,37 +64,50 @@ def _capture_function_code_using_cloudpickle(func, modules_to_capture: List[str]
func_pickle = base64.b64encode(cloudpickle.dumps(func, pickle.DEFAULT_PROTOCOL))
finally:
sys.modules.update(old_modules)
func_code = '{func_name} = pickle.loads(base64.b64decode({func_pickle}))'.format(

function_loading_code = '''\
import sys
try:
import cloudpickle as _cloudpickle
except ImportError:
import subprocess
try:
print("cloudpickle is not installed. Installing it globally", file=sys.stderr)
subprocess.run([sys.executable, "-m", "pip", "install", "cloudpickle==1.1.1", "--quiet"], env={"PIP_DISABLE_PIP_VERSION_CHECK": "1"}, check=True)
Ark-kun marked this conversation as resolved.
Show resolved Hide resolved
print("Installed cloudpickle globally", file=sys.stderr)
except:
print("Failed to install cloudpickle globally. Installing for the current user.", file=sys.stderr)
subprocess.run([sys.executable, "-m", "pip", "install", "cloudpickle==1.1.1", "--user", "--quiet"], env={"PIP_DISABLE_PIP_VERSION_CHECK": "1"}, check=True)
print("Installed cloudpickle for the current user", file=sys.stderr)
# Enable loading from user-installed package directory. Python does not add it to sys.path if it was empty at start. Running pip does not refresh `sys.path`.
import site
sys.path.append(site.getusersitepackages())
import cloudpickle as _cloudpickle
print("cloudpickle loaded successfully after installing.", file=sys.stderr)
''' + '''
pickler_python_version = {pickler_python_version}
current_python_version = tuple(sys.version_info)
if (
current_python_version[0] != pickler_python_version[0] or
current_python_version[1] < pickler_python_version[1] or
current_python_version[0] == 3 and ((pickler_python_version[1] < 6) != (current_python_version[1] < 6))
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

suggesting to use '()' to group the conditions combined by 'and'.
Because the 'or' operator has higher precedence than 'and', the codes are fine.
The suggestion is simply for readability.

):
raise RuntimeError("Incompatible python versions: " + str(current_python_version) + " instead of " + str(pickler_python_version))

if current_python_version != pickler_python_version:
print("Warning!: Different python versions. The code may crash! Current environment python version: " + str(current_python_version) + ". Component code python version: " + str(pickler_python_version), file=sys.stderr)

import base64
import pickle

{func_name} = pickle.loads(base64.b64decode({func_pickle}))
'''.format(
func_name=func.__name__,
func_pickle=repr(func_pickle)
func_pickle=repr(func_pickle),
pickler_python_version=repr(tuple(sys.version_info)),
)

code_lines = [
'try:',
' import cloudpickle as _cloudpickle',
'except ImportError:',
' import subprocess',
' import sys',
' try:',
' print("cloudpickle is not installed. Installing it globally", file=sys.stderr)',
' subprocess.run([sys.executable, "-m", "pip", "install", "cloudpickle==1.1.1", "--quiet"], env={"PIP_DISABLE_PIP_VERSION_CHECK": "1"}, check=True)',
' print("Installed cloudpickle globally", file=sys.stderr)',
' except:',
' print("Failed to install cloudpickle globally. Installing for the current user.", file=sys.stderr)',
' subprocess.run([sys.executable, "-m", "pip", "install", "cloudpickle==1.1.1", "--user", "--quiet"], env={"PIP_DISABLE_PIP_VERSION_CHECK": "1"}, check=True)',
' print("Installed cloudpickle for the current user", file=sys.stderr)',
' import site',
' sys.path.append(site.getusersitepackages())', # Enable loading from user-installed package directory. Python does not add it to sys.path if it was empty at start. Running pip does not refresh `sys.path`.
' import cloudpickle as _cloudpickle',
' print("cloudpickle loaded successfully after installing.", file=sys.stderr)',
'',
'import base64',
'import pickle',
'',
func_code,
]

return '\n'.join(code_lines)
return function_loading_code


def _extract_component_interface(func) -> ComponentSpec:
Expand Down