From 82141b08cfbf49fa81cb19e3be3ab5f962430e02 Mon Sep 17 00:00:00 2001 From: Alexey Volkov Date: Sun, 22 Mar 2020 02:54:44 -0700 Subject: [PATCH] SDK - Support kubernetes client v11 (#3319) Fixes https://github.com/kubeflow/pipelines/issues/3275 --- sdk/python/kfp/compiler/_k8s_helper.py | 4 +--- sdk/python/kfp/compiler/_op_to_template.py | 12 ++-------- sdk/python/kfp/dsl/_container_op.py | 28 ++++++++++++++-------- sdk/python/kfp/dsl/_pipeline_param.py | 14 +++-------- sdk/python/kfp/dsl/_resource_op.py | 14 +++++------ sdk/python/setup.py | 2 +- 6 files changed, 32 insertions(+), 42 deletions(-) diff --git a/sdk/python/kfp/compiler/_k8s_helper.py b/sdk/python/kfp/compiler/_k8s_helper.py index 8992047d8e49..1787eec6decf 100644 --- a/sdk/python/kfp/compiler/_k8s_helper.py +++ b/sdk/python/kfp/compiler/_k8s_helper.py @@ -80,10 +80,8 @@ def convert_k8s_obj_to_json(k8s_obj): # and attributes which value is not None. # Convert attribute name to json key in # model definition for request. - attr_types = (k8s_obj.swagger_types if hasattr(k8s_obj, "swagger_types") - else k8s_obj.openapi_types) obj_dict = {k8s_obj.attribute_map[attr]: getattr(k8s_obj, attr) - for attr, _ in iteritems(attr_types) + for attr in k8s_obj.attribute_map if getattr(k8s_obj, attr) is not None} return {key: convert_k8s_obj_to_json(val) diff --git a/sdk/python/kfp/compiler/_op_to_template.py b/sdk/python/kfp/compiler/_op_to_template.py index 01af772b4cc2..2b95f75269e5 100644 --- a/sdk/python/kfp/compiler/_op_to_template.py +++ b/sdk/python/kfp/compiler/_op_to_template.py @@ -69,17 +69,9 @@ def _process_obj(obj: Any, map_to_tmpl_var: dict): return map_to_tmpl_var.get(str(obj), '{{inputs.parameters.%s}}' % obj.full_name) # k8s objects (generated from swaggercodegen) - if hasattr(obj, 'swagger_types') and isinstance(obj.swagger_types, dict): + if hasattr(obj, 'attribute_map') and isinstance(obj.attribute_map, dict): # process everything inside recursively - for key in obj.swagger_types.keys(): - setattr(obj, key, _process_obj(getattr(obj, key), map_to_tmpl_var)) - # return json representation of the k8s obj - return convert_k8s_obj_to_json(obj) - - # k8s objects (generated from openapi) - if hasattr(obj, 'openapi_types') and isinstance(obj.openapi_types, dict): - # process everything inside recursively - for key in obj.openapi_types.keys(): + for key in obj.attribute_map.keys(): setattr(obj, key, _process_obj(getattr(obj, key), map_to_tmpl_var)) # return json representation of the k8s obj return convert_k8s_obj_to_json(obj) diff --git a/sdk/python/kfp/dsl/_container_op.py b/sdk/python/kfp/dsl/_container_op.py index 816341142d1c..e39aed7c575a 100644 --- a/sdk/python/kfp/dsl/_container_op.py +++ b/sdk/python/kfp/dsl/_container_op.py @@ -130,16 +130,21 @@ class Container(V1Container): """ """ Attributes: - swagger_types (dict): The key is attribute name - and the value is attribute type. attribute_map (dict): The key is attribute name and the value is json key in definition. """ - # remove `name` from swagger_types so `name` is not generated in the JSON - swagger_types = { - key: value - for key, value in V1Container.swagger_types.items() if key != 'name' - } + # remove `name` from attribute_map, swagger_types and openapi_types so `name` is not generated in the JSON + + if hasattr(V1Container, 'swagger_types'): + swagger_types = { + key: value + for key, value in V1Container.swagger_types.items() if key != 'name' + } + if hasattr(V1Container, 'openapi_types'): + openapi_types = { + key: value + for key, value in V1Container.openapi_types.items() if key != 'name' + } attribute_map = { key: value for key, value in V1Container.attribute_map.items() if key != 'name' @@ -572,9 +577,12 @@ class UserContainer(Container): # adds `mirror_volume_mounts` to `UserContainer` swagger definition # NOTE inherits definition from `V1Container` rather than `Container` # because `Container` has no `name` property. - swagger_types = dict( - **V1Container.swagger_types, mirror_volume_mounts='bool') - + if hasattr(V1Container, 'swagger_types'): + swagger_types = dict( + **V1Container.swagger_types, mirror_volume_mounts='bool') + if hasattr(V1Container, 'openapi_types'): + openapi_types = dict( + **V1Container.openapi_types, mirror_volume_mounts='bool') attribute_map = dict( **V1Container.attribute_map, mirror_volume_mounts='mirrorVolumeMounts') diff --git a/sdk/python/kfp/dsl/_pipeline_param.py b/sdk/python/kfp/dsl/_pipeline_param.py index 0103fad47fd4..9a06f6b5b0ba 100644 --- a/sdk/python/kfp/dsl/_pipeline_param.py +++ b/sdk/python/kfp/dsl/_pipeline_param.py @@ -116,18 +116,10 @@ def extract_pipelineparams_from_any(payload) -> List['PipelineParam']: pipeline_params += extract_pipelineparams_from_any(item) return list(set(pipeline_params)) - # k8s swagger object - if hasattr(payload, 'swagger_types') and isinstance(payload.swagger_types, dict): + # k8s OpenAPI object + if hasattr(payload, 'attribute_map') and isinstance(payload.attribute_map, dict): pipeline_params = [] - for key in payload.swagger_types.keys(): - pipeline_params += extract_pipelineparams_from_any(getattr(payload, key)) - - return list(set(pipeline_params)) - - # k8s openapi object - if hasattr(payload, 'openapi_types') and isinstance(payload.openapi_types, dict): - pipeline_params = [] - for key in payload.openapi_types.keys(): + for key in payload.attribute_map: pipeline_params += extract_pipelineparams_from_any(getattr(payload, key)) return list(set(pipeline_params)) diff --git a/sdk/python/kfp/dsl/_resource_op.py b/sdk/python/kfp/dsl/_resource_op.py index 70782c093713..0ba3d75ae205 100644 --- a/sdk/python/kfp/dsl/_resource_op.py +++ b/sdk/python/kfp/dsl/_resource_op.py @@ -26,13 +26,6 @@ class Resource(object): which is used to represent the `resource` property in argo's workflow template (io.argoproj.workflow.v1alpha1.Template). """ - """ - Attributes: - swagger_types (dict): The key is attribute name - and the value is attribute type. - attribute_map (dict): The key is attribute name - and the value is json key in definition. - """ swagger_types = { "action": "str", "merge_strategy": "str", @@ -40,6 +33,13 @@ class Resource(object): "failure_condition": "str", "manifest": "str" } + openapi_types = { + "action": "str", + "merge_strategy": "str", + "success_condition": "str", + "failure_condition": "str", + "manifest": "str" + } attribute_map = { "action": "action", "merge_strategy": "mergeStrategy", diff --git a/sdk/python/setup.py b/sdk/python/setup.py index 74544980408a..9634e577b661 100644 --- a/sdk/python/setup.py +++ b/sdk/python/setup.py @@ -26,7 +26,7 @@ 'python-dateutil', 'PyYAML', 'google-cloud-storage>=1.13.0', - 'kubernetes>=8.0.0, <=10.0.0', + 'kubernetes>=8.0.0, <12.0.0', 'PyJWT>=1.6.4', 'cryptography>=2.4.2', 'google-auth>=1.6.1',