Filter directory keys (kserve#380)

python/kfserving/kfserving/storage.py

@@ -71,10 +71,12 @@ def _download_s3(uri, temp_dir: str):
         for obj in objects:
             # Replace any prefix from the object key with temp_dir
             subdir_object_key = obj.object_name.replace(bucket_path, "", 1).strip("/")
-            if subdir_object_key == "":
-                subdir_object_key = obj.object_name.split("/")[-1]
-            client.fget_object(bucket_name, obj.object_name,
-                               os.path.join(temp_dir, subdir_object_key))
+            # fget_object handles directory creation if does not exist
+            if not obj.is_dir:
+                if subdir_object_key == "":
+                    subdir_object_key = obj.object_name
+                client.fget_object(bucket_name, obj.object_name,
+                                   os.path.join(temp_dir, subdir_object_key))
 
     @staticmethod
     def _download_gcs(uri, temp_dir: str):

python/kfserving/test/test_s3_storage.py

@@ -15,29 +15,35 @@
 import unittest.mock as mock
 import kfserving
 
+
 def create_mock_obj(path):
     mock_obj = mock.MagicMock()
     mock_obj.object_name = path
+    mock_obj.is_dir = False
     return mock_obj
 
+
 def create_mock_minio_client(mock_storage, paths):
     mock_minio_client = mock_storage.return_value
     mock_minio_client.list_objects.return_value = [create_mock_obj(p) for p in paths]
     return mock_minio_client
 
+
 def get_call_args(call_args_list):
     arg_list = []
     for call in call_args_list:
         args, _ = call
         arg_list.append(args)
     return arg_list
 
+
 def expected_call_args_list(bucket_name, parent_key, dest, paths):
     return [(bucket_name, f'{parent_key}/{p}'.strip('/'), f'{dest}/{p}'.strip('/'))
             for p in paths]
 
 # pylint: disable=protected-access
 
+
 @mock.patch('kfserving.storage.Minio')
 def test_parent_key(mock_storage):
 
@@ -56,6 +62,7 @@ def test_parent_key(mock_storage):
 
     mock_minio_client.list_objects.assert_called_with(bucket_name, prefix='bar', recursive=True)
 
+
 @mock.patch('kfserving.storage.Minio')
 def test_no_key(mock_storage):
 
@@ -73,6 +80,7 @@ def test_no_key(mock_storage):
 
     mock_minio_client.list_objects.assert_called_with(bucket_name, prefix='', recursive=True)
 
+
 @mock.patch('kfserving.storage.Minio')
 def test_full_name_key(mock_storage):
 
@@ -86,8 +94,8 @@ def test_full_name_key(mock_storage):
 
     # then
     arg_list = get_call_args(mock_minio_client.fget_object.call_args_list)
-    assert arg_list == expected_call_args_list(bucket_name, 'path/to/model', 'dest_path',
-                                               ['name.pt'])
+    assert arg_list == expected_call_args_list(bucket_name, '', 'dest_path',
+                                               [object_key])
 
     mock_minio_client.list_objects.assert_called_with(bucket_name, prefix=object_key,
                                                       recursive=True)