Skip to content

Commit ac75e1c

Browse files
authored
Switch to modular file system for s3 (#1312)
This PR is part of the effort to switch to modular file system for s3. When TF_ENABLE_LEGACY_FILESYSTEM=1 is provided, old behavior will be preserved. Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
1 parent a59fac2 commit ac75e1c

File tree

2 files changed

+15
-4
lines changed

2 files changed

+15
-4
lines changed

tensorflow_io/core/plugins/file_system_plugins.cc

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,12 +30,25 @@ void TF_InitPlugin(TF_FilesystemPluginInfo* info) {
3030
sizeof(info->ops[0])));
3131
tensorflow::io::az::ProvideFilesystemSupportFor(&info->ops[0], "az");
3232
tensorflow::io::http::ProvideFilesystemSupportFor(&info->ops[1], "http");
33-
tensorflow::io::s3::ProvideFilesystemSupportFor(&info->ops[2], "s3e");
3433
if (enable_legacy == "true" || enable_legacy == "1") {
34+
// TODO: enable on windows once tf-nightly releases windows build
35+
// that contains TF_ENABLE_LEGACY_FILESYSTEM.
36+
#if defined(_MSC_VER)
37+
tensorflow::io::s3::ProvideFilesystemSupportFor(&info->ops[2], "s3");
38+
#else
39+
tensorflow::io::s3::ProvideFilesystemSupportFor(&info->ops[2], "s3e");
40+
#endif
3541
tensorflow::io::hdfs::ProvideFilesystemSupportFor(&info->ops[3], "hdfse");
3642
tensorflow::io::hdfs::ProvideFilesystemSupportFor(&info->ops[4], "viewfse");
3743
tensorflow::io::hdfs::ProvideFilesystemSupportFor(&info->ops[5], "hare");
3844
} else {
45+
// TODO: enable on windows once tf-nightly releases windows build
46+
// that contains TF_ENABLE_LEGACY_FILESYSTEM.
47+
#if defined(_MSC_VER)
48+
tensorflow::io::s3::ProvideFilesystemSupportFor(&info->ops[2], "s3e");
49+
#else
50+
tensorflow::io::s3::ProvideFilesystemSupportFor(&info->ops[2], "s3");
51+
#endif
3952
tensorflow::io::hdfs::ProvideFilesystemSupportFor(&info->ops[3], "hdfs");
4053
tensorflow::io::hdfs::ProvideFilesystemSupportFor(&info->ops[4], "viewfs");
4154
tensorflow::io::hdfs::ProvideFilesystemSupportFor(&info->ops[5], "har");

tests/test_s3.py

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,5 @@ def test_read_file():
5555
os.environ["S3_USE_HTTPS"] = "0"
5656
os.environ["S3_VERIFY_SSL"] = "0"
5757

58-
# TODO: The following is not working yet, need update to use
59-
# s3 implementation with module file system
60-
content = tf.io.read_file("s3e://{}/{}".format(bucket_name, key_name))
58+
content = tf.io.read_file("s3://{}/{}".format(bucket_name, key_name))
6159
assert content == body

0 commit comments

Comments
 (0)