add initial setup for gcs

Author: vnghia

tensorflow_io/core/plugins/gs/gcs_filesystem.cc

@@ -1249,6 +1249,11 @@ void ProvideFilesystemSupportFor(TF_FilesystemPluginOps* ops, const char* uri) {
   ops->writable_file_ops = static_cast<TF_WritableFileOps*>(
       plugin_memory_allocate(TF_WRITABLE_FILE_OPS_SIZE));
   ops->writable_file_ops->cleanup = tf_writable_file::Cleanup;
+  ops->writable_file_ops->append = tf_writable_file::Append;
+  ops->writable_file_ops->tell = tf_writable_file::Tell;
+  ops->writable_file_ops->flush = tf_writable_file::Flush;
+  ops->writable_file_ops->sync = tf_writable_file::Sync;
+  ops->writable_file_ops->close = tf_writable_file::Close;
 
   ops->read_only_memory_region_ops = static_cast<TF_ReadOnlyMemoryRegionOps*>(
       plugin_memory_allocate(TF_READ_ONLY_MEMORY_REGION_OPS_SIZE));

tests/test_fs_plugins.py

@@ -61,8 +61,6 @@ def get_uri(key_name):
 
     def read_file(key_name):
         response = client.get_object(Bucket=bucket_name, Key=key_name)
-        print(bucket_name)
-        print(key_name)
         return response["Body"].read()
 
     def write_file(key_name, body):
@@ -71,7 +69,39 @@ def write_file(key_name, body):
     return get_uri, read_file, write_file
 
 
-MAP_URI_HELPER = {S3_URI: s3_init}
+def gcs_init(monkeypatch):
+    from google.cloud import storage
+
+    bucket_name = os.environ.get("GCS_TEST_BUCKET")
+    client = None
+    bucket = None
+    if bucket_name is None:
+        # This means we are running against emulator.
+        monkeypatch.setenv("STORAGE_EMULATOR_HOST", "http://localhost:9099")
+        monkeypatch.setenv("CLOUD_STORAGE_EMULATOR_ENDPOINT", "http://localhost:9099")
+        bucket_name = "tf-io-{}-{}".format(GCS_URI, int(time.time()))
+        client = storage.Client.create_anonymous_client()
+        client.project = "test_project"
+        bucket = client.create_bucket(bucket_name)
+    else:
+        client = storage.Client()
+        bucket = client.get_bucket(bucket)
+
+    def get_uri(key_name):
+        return "{}://{}/{}".format(GCS_URI, bucket_name, key_name)
+
+    def read_file(key_name):
+        blob = bucket.get_blob(key_name)
+        return blob.download_as_bytes()
+
+    def write_file(key_name, body):
+        blob = bucket.blob(key_name)
+        blob.upload_from_string(body)
+
+    return get_uri, read_file, write_file
+
+
+MAP_URI_HELPER = {S3_URI: s3_init, GCS_URI: gcs_init}
 
 
 # ------------------------ URI CONDITION FOR EACH TEST ----------------------- #
@@ -138,13 +168,15 @@ def check_test_condition_and_setup_env(uri, envs, monkeypatch):
         pytest.param(HDFS_URI, marks=pytest.mark.skip(reason="TODO")),
         pytest.param(VIEWFS_URI, marks=pytest.mark.skip(reason="TODO")),
         pytest.param(HAR_URI, marks=pytest.mark.skip(reason="TODO")),
-        pytest.param(GCS_URI, marks=pytest.mark.skip(reason="TODO")),
+        GCS_URI,
     ],
 )
 def uri_init(request):
     uri = request.param
     monkeypatch = pytest.MonkeyPatch()
-    yield uri, *MAP_URI_HELPER[uri](monkeypatch)
+
+    get_uri, read_file, write_file = MAP_URI_HELPER[uri](monkeypatch)
+    yield uri, get_uri, read_file, write_file
     monkeypatch.undo()
 
 
@@ -186,7 +218,7 @@ def test_io_write_file(uri_init, envs, monkeypatch):
     assert read_file(base_file_name) == body
 
 
-@pytest.mark.parametrize("envs", [("+all", None)])
+@pytest.mark.parametrize("envs", [("+all:-gse", None)])
 def test_gfile_GFile_readable(uri_init, envs, monkeypatch):
     uri, get_uri, _, write_file = uri_init
     check_test_condition_and_setup_env(uri, envs, monkeypatch)
@@ -208,7 +240,7 @@ def test_gfile_GFile_readable(uri_init, envs, monkeypatch):
     with pytest.raises(tf.errors.NotFoundError) as excinfo:
         fname_not_found = fname + "_not_found"
         with tf.io.gfile.GFile(fname_not_found, "rb") as f:
-            pass
+            _ = f.read()
         assert fname_not_found in str(excinfo.value)
 
     # Read length
@@ -271,10 +303,3 @@ def test_gfile_GFile_writable(uri_init, envs, monkeypatch):
         f.write(base_body)
         f.flush()
         assert read_file(base_file_name) == body + base_body
-
-    # Notfound
-    with pytest.raises(tf.errors.NotFoundError) as excinfo:
-        fname_not_found = fname + "_not_found"
-        with tf.io.gfile.GFile(fname_not_found, "rb") as f:
-            pass
-        assert fname_not_found in str(excinfo.value)