Implements RenameFile for folders.

Change: 133094766

tensorflow/core/platform/cloud/gcs_file_system.cc

@@ -26,6 +26,7 @@ limitations under the License.
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/gtl/map_util.h"
 #include "tensorflow/core/lib/gtl/stl_util.h"
+#include "tensorflow/core/lib/io/path.h"
 #include "tensorflow/core/lib/strings/numbers.h"
 #include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/platform/cloud/google_auth_provider.h"
@@ -851,8 +852,22 @@ Status GcsFileSystem::GetFileSize(const string& fname, uint64* file_size) {
   return Status::OK();
 }
 
-// Uses a GCS API command to copy the object and then deletes the old one.
 Status GcsFileSystem::RenameFile(const string& src, const string& target) {
+  if (!IsDirectory(src).ok()) {
+    return RenameObject(src, target);
+  }
+  // Rename all individual objects in the directory one by one.
+  std::vector<string> children;
+  TF_RETURN_IF_ERROR(GetChildren(src, &children));
+  for (const string& subpath : children) {
+    TF_RETURN_IF_ERROR(RenameObject(io::JoinPath(src, subpath),
+                                    io::JoinPath(target, subpath)));
+  }
+  return Status::OK();
+}
+
+// Uses a GCS API command to copy the object and then deletes the old one.
+Status GcsFileSystem::RenameObject(const string& src, const string& target) {
   string src_bucket, src_object, target_bucket, target_object;
   TF_RETURN_IF_ERROR(ParseGcsPath(src, &src_bucket, &src_object));
   TF_RETURN_IF_ERROR(ParseGcsPath(target, &target_bucket, &target_object));

tensorflow/core/platform/cloud/gcs_file_system.h

@@ -78,6 +78,7 @@ class GcsFileSystem : public FileSystem {
   /// Retrieves file statistics assuming fname points to a GCS object.
   Status StatForObject(const string& bucket, const string& object,
                        FileStatistics* stat);
+  Status RenameObject(const string& src, const string& target);
 
   std::unique_ptr<AuthProvider> auth_provider_;
   std::unique_ptr<HttpRequest::Factory> http_request_factory_;

tensorflow/core/platform/cloud/gcs_file_system_test.cc

@@ -731,14 +731,85 @@ TEST(GcsFileSystemTest, GetFileSize) {
   EXPECT_EQ(1010, size);
 }
 
-TEST(GcsFileSystemTest, RenameFile) {
+TEST(GcsFileSystemTest, RenameFile_Folder) {
   std::vector<HttpRequest*> requests(
-      {new FakeHttpRequest(
+      {// Check if this is a folder or an object.
+       new FakeHttpRequest(
+           "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
+           "fields=items%2Fname%2CnextPageToken&prefix=path1%2F"
+           "&maxResults=1\n"
+           "Auth Token: fake_token\n",
+           "{\"items\": [ "
+           "  { \"name\": \"path1/subfolder/file1.txt\" }]}"),
+       // Requesting the full list of files in the folder.
+       new FakeHttpRequest(
+           "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
+           "fields=items%2Fname%2CnextPageToken&prefix=path1%2F\n"
+           "Auth Token: fake_token\n",
+           "{\"items\": [ "
+           "  { \"name\": \"path1/subfolder/file1.txt\" },"
+           "  { \"name\": \"path1/file2.txt\" }]}"),
+       // Copying the first file.
+       new FakeHttpRequest(
+           "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
+           "path1%2Fsubfolder%2Ffile1.txt/rewriteTo/b/bucket/o/"
+           "path2%2Fsubfolder%2Ffile1.txt\n"
+           "Auth Token: fake_token\n"
+           "Post: yes\n",
+           ""),
+       // Deleting the first original file.
+       new FakeHttpRequest(
+           "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
+           "path1%2Fsubfolder%2Ffile1.txt\n"
+           "Auth Token: fake_token\n"
+           "Delete: yes\n",
+           ""),
+       // Copying the second file.
+       new FakeHttpRequest(
+           "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
+           "path1%2Ffile2.txt/rewriteTo/b/bucket/o/path2%2Ffile2.txt\n"
+           "Auth Token: fake_token\n"
+           "Post: yes\n",
+           ""),
+       // Deleting the second original file.
+       new FakeHttpRequest(
+           "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
+           "path1%2Ffile2.txt\n"
+           "Auth Token: fake_token\n"
+           "Delete: yes\n",
+           "")});
+  GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
+                   std::unique_ptr<HttpRequest::Factory>(
+                       new FakeHttpRequestFactory(&requests)),
+                   0 /* read ahead bytes */, 5 /* max upload attempts */);
+
+  TF_EXPECT_OK(fs.RenameFile("gs://bucket/path1", "gs://bucket/path2/"));
+}
+
+TEST(GcsFileSystemTest, RenameFile_Object) {
+  std::vector<HttpRequest*> requests(
+      {// IsDirectory is checking whether there are children objects.
+       new FakeHttpRequest(
+           "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
+           "fields=items%2Fname%2CnextPageToken&prefix=path%2Fsrc.txt%2F"
+           "&maxResults=1\n"
+           "Auth Token: fake_token\n",
+           "{}"),
+       // IsDirectory is checking if the path exists as an object.
+       new FakeHttpRequest(
+           "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
+           "path%2Fsrc.txt?fields=size%2Cupdated\n"
+           "Auth Token: fake_token\n",
+           strings::StrCat("{\"size\": \"1010\","
+                           "\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
+       // Copying to the new location.
+       new FakeHttpRequest(
            "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
            "path%2Fsrc.txt/rewriteTo/b/bucket/o/path%2Fdst.txt\n"
            "Auth Token: fake_token\n"
            "Post: yes\n",
            ""),
+       // Deleting the original file.
        new FakeHttpRequest(
            "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
            "path%2Fsrc.txt\n"