fs: simplify auth

Compared to how we were using it in dvc before:

1) doesn't require/accept `tmp_dir` at all, caches creds in
   `appdirs("pydrive2fs", "iterative")`.
2) caching is per client_id, so no risk of using mismatched client_id and
   cached creds.
3) doesn't accept `user_credentials_file` (was used as a manual caching
   destination), because no one really wants to use it anyway. This eliminates
   the risk of users using mismatched client_id and cached creds.
4) renamed `service_account_json_file_path` into `client_json_file_path` for consistency.
5) doesn't accept `use_service_account`, as this is easilly derived from
   `client_json_file_path` being used.

Other than that, this is mostly old code.

Fixes #193

Author: efiop

pydrive2/fs/spec.py

@@ -1,3 +1,4 @@
+import appdirs
 import errno
 import io
 import logging
@@ -13,12 +14,17 @@
 
 from pydrive2.drive import GoogleDrive
 from pydrive2.fs.utils import IterStream
+from pydrive2.auth import GoogleAuth
 
 logger = logging.getLogger(__name__)
 
 FOLDER_MIME_TYPE = "application/vnd.google-apps.folder"
 
 
+class GDriveAuthError(Exception):
+    pass
+
+
 def _gdrive_retry(func):
     def should_retry(exc):
         from pydrive2.files import ApiRequestError
@@ -49,13 +55,120 @@ def should_retry(exc):
     )(func)
 
 
+def _cache_file(client_id):
+    cache_dir = appdirs.user_cache_dir("pydrive2fs", "iterative")
+    os.makedirs(cache_dir, exist_ok=True)
+    return os.path.join(cache_dir, client_id + ".json")
+
+
 class GDriveFileSystem(AbstractFileSystem):
-    def __init__(self, path, google_auth, trash_only=True, **kwargs):
+    GDRIVE_CREDENTIALS_DATA = "GDRIVE_CREDENTIALS_DATA"
+
+    def __init__(
+        self,
+        path,
+        google_auth=None,
+        trash_only=True,
+        client_id=None,
+        client_secret=None,
+        client_user_email=None,
+        client_json_file_path=None,
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
         self.path = path
         self.root, self.base = self.split_path(self.path)
+
+        if not google_auth:
+            google_auth = self._auth(
+                client_id=client_id,
+                client_secret=client_secret,
+                client_user_email=client_user_email,
+                client_json_file_path=client_json_file_path,
+            )
+
         self.client = GoogleDrive(google_auth)
         self._trash_only = trash_only
-        super().__init__(**kwargs)
+
+    def _auth(
+        self,
+        client_id=None,
+        client_secret=None,
+        client_user_email=None,
+        client_json_file_path=None,
+    ):
+        settings = {
+            "get_refresh_token": True,
+            "oauth_scope": [
+                "https://www.googleapis.com/auth/drive",
+                "https://www.googleapis.com/auth/drive.appdata",
+            ],
+        }
+
+        env_creds = os.getenv(self.GDRIVE_CREDENTIALS_DATA)
+
+        if (
+            not env_creds
+            and not client_json_file_path
+            and not (client_id and client_secret)
+        ):
+            raise ValueError(
+                "Specify credentials using one of these methods: "
+                "client_id/client_secret or "
+                "client_json_file_path or "
+                f"{self.GDRIVE_CREDENTIALS_DATA}"
+            )
+
+        if env_creds:
+            settings["save_credentials"] = True
+            settings["save_credentials_backend"] = "dictionary"
+            settings["save_credentials_dict"] = {"creds": env_creds}
+            settings["save_credentials_key"] = "creds"
+        elif not client_json_file_path:
+            settings["save_credentials"] = True
+            settings["save_credentials_backend"] = "file"
+            settings["save_credentials_file"] = _cache_file(client_id)
+
+        if client_json_file_path:
+            config = {}
+
+            if client_user_email:
+                config["client_user_email"] = client_user_email
+
+            if env_creds:
+                config["client_json"] = env_creds
+            else:
+                config["client_json_file_path"] = client_json_file_path
+
+            settings["client_config_backend"] = "service"
+            settings["service_config"] = config
+        else:
+            settings["client_config_backend"] = "settings"
+            settings["client_config"] = {
+                "client_id": client_id,
+                "client_secret": client_secret,
+                "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+                "token_uri": "https://oauth2.googleapis.com/token",
+                "revoke_uri": "https://oauth2.googleapis.com/revoke",
+                "redirect_uri": "",
+            }
+
+        google_auth = GoogleAuth(settings=settings)
+
+        try:
+            logger.debug("GDrive auth with config '%s'.", settings)
+            if client_json_file_path:
+                google_auth.ServiceAuth()
+            else:
+                google_auth.LocalWebserverAuth()
+        except Exception as exc:
+            # Handle AuthenticationError, RefreshError and other auth failures
+            # It's hard to come up with a narrow exception, since PyDrive throws
+            # a lot of different errors - broken credentials file, refresh token
+            # expired, flow failed, etc.
+            raise GDriveAuthError("Failed to authenticate GDrive") from exc
+
+        return google_auth
 
     def split_path(self, path):
         parts = path.replace("//", "/").rstrip("/").split("/", 1)

pydrive2/test/test_fs.py

@@ -36,6 +36,24 @@ def fs(tmpdir, base_remote_dir):
     return fs
 
 
+@pytest.mark.manual
+def test_fs_oauth(base_remote_dir):
+    GDriveFileSystem(
+        base_remote_dir,
+        client_id="47794215776-cd9ssb6a4vv5otkq6n0iadpgc4efgjb1.apps.googleusercontent.com",  # noqa: E501
+        client_secret="i2gerGA7uBjZbR08HqSOSt9Z",
+    )
+
+
+def test_fs_service_from_settings(base_remote_dir):
+    creds = "credentials/fs.dat"
+    setup_credentials(creds)
+    GDriveFileSystem(
+        base_remote_dir,
+        client_json_file_path=creds,
+    )
+
+
 def test_info(fs, tmpdir, remote_dir):
     fs.touch(remote_dir + "/info/a.txt")
     fs.touch(remote_dir + "/info/b.txt")

setup.py

@@ -37,7 +37,12 @@
         "pyOpenSSL >= 19.1.0",
     ],
     extras_require={
-        "fsspec": ["fsspec >= 2021.07.0", "tqdm >= 4.0.0", "funcy >= 1.14"],
+        "fsspec": [
+            "fsspec >= 2021.07.0",
+            "tqdm >= 4.0.0",
+            "funcy >= 1.14",
+            "appdirs >= 1.4.3",
+        ],
         "tests": tests_requirements,
     },
     python_requires=">=3.7",