Implement cluster spawning API

examples/cluster/experimental_cloud.py

@@ -0,0 +1,35 @@
+# Licensed to Modin Development Team under one or more contributor license agreements.
+# See the NOTICE file distributed with this work for additional information regarding
+# copyright ownership.  The Modin Development Team licenses this file to you under the
+# Apache License, Version 2.0 (the "License"); you may not use this file except in
+# compliance with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software distributed under
+# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific language
+# governing permissions and limitations under the License.
+
+"""
+This is a very basic sample script for running things remotely.
+It requires `aws_credentials` file to be present in current working directory.
+
+On credentials file format see https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html#cli-configure-files-where
+"""
+
+import logging
+
+import modin.pandas as pd
+from modin.experimental.cloud import cluster
+
+# set up verbose logging so Ray autoscaler would print a lot of things
+# and we'll see that stuff is alive and kicking
+logging.basicConfig(format="%(asctime)s %(message)s")
+logger = logging.getLogger()
+logger.setLevel(logging.DEBUG)
+
+example_cluster = cluster.create("aws", "aws_credentials")
+with example_cluster:
+    remote_df = pd.DataFrame([1, 2, 3, 4])
+    print(len(remote_df))  # len() is executed remotely

modin/experimental/cloud/__init__.py

@@ -0,0 +1,33 @@
+# Licensed to Modin Development Team under one or more contributor license agreements.
+# See the NOTICE file distributed with this work for additional information regarding
+# copyright ownership.  The Modin Development Team licenses this file to you under the
+# Apache License, Version 2.0 (the "License"); you may not use this file except in
+# compliance with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software distributed under
+# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific language
+# governing permissions and limitations under the License.
+
+from .base import ClusterError, CannotSpawnCluster, CannotDestroyCluster
+from .cluster import Provider, create as create_cluster
+from .connection import Connection
+
+
+def get_connection():
+    """
+    Returns an RPyC connection object to execute Python code remotely on the active cluster.
+    """
+    return Connection.get()
+
+
+__all__ = [
+    "ClusterError",
+    "CannotSpawnCluster",
+    "CannotDestroyCluster",
+    "Provider",
+    "create_cluster",
+    "get_connection",
+]

modin/experimental/cloud/base.py

@@ -0,0 +1,71 @@
+# Licensed to Modin Development Team under one or more contributor license agreements.
+# See the NOTICE file distributed with this work for additional information regarding
+# copyright ownership.  The Modin Development Team licenses this file to you under the
+# Apache License, Version 2.0 (the "License"); you may not use this file except in
+# compliance with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software distributed under
+# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific language
+# governing permissions and limitations under the License.
+
+from typing import NamedTuple
+import os
+import sys
+
+
+class ClusterError(Exception):
+    """
+    Generic cluster operating exception
+    """
+
+    def __init__(self, *args, cause: BaseException = None, traceback: str = None, **kw):
+        self.cause = cause
+        self.traceback = traceback
+        super().__init__(*args, **kw)
+
+
+class CannotSpawnCluster(ClusterError):
+    """
+    Raised when cluster cannot be spawned in the cloud
+    """
+
+
+class CannotDestroyCluster(ClusterError):
+    """
+    Raised when cluster cannot be destroyed in the cloud
+    """
+
+
+class ConnectionDetails(NamedTuple):
+    user_name: str = "modin"
+    key_file: str = None
+    address: str = None
+    port: int = 22
+
+
+_EXT = (".exe", ".com", ".cmd", ".bat", "") if sys.platform == "win32" else ("",)
+
+
+def _which(prog):
+    for entry in os.environ["PATH"].split(os.pathsep):
+        for ext in _EXT:
+            path = os.path.join(entry, prog + ext)
+            if os.access(path, os.X_OK):
+                return path
+    return None
+
+
+def _get_ssh_proxy_command():
+    socks_proxy = os.environ.get("MODIN_SOCKS_PROXY", None)
+    if socks_proxy is None:
+        return None
+    if _which("nc"):
+        return f"nc -x {socks_proxy} %h %p"
+    elif _which("connect"):
+        return f"connect -S {socks_proxy} %h %p"
+    raise ClusterError(
+        "SSH through proxy required but no supported proxying tools found"
+    )