Much more progress on Windows support in ducktape

README.md

@@ -165,9 +165,9 @@ functionality: normal services might provide a `bounce` method.
 
 Most of the code you'll write for a service will just be series of SSH commands
 and tests of output. You should request the number of nodes you'll need using
-the `num_nodes` parameter to the Service base class's constructor. Then, in your
-Service's methods you'll have access to `self.nodes` to access the nodes
-allocated to your service. Each node has an associated
+the `num_nodes` or `node_spec` parameter to the Service base class's constructor.
+Then, in your Service's methods you'll have access to `self.nodes` to access the
+nodes allocated to your service. Each node has an associated
 `ducktape.cluster.RemoteAccount` instance which lets you easily perform remote
 operations such as running commands via SSH or creating files. By default, these
 operations try to hide output (but provide it to you if you need to extract
@@ -197,6 +197,22 @@ Alternatively, if you've installed pytest (`sudo pip install pytest`) you can ru
 it directly on the `tests` directory`:
 
     py.test tests
+
+Windows
+-------
+Ducktape supports `Service`s that run on Windows, but only in EC2.
+
+When a `Service` requires a Windows machine, AWS credentials must be configured on the machine running ducktape.
+
+Ducktape uses the [boto3](https://aws.amazon.com/sdk-for-python/) Python module to connect to AWS. And `boto3` supports many different [configuration options](https://boto3.readthedocs.io/en/latest/guide/configuration.html#guide-configuration).
+
+Here's an example bare minimum configuration using environment variables:
+
+    export AWS_ACCESS_KEY_ID="ABC123"
+    export AWS_SECRET_ACCESS_KEY="secret"
+    export AWS_DEFAULT_REGION="us-east-1"
+
+The region can be any AWS region, not just `us-east-1`.
 
 Contribute
 ----------

ducktape/__init__.py

@@ -1 +1 @@
-__version__ = '0.6.1'
+__version__ = '0.6.2'

ducktape/cluster/cluster.py

@@ -47,10 +47,6 @@ def request(self, num_nodes):
         """Identical to alloc. Keeping for compatibility"""
         return self.alloc(num_nodes)
 
-    def num_available_nodes(self):
-        """Number of available nodes."""
-        raise NotImplementedError()
-
     def free(self, nodes):
         """Free the given node or list of nodes"""
         if isinstance(nodes, collections.Iterable):
@@ -69,7 +65,8 @@ def __hash__(self):
         return hash(tuple(sorted(self.__dict__.items())))
 
     def num_nodes_for_operating_system(self, operating_system):
-        return self.in_use_nodes_for_operating_system(operating_system) + self.num_available_nodes(operating_system)
+        return self.in_use_nodes_for_operating_system(operating_system) + \
+               self.num_available_nodes(operating_system=operating_system)
 
     def num_available_nodes(self, operating_system=RemoteAccount.LINUX):
         """Number of available nodes."""
@@ -78,6 +75,31 @@ def num_available_nodes(self, operating_system=RemoteAccount.LINUX):
     def in_use_nodes_for_operating_system(self, operating_system):
         return Cluster._node_count_helper(self._in_use_nodes, operating_system)
 
+    @property
+    def node_spec(self):
+        node_spec = {}
+        for operating_system in RemoteAccount.SUPPORTED_OS_TYPES:
+            node_spec[operating_system] = self.num_nodes_for_operating_system(operating_system)
+        return node_spec
+
+    def test_capacity_comparison(self, test):
+        """
+        Checks if the test can 'fit' into the cluster, using the test's node_spec.
+
+        A negative return value (int) means the test needs more capacity than is available in the cluster.
+        A return value of 0 means the cluster has exactly the right number of resources as the test needs.
+        A positive return value (int) means the cluster has more capacity than the test needs.
+        """
+        num_available = 0
+        for (operating_system, node_count) in test.expected_node_spec.iteritems():
+            if node_count > self.num_nodes_for_operating_system(operating_system):
+                # return -1 immediately if the cluster doesn't have enough capacity for any operating system.
+                return -1
+            else:
+                num_available += self.num_nodes_for_operating_system(operating_system) - node_count
+
+        return num_available
+
     @staticmethod
     def _node_count_helper(nodes, operating_system):
         return len([node for node in nodes if node.operating_system == operating_system])

ducktape/cluster/json.py

@@ -19,7 +19,7 @@
 from .remoteaccount import RemoteAccount
 from ducktape.cluster.linux_remoteaccount import LinuxRemoteAccount
 from ducktape.cluster.windows_remoteaccount import WindowsRemoteAccount
-from .linux_remoteaccount import RemoteAccountSSHConfig
+from .remoteaccount import RemoteAccountSSHConfig
 
 import collections
 import json
@@ -82,12 +82,12 @@ def __init__(self, cluster_json=None, *args, **kwargs):
         try:
             node_accounts = []
             for ninfo in cluster_json["nodes"]:
-                remote_command_config_dict = ninfo.get("ssh_config")
-                assert remote_command_config_dict is not None, \
-                    "Cluster json has a node without an ssh_config field: %s\n Cluster json: %s" % (ninfo, cluster_json)
+                ssh_config_dict = ninfo.get("ssh_config")
+                assert ssh_config_dict is not None, \
+                    "Cluster json has a node without a ssh_config field: %s\n Cluster json: %s" % (ninfo, cluster_json)
 
-                remote_command_config = RemoteAccountSSHConfig(**ninfo.get("ssh_config", {}))
-                node_accounts.append(JsonCluster.make_remote_account(remote_command_config, ninfo.get("externally_routable_ip")))
+                ssh_config = RemoteAccountSSHConfig(**ninfo.get("ssh_config", {}))
+                node_accounts.append(JsonCluster.make_remote_account(ssh_config, ninfo.get("externally_routable_ip")))
 
             for node_account in node_accounts:
                 if node_account.externally_routable_ip is None:
@@ -102,14 +102,14 @@ def __init__(self, cluster_json=None, *args, **kwargs):
         self._id_supplier = 0
 
     @staticmethod
-    def make_remote_account(remote_command_config, externally_routable_ip=None):
+    def make_remote_account(ssh_config, externally_routable_ip=None):
         """Factory function for creating the correct RemoteAccount implementation."""
 
-        if remote_command_config.host and RemoteAccount.WINDOWS in remote_command_config.host:
-            return WindowsRemoteAccount(winrm_config=remote_command_config,
+        if ssh_config.host and RemoteAccount.WINDOWS in ssh_config.host:
+            return WindowsRemoteAccount(ssh_config=ssh_config,
                                         externally_routable_ip=externally_routable_ip)
         else:
-            return LinuxRemoteAccount(ssh_config=remote_command_config,
+            return LinuxRemoteAccount(ssh_config=ssh_config,
                                       externally_routable_ip=externally_routable_ip)
 
     def __len__(self):