JCLOUDS-517: New ElasticHosts images and regions

Added the new ElasticHosts regions. Updated the ElasticStack api to get the list of standard drives using an API call. All providers except ServerLove support the new API call, so the old logic in the ElasticStack api has been moved to that provider. The rest of providers will now extract all the OperatingSystem information by parsing the name of the StandardDrive. A unit test has been added to the ElasticStack api with all the images that were hardcoded, to make sure all names are still parsed as expected and all information in the existing providers is kept. Modified the default template for all ElasticHosts providers to match newer Ubuntu images and updated the Template*Live tests accordingly. Also refactored the WellKnownImage map to a supplier to lazy load it when needed and avoid unexpected errors when building the Guice injector if there are authentication errors or similar.
arr-ee · Jun 2, 2014 · a7e3424 · a7e3424
1 parent da8517f
commit a7e3424
Show file tree

Hide file tree

Showing 83 changed files with 3,326 additions and 721 deletions.
diff --git a/apis/elasticstack/pom.xml b/apis/elasticstack/pom.xml
@@ -78,6 +78,18 @@
       <version>${project.version}</version>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>com.squareup.okhttp</groupId>
+      <artifactId>mockwebserver</artifactId>
+      <scope>test</scope>
+      <exclusions>
+        <!-- Already provided by jclouds-sshj -->
+        <exclusion>
+          <groupId>org.bouncycastle</groupId>
+          <artifactId>bcprov-jdk15on</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
   </dependencies>
   <profiles>
     <profile>

diff --git a/apis/elasticstack/src/main/java/org/jclouds/elasticstack/ElasticStackApi.java b/apis/elasticstack/src/main/java/org/jclouds/elasticstack/ElasticStackApi.java
@@ -38,10 +38,12 @@
 import org.jclouds.elasticstack.domain.ImageConversionType;
 import org.jclouds.elasticstack.domain.Server;
 import org.jclouds.elasticstack.domain.ServerInfo;
+import org.jclouds.elasticstack.domain.StandardDrive;
 import org.jclouds.elasticstack.functions.KeyValuesDelimitedByBlankLinesToDriveInfo;
 import org.jclouds.elasticstack.functions.KeyValuesDelimitedByBlankLinesToServerInfo;
 import org.jclouds.elasticstack.functions.ListOfKeyValuesDelimitedByBlankLinesToDriveInfoSet;
 import org.jclouds.elasticstack.functions.ListOfKeyValuesDelimitedByBlankLinesToServerInfoSet;
+import org.jclouds.elasticstack.functions.ListOfKeyValuesDelimitedByBlankLinesToStandardDriveSet;
 import org.jclouds.elasticstack.functions.ReturnPayload;
 import org.jclouds.elasticstack.functions.SplitNewlines;
 import org.jclouds.http.filters.BasicAuthentication;
@@ -201,6 +203,26 @@ ServerInfo setServerConfiguration(@PathParam("uuid") String uuid,
    @Path("/drives/info")
    @ResponseParser(ListOfKeyValuesDelimitedByBlankLinesToDriveInfoSet.class)
    Set<DriveInfo> listDriveInfo();
+
+   /**
+    * Lists standard drive UUIDs in your account
+    *
+    * @return or empty set if no standard drives are found
+    */
+   @GET
+   @Path("/drives/list/standard")
+   @ResponseParser(SplitNewlines.class)
+   Set<String> listStandardDrives();
+
+   /**
+    * Gets information about all standard drives
+    *
+    * @return or empty set if no standard drives are found
+    */
+   @GET
+   @Path("/drives/info/standard")
+   @ResponseParser(ListOfKeyValuesDelimitedByBlankLinesToStandardDriveSet.class)
+   Set<StandardDrive> listStandardDriveInfo();
 
    /**
     * @param uuid

diff --git a/apis/elasticstack/src/main/java/org/jclouds/elasticstack/ElasticStackApiMetadata.java b/apis/elasticstack/src/main/java/org/jclouds/elasticstack/ElasticStackApiMetadata.java
@@ -62,7 +62,6 @@ public static Properties defaultProperties() {
 
    public static class Builder extends BaseHttpApiMetadata.Builder<ElasticStackApi, Builder> {
 
-      @SuppressWarnings("deprecation")
       protected Builder() {
          id("elasticstack")
          .name("ElasticStack API")

diff --git a/...ack/src/main/java/org/jclouds/elasticstack/compute/ElasticStackComputeServiceAdapter.java b/...ack/src/main/java/org/jclouds/elasticstack/compute/ElasticStackComputeServiceAdapter.java
@@ -32,6 +32,7 @@
 import javax.inject.Singleton;
 
 import org.jclouds.Constants;
+import org.jclouds.collect.Memoized;
 import org.jclouds.compute.ComputeServiceAdapter;
 import org.jclouds.compute.domain.Hardware;
 import org.jclouds.compute.domain.HardwareBuilder;
@@ -57,6 +58,7 @@
 
 import com.google.common.base.Function;
 import com.google.common.base.Predicate;
+import com.google.common.base.Supplier;
 import com.google.common.cache.CacheLoader;
 import com.google.common.cache.LoadingCache;
 import com.google.common.collect.FluentIterable;
@@ -70,15 +72,15 @@
 
 /**
  * defines the connection between the {@link org.jclouds.elasticstack.ElasticStackApi} implementation
- * and the jclouds {@link ComputeService}
+ * and the jclouds {@link org.jclouds.compute.ComputeService}
  * 
  */
 @Singleton
 public class ElasticStackComputeServiceAdapter implements
       ComputeServiceAdapter<ServerInfo, Hardware, DriveInfo, Location> {
    private final ElasticStackApi client;
    private final Predicate<DriveInfo> driveNotClaimed;
-   private final Map<String, WellKnownImage> preinstalledImages;
+   private final Supplier<Map<String, WellKnownImage>> preinstalledImages;
    private final LoadingCache<String, DriveInfo> cache;
    private final String defaultVncPassword;
    private final ListeningExecutorService userExecutor;
@@ -89,7 +91,7 @@ public class ElasticStackComputeServiceAdapter implements
 
    @Inject
    public ElasticStackComputeServiceAdapter(ElasticStackApi client, Predicate<DriveInfo> driveNotClaimed,
-         Map<String, WellKnownImage> preinstalledImages, LoadingCache<String, DriveInfo> cache,
+         @Memoized Supplier<Map<String, WellKnownImage>> preinstalledImages, LoadingCache<String, DriveInfo> cache,
          @Named(ElasticStackConstants.PROPERTY_VNC_PASSWORD) String defaultVncPassword,
          @Named(Constants.PROPERTY_USER_THREADS) ListeningExecutorService userExecutor) {
       this.client = checkNotNull(client, "client");
@@ -108,16 +110,22 @@ public NodeAndInitialCredentials<ServerInfo> createNodeWithGroupEncodedIntoName(
       logger.debug(">> creating boot drive bytes(%d)", bootSize);
       DriveInfo drive = client
             .createDrive(new Drive.Builder().name(template.getImage().getId()).size(bootSize).build());
-      logger.debug("<< drive(%s)", drive.getUuid());
+      logger.debug("<< drive (%s)", drive);
 
-      logger.debug(">> imaging boot drive source(%s)", template.getImage().getId());
-      client.imageDrive(template.getImage().getId(), drive.getUuid(), ImageConversionType.GUNZIP);
       boolean success = driveNotClaimed.apply(drive);
-      logger.debug("<< imaged (%s)", success);
       if (!success) {
          client.destroyDrive(drive.getUuid());
-         throw new IllegalStateException("could not image drive in time!");
+         throw new IllegalStateException(String.format("could not create drive %s in time!", drive));
+      }
+
+      logger.debug(">> imaging boot drive source(%s)", template.getImage().getId());
+      client.imageDrive(template.getImage().getId(), drive.getUuid(), ImageConversionType.GUNZIP);
+      boolean ready = driveNotClaimed.apply(drive);
+      if (!ready) {
+         client.destroyDrive(drive.getUuid());
+         throw new IllegalStateException(String.format("could not image drive %s in time!", drive));
       }
+      logger.debug("<< imaged (%s)", drive);
 
       template.getOptions().userMetadata(ComputeServiceConstants.NODE_GROUP_KEY, tag);
 
@@ -126,6 +134,7 @@ public NodeAndInitialCredentials<ServerInfo> createNodeWithGroupEncodedIntoName(
                .tags(template.getOptions().getTags()).userMetadata(template.getOptions().getUserMetadata()).build();
 
       ServerInfo from = client.createServer(toCreate);
+
       client.startServer(from.getUuid());
       from = client.getServerInfo(from.getUuid());
       return new NodeAndInitialCredentials<ServerInfo>(from, from.getUuid(), LoginCredentials.builder()
@@ -164,7 +173,7 @@ public String toString() {
     */
    @Override
    public Iterable<DriveInfo> listImages() {
-      return FluentIterable.from(transformParallel(preinstalledImages.keySet(),
+      return FluentIterable.from(transformParallel(preinstalledImages.get().keySet(),
             new Function<String, ListenableFuture<? extends DriveInfo>>() {
 
                @Override
@@ -187,7 +196,6 @@ public String toString() {
             }, userExecutor, null, logger, "drives")).filter(notNull());
    }
 
-   @SuppressWarnings("unchecked")
    @Override
    public Iterable<ServerInfo> listNodes() {
       return (Iterable<ServerInfo>) client.listServerInfo();

diff --git a/...java/org/jclouds/elasticstack/compute/config/ElasticStackComputeServiceContextModule.java b/...java/org/jclouds/elasticstack/compute/config/ElasticStackComputeServiceContextModule.java
@@ -16,16 +16,22 @@
  */
 package org.jclouds.elasticstack.compute.config;
 
+import static com.google.common.base.Suppliers.memoizeWithExpiration;
 import static java.util.concurrent.TimeUnit.MILLISECONDS;
+import static org.jclouds.Constants.PROPERTY_SESSION_INTERVAL;
 import static org.jclouds.util.Predicates2.retry;
 
 import java.io.IOException;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
 
 import javax.inject.Inject;
+import javax.inject.Named;
 import javax.inject.Singleton;
 
+import org.jclouds.collect.Memoized;
 import org.jclouds.compute.ComputeServiceAdapter;
 import org.jclouds.compute.config.ComputeServiceAdapterContextModule;
 import org.jclouds.compute.domain.Hardware;
@@ -39,21 +45,24 @@
 import org.jclouds.elasticstack.compute.functions.ServerInfoToNodeMetadata;
 import org.jclouds.elasticstack.compute.functions.ServerInfoToNodeMetadata.DeviceToVolume;
 import org.jclouds.elasticstack.compute.functions.ServerInfoToNodeMetadata.GetImageIdFromServer;
+import org.jclouds.elasticstack.compute.functions.StandardDriveToWellKnownImage;
 import org.jclouds.elasticstack.compute.functions.WellKnownImageToImage;
 import org.jclouds.elasticstack.domain.Device;
 import org.jclouds.elasticstack.domain.DriveInfo;
 import org.jclouds.elasticstack.domain.Server;
 import org.jclouds.elasticstack.domain.ServerInfo;
+import org.jclouds.elasticstack.domain.StandardDrive;
 import org.jclouds.elasticstack.domain.WellKnownImage;
 import org.jclouds.elasticstack.predicates.DriveClaimed;
+import org.jclouds.elasticstack.suppliers.WellKnownImageSupplier;
 import org.jclouds.functions.IdentityFunction;
-import org.jclouds.json.Json;
-import org.jclouds.location.Provider;
-import org.jclouds.util.Strings2;
+import org.jclouds.rest.AuthorizationException;
+import org.jclouds.rest.suppliers.MemoizedRetryOnTimeOutButNotOnAuthorizationExceptionSupplier;
 
 import com.google.common.base.Function;
 import com.google.common.base.Predicate;
 import com.google.common.base.Predicates;
+import com.google.common.base.Supplier;
 import com.google.common.cache.CacheBuilder;
 import com.google.common.cache.CacheLoader;
 import com.google.common.cache.LoadingCache;
@@ -66,7 +75,7 @@
  * @author Adrian Cole
  */
 public class ElasticStackComputeServiceContextModule extends
-         ComputeServiceAdapterContextModule<ServerInfo, Hardware, DriveInfo, Location> {
+      ComputeServiceAdapterContextModule<ServerInfo, Hardware, DriveInfo, Location> {
 
    @SuppressWarnings("unchecked")
    @Override
@@ -88,8 +97,12 @@ protected void configure() {
       }).to(GetImageIdFromServer.class);
       bind(new TypeLiteral<Function<DriveInfo, Image>>() {
       }).to(WellKnownImageToImage.class);
+      bind(new TypeLiteral<Function<StandardDrive, WellKnownImage>>() {
+      }).to(StandardDriveToWellKnownImage.class);
+      bind(new TypeLiteral<Supplier<List<WellKnownImage>>>() {
+      }).to(WellKnownImageSupplier.class);
    }
-   
+
    @Provides
    @Singleton
    protected LoadingCache<String, DriveInfo> cache(GetDrive getDrive) {
@@ -113,18 +126,33 @@ public DriveInfo load(String input) {
 
    @Singleton
    @Provides
-   protected Map<String, WellKnownImage> provideImages(Json json, @Provider String providerName) throws IOException {
-      List<WellKnownImage> wellKnowns = json.fromJson(Strings2.toStringAndClose(getClass().getResourceAsStream(
-               "/" + providerName + "/preinstalled_images.json")), new TypeLiteral<List<WellKnownImage>>() {
-      }.getType());
-      return Maps.uniqueIndex(wellKnowns, new Function<WellKnownImage, String>() {
-
+   @Memoized
+   protected Supplier<Map<String, WellKnownImage>> provideImages(@Named(PROPERTY_SESSION_INTERVAL) long seconds,
+         @Memoized final Supplier<List<WellKnownImage>> wellKnownImageSupplier) throws IOException {
+      // The image map won't change. Memoize it during the session.
+      // This map can't be created directly as a singleton, as Guice needs it to construct the ElasticStackComputeServiceAdapter
+      // and a misconfiguration such as invalid credentials, etc would cause the Guice injection to fail
+      return memoizeWithExpiration(new Supplier<Map<String, WellKnownImage>>() {
          @Override
-         public String apply(WellKnownImage input) {
-            return input.getUuid();
+         public Map<String, WellKnownImage> get() {
+            return Maps.uniqueIndex(wellKnownImageSupplier.get(), new Function<WellKnownImage, String>() {
+               @Override
+               public String apply(WellKnownImage input) {
+                  return input.getUuid();
+               }
+            });
          }
-
-      });
+      }, seconds, TimeUnit.SECONDS);
+   }
+
+   @Singleton
+   @Provides
+   @Memoized
+   protected Supplier<List<WellKnownImage>> provideWellKnownImageSupplier(AtomicReference<AuthorizationException> authException,
+         @Named(PROPERTY_SESSION_INTERVAL) long seconds, WellKnownImageSupplier uncached)
+         throws IOException {
+      return MemoizedRetryOnTimeOutButNotOnAuthorizationExceptionSupplier.create(authException, uncached, seconds,
+            TimeUnit.SECONDS);
    }
 
    @Provides