YARN-7186. Add examples in yarn-service. Contributed by Jian He

kunpengcompute · Nov 6, 2017 · 37c9b73 · 37c9b73
1 parent 307d55b
commit 37c9b73
Show file tree

Hide file tree

Showing 14 changed files with 91 additions and 114 deletions.
diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
@@ -97,6 +97,13 @@
       <directory>hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/conf</directory>
       <outputDirectory>etc/hadoop</outputDirectory>
     </fileSet>
+    <fileSet>
+      <directory>hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples</directory>
+      <outputDirectory>/share/hadoop/${hadoop.component}/yarn-service-examples</outputDirectory>
+      <includes>
+        <include>**/*</include>
+      </includes>
+    </fileSet>
     <fileSet>
       <directory>hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/target</directory>
       <outputDirectory>/share/hadoop/${hadoop.component}/sources</outputDirectory>

diff --git a/...applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/sleeper/sleeper.json b/...applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/sleeper/sleeper.json
@@ -0,0 +1,15 @@
+{
+  "name": "sleeper-service",
+  "components" :
+  [
+    {
+      "name": "sleeper",
+      "number_of_containers": 2,
+      "launch_command": "sleep 900000",
+      "resource": {
+        "cpus": 1,
+        "memory": "256"
+      }
+    }
+  ]
+}
diff --git a/...yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java b/...yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
@@ -66,6 +66,7 @@
 import org.apache.hadoop.yarn.service.api.records.Component;
 import org.apache.hadoop.yarn.service.api.records.ServiceState;
 import org.apache.hadoop.yarn.service.client.params.AbstractClusterBuildingActionArgs;
+import org.apache.hadoop.yarn.service.client.params.ActionCreateArgs;
 import org.apache.hadoop.yarn.service.client.params.ActionDependencyArgs;
 import org.apache.hadoop.yarn.service.client.params.ActionFlexArgs;
 import org.apache.hadoop.yarn.service.client.params.Arguments;
@@ -160,12 +161,15 @@ private Service loadAppJsonFromLocalFS(
       AbstractClusterBuildingActionArgs args) throws IOException {
     File file = args.getFile();
     Path filePath = new Path(file.getAbsolutePath());
-    LOG.info("Loading app json from: " + filePath);
+    LOG.info("Loading service definition from: " + filePath);
     Service service = jsonSerDeser
         .load(FileSystem.getLocal(getConfig()), filePath);
     if (args.lifetime > 0) {
       service.setLifetime(args.lifetime);
     }
+    if (!StringUtils.isEmpty(args.getServiceName())) {
+      service.setName(args.getServiceName());
+    }
     return service;
   }
 
@@ -182,9 +186,23 @@ public int actionBuild(Service service)
     return EXIT_SUCCESS;
   }
 
-  public int actionCreate(AbstractClusterBuildingActionArgs args)
+  public int actionCreate(ActionCreateArgs args)
       throws IOException, YarnException {
-    actionCreate(loadAppJsonFromLocalFS(args));
+    Service serviceDef;
+    if (args.file != null) {
+      serviceDef = loadAppJsonFromLocalFS(args);
+    } else if (!StringUtils.isEmpty(args.example)) {
+      // create an example service
+      String yarnHome = System
+          .getenv(ApplicationConstants.Environment.HADOOP_YARN_HOME.key());
+      args.file = new File(MessageFormat
+          .format("{0}/share/hadoop/yarn/yarn-service-examples/{1}/{2}.json",
+              yarnHome, args.example, args.example));
+      serviceDef = loadAppJsonFromLocalFS(args);
+    } else {
+      throw new YarnException("No service definition provided!");
+    }
+    actionCreate(serviceDef);
     return EXIT_SUCCESS;
   }
 
@@ -213,7 +231,7 @@ protected int actionFlexByCLI(ClientArgs args)
     Map<String, Long> componentCounts =
         new HashMap<>(flexArgs.getComponentMap().size());
     Service persistedService =
-        ServiceApiUtil.loadService(fs, flexArgs.getClusterName());
+        ServiceApiUtil.loadService(fs, flexArgs.getServiceName());
     if (!StringUtils.isEmpty(persistedService.getId())) {
       cachedAppIds.put(persistedService.getName(),
           ApplicationId.fromString(persistedService.getId()));

diff --git a/...s-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractActionArgs.java b/...s-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractActionArgs.java
@@ -50,7 +50,7 @@ protected AbstractActionArgs() {
    * get the name: relies on arg 1 being the cluster name in all operations 
    * @return the name argument, null if there is none
    */
-  public String getClusterName() {
+  public String getServiceName() {
     return (parameters.isEmpty()) ? null : parameters.get(0);
   }
 

diff --git a/.../java/org/apache/hadoop/yarn/service/client/params/AbstractClusterBuildingActionArgs.java b/.../java/org/apache/hadoop/yarn/service/client/params/AbstractClusterBuildingActionArgs.java
@@ -28,7 +28,7 @@
  */
 public abstract class AbstractClusterBuildingActionArgs
     extends AbstractActionArgs {
-  @Parameter(names = { ARG_FILE, ARG_FILE_SHORT },  required = true,
+  @Parameter(names = { ARG_FILE, ARG_FILE_SHORT },
       description = "The path to the service definition file in JSON format.")
   public File file;
 

diff --git a/...ices-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionBuildArgs.java b/...ices-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionBuildArgs.java
@@ -18,6 +18,7 @@
 package org.apache.hadoop.yarn.service.client.params;
 
 import com.beust.jcommander.Parameters;
+import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
 
 @Parameters(commandNames = { SliderActions.ACTION_BUILD},
             commandDescription = SliderActions.DESCRIBE_ACTION_BUILD)
@@ -30,7 +31,9 @@ public String getActionName() {
   }
 
   @Override
-  public int getMinParams() {
-    return 0;
+  public void validate() throws BadCommandArgumentsException {
+    if (file == null) {
+      throw new BadCommandArgumentsException("No service definition provided.");
+    }
   }
 }
diff --git a/...ces-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionCreateArgs.java b/...ces-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionCreateArgs.java
@@ -18,21 +18,29 @@
 
 package org.apache.hadoop.yarn.service.client.params;
 
+import com.beust.jcommander.Parameter;
 import com.beust.jcommander.Parameters;
+import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
 
 @Parameters(commandNames = { SliderActions.ACTION_CREATE},
             commandDescription = SliderActions.DESCRIBE_ACTION_CREATE)
 
 public class ActionCreateArgs extends AbstractClusterBuildingActionArgs {
 
+  @Parameter(names = { ARG_EXAMPLE, ARG_EXAMPLE_SHORT },
+      description = "The name of the example service such as sleeper")
+  public String example;
+
   @Override
   public String getActionName() {
     return SliderActions.ACTION_CREATE;
   }
 
   @Override
-  public int getMinParams() {
-    return 0;
+  public void validate() throws BadCommandArgumentsException {
+    if (file == null && example == null) {
+      throw new BadCommandArgumentsException("No service definition provided.");
+    }
   }
 }
 
diff --git a/...vices-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionExamples.java b/...vices-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionExamples.java
diff --git a/...n-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/Arguments.java b/...n-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/Arguments.java
@@ -42,8 +42,8 @@ public interface Arguments {
   String ARG_DELETE = "--delete";
   String ARG_DEST = "--dest";
   String ARG_DESTDIR = "--destdir";
-  String ARG_FILESYSTEM = "--fs";
-  String ARG_FILESYSTEM_LONG = "--filesystem";
+  String ARG_EXAMPLE = "--example";
+  String ARG_EXAMPLE_SHORT = "-e";
   String ARG_FOLDER = "--folder";
   String ARG_FORCE = "--force";
   String ARG_FORMAT = "--format";

diff --git a/...-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ClientArgs.java b/...-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ClientArgs.java
@@ -19,8 +19,6 @@
 package org.apache.hadoop.yarn.service.client.params;
 
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.service.conf.YarnServiceConf;
 import org.apache.hadoop.yarn.service.utils.SliderUtils;
 import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
 import org.apache.hadoop.yarn.service.exceptions.ErrorStrings;
@@ -104,51 +102,10 @@ public ActionDependencyArgs getActionDependencyArgs() {
     return actionDependencyArgs;
   }
 
-  public ActionDestroyArgs getActionDestroyArgs() {
-    return actionDestroyArgs;
-  }
-
-  public ActionExistsArgs getActionExistsArgs() {
-    return actionExistsArgs;
-  }
-
   public ActionFlexArgs getActionFlexArgs() {
     return actionFlexArgs;
   }
 
-  public ActionFreezeArgs getActionFreezeArgs() {
-    return actionFreezeArgs;
-  }
-
-  public ActionListArgs getActionListArgs() {
-    return actionListArgs;
-  }
-
-
-  public ActionRegistryArgs getActionRegistryArgs() {
-    return actionRegistryArgs;
-  }
-
-  public ActionResolveArgs getActionResolveArgs() {
-    return actionResolveArgs;
-  }
-
-  public ActionResourceArgs getActionResourceArgs() {
-    return actionResourceArgs;
-  }
-
-  public ActionStatusArgs getActionStatusArgs() {
-    return actionStatusArgs;
-  }
-
-  public ActionThawArgs getActionThawArgs() {
-    return actionThawArgs;
-  }
-
-  public ActionTokensArgs getActionTokenArgs() {
-    return actionTokenArgs;
-  }
-
   /**
    * Look at the chosen action and bind it as the core action for the operation.
    * @throws SliderException bad argument or similar
@@ -227,7 +184,6 @@ public void applyAction() throws SliderException {
       case ACTION_UPDATE:
         bindCoreAction(actionUpdateArgs);
         break;
-
       default:
         throw new BadCommandArgumentsException(ErrorStrings.ERROR_UNKNOWN_ACTION
         + " " + action);

diff --git a/...-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/CommonArgs.java b/...-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/CommonArgs.java
@@ -24,7 +24,6 @@
 import com.beust.jcommander.ParameterException;
 
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.yarn.service.utils.SliderUtils;
 import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
 import org.apache.hadoop.yarn.service.exceptions.ErrorStrings;
@@ -88,7 +87,7 @@ public abstract class CommonArgs extends ArgOps implements SliderActions,
    * @return the name argument, null if there is none
    */
   public String getClusterName() {
-    return coreAction.getClusterName();
+    return coreAction.getServiceName();
   }
 
   protected CommonArgs(String[] args) {
@@ -142,10 +141,6 @@ public static String usage(CommonArgs serviceArgs, String commandOfInterest) {
     return result;
   }
 
-  public static String usage(CommonArgs serviceArgs) {
-    return usage(serviceArgs, null);
-  }
-
   /**
    * Parse routine -includes registering the action-specific argument classes
    * and postprocess it
@@ -164,14 +159,6 @@ public void parse() throws SliderException {
     postProcess();
   }
 
-  /**
-   * Add a command
-   * @param name action
-   * @param arg value
-   */
-  protected void addAction(String name, Object arg) {
-    commander.addCommand(name, arg);
-  }
 
   protected void addActions(Object... actions) {
     for (Object action : actions) {

diff --git a/...rvices-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderActions.java b/...rvices-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderActions.java
@@ -30,7 +30,6 @@ public interface SliderActions {
   String ACTION_UPGRADE = "upgrade";
   String ACTION_DESTROY = "destroy";
   String ACTION_EXISTS = "exists";
-  String ACTION_EXAMPLES = "examples";
   String ACTION_FLEX = "flex";
   String ACTION_STOP = "stop";
   String ACTION_HELP = "help";
@@ -59,7 +58,6 @@ public interface SliderActions {
         "Destroy a stopped service, service must be stopped first before destroying.";
   String DESCRIBE_ACTION_EXISTS =
             "Probe for a service running";
-  String DESCRIBE_ACTION_EXAMPLES = "Run an example service on YARN";
   String DESCRIBE_ACTION_FLEX = "Flex a service's component by increasing or decreasing the number of containers.";
   String DESCRIBE_ACTION_FREEZE =
               "Stop a running service";

diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md
@@ -76,28 +76,32 @@ Usage `yarn service [sub-command] [service-name] [options]`
 
 * `build`:  Build a service with its specifications, but do not start it.
     ```
-    Usage: yarn service build --file [file]
-    ```
+    Usage: yarn service build [service-name] --file [file]
+    Fields:
+    service-name     Optional. If specified, it will override the name in the service definition.
 
-  | COMMAND\_OPTIONS | Description |
-  |:---- |:---- |
-  | --file or -f | The local path to the service definition file |
+    Options:
+      --file,-f      The local path to the service definition file
+    ```
 
 * `create`:  create a service, it's equivalent to first invoke build and then start.
    ```
-   Usage: yarn service create --file [file]
+   Usage: yarn service create [service-name] --file [file]
+   Fields:
+    service-name    Optional. If specified, it will override the name in the service definition.
+
+   Options:
+    --file,-f       The local path to the service definition file.
+    --example,-e    The name of the example service such as:
+                    Sleeper      A simple service that launches a few non-docker sleep containers on YARN.
    ```
-  | COMMAND\_OPTIONS | Description |
-  |:---- |:---- |
-  | --file or -f | The local path to the service definition file |
 
 * `dependency`:  Yarn service framework dependency (libraries) management.
   ```
   Usage: yarn service dependency [options]
+  Option:
+     --upload      Pre-upload the dependency jars onto HDFS to expediate service launch process.
   ```
-  | COMMAND\_OPTIONS | Description |
-  |:---- |:---- |
-  | --upload | Pre-upload the dependency jars onto HDFS to expediate service launch process. |
 
 * `destroy`:  Destroy a stopped service, service must be stopped first before destroying.
   ```
@@ -106,10 +110,10 @@ Usage `yarn service [sub-command] [service-name] [options]`
 * `flex`:   Flex a service's component by increasing or decreasing the number of containers.
   ```
   Usage: yarn service flex [service-name] --component [component-name] [count]
+  Options:
+    --component [component-name] [count]
+            Specifies the component name and its number of containers. e.g. +1 incr by 1, -2 decr by 2, and 3 makes final count 3.
   ```
-  | COMMAND\_OPTIONS | Description |
-  |:---- |:---- |
-  | --component [component-name] [count] | Specifies the component name and its number of containers. e.g. +1 incr by 1, -2 decr by 2, and 3 makes final count 3.|
 * `status`:  Get the status of a service.
   ```
   Usage: yarn service status [service-name]