YARN-9379. Can't specify docker runtime through environment. Contributed by caozhiqiang

Author: ericbadger

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java

@@ -2030,6 +2030,16 @@ public static boolean isAclEnabled(Configuration conf) {
   public static final String DEFAULT_NM_DOCKER_DEFAULT_CONTAINER_NETWORK =
       "host";
 
+  /** The set of runtimes allowed when launching containers using the
+   * DockerContainerRuntime. */
+  public static final String NM_DOCKER_ALLOWED_CONTAINER_RUNTIMES =
+          DOCKER_CONTAINER_RUNTIME_PREFIX + "allowed-container-runtimes";
+
+  /** The set of runtimes allowed when launching containers using the
+   * DockerContainerRuntime. */
+  public static final String[] DEFAULT_NM_DOCKER_ALLOWED_CONTAINER_RUNTIMES =
+      {"runc"};
+
   /** Allow host pid namespace for containers. Use with care. */
   public static final String NM_DOCKER_ALLOW_HOST_PID_NAMESPACE =
       DOCKER_CONTAINER_RUNTIME_PREFIX + "host-pid-namespace.allowed";

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml

@@ -1803,6 +1803,13 @@
     <value>host</value>
   </property>
 
+  <property>
+    <description>The set of runtimes allowed when launching containers using the
+      DockerContainerRuntime.</description>
+    <name>yarn.nodemanager.runtime.linux.docker.allowed-container-runtimes</name>
+    <value>runc</value>
+  </property>
+
   <property>
     <description>This configuration setting determines whether the host's PID
       namespace is allowed for docker containers on this cluster.

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java

@@ -265,6 +265,9 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
   @InterfaceAudience.Private
   public static final String ENV_DOCKER_CONTAINER_YARN_SYSFS =
       "YARN_CONTAINER_RUNTIME_YARN_SYSFS_ENABLE";
+  @InterfaceAudience.Private
+  public static final String ENV_DOCKER_CONTAINER_DOCKER_RUNTIME =
+      "YARN_CONTAINER_RUNTIME_DOCKER_RUNTIME";
   public static final String YARN_SYSFS_PATH =
       "/hadoop/yarn/sysfs";
   private Configuration conf;
@@ -275,7 +278,9 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
   private String defaultImageName;
   private Boolean defaultImageUpdate;
   private Set<String> allowedNetworks = new HashSet<>();
+  private Set<String> allowedRuntimes = new HashSet<>();
   private String defaultNetwork;
+  private String defaultRuntime;
   private CGroupsHandler cGroupsHandler;
   private AccessControlList privilegedContainersAcl;
   private boolean enableUserReMapping;
@@ -349,6 +354,7 @@ public void initialize(Configuration conf, Context nmContext)
     this.conf = conf;
     dockerClient = new DockerClient();
     allowedNetworks.clear();
+    allowedRuntimes.clear();
     defaultROMounts.clear();
     defaultRWMounts.clear();
     defaultTmpfsMounts.clear();
@@ -363,6 +369,10 @@ public void initialize(Configuration conf, Context nmContext)
     defaultNetwork = conf.getTrimmed(
         YarnConfiguration.NM_DOCKER_DEFAULT_CONTAINER_NETWORK,
         YarnConfiguration.DEFAULT_NM_DOCKER_DEFAULT_CONTAINER_NETWORK);
+    allowedRuntimes.addAll(Arrays.asList(
+        conf.getTrimmedStrings(
+            YarnConfiguration.NM_DOCKER_ALLOWED_CONTAINER_RUNTIMES,
+            YarnConfiguration.DEFAULT_NM_DOCKER_ALLOWED_CONTAINER_RUNTIMES)));
 
     if(!allowedNetworks.contains(defaultNetwork)) {
       String message = "Default network: " + defaultNetwork
@@ -529,6 +539,19 @@ private void validateContainerNetworkType(String network)
     throw new ContainerExecutionException(msg);
   }
 
+  private void validateContainerRuntimeType(String runtime)
+          throws ContainerExecutionException {
+    if (runtime == null || runtime.isEmpty()
+        || allowedRuntimes.contains(runtime)) {
+      return;
+    }
+
+    String msg = "Disallowed runtime:  '" + runtime
+            + "' specified. Allowed networks: are " + allowedRuntimes
+            .toString();
+    throw new ContainerExecutionException(msg);
+  }
+
   /**
    * Return whether the YARN container is allowed to run using the host's PID
    * namespace for the Docker container. For this to be allowed, the submitting
@@ -801,6 +824,7 @@ public void launchContainer(ContainerRuntimeContext ctx)
     String imageName = environment.get(ENV_DOCKER_CONTAINER_IMAGE);
     String network = environment.get(ENV_DOCKER_CONTAINER_NETWORK);
     String hostname = environment.get(ENV_DOCKER_CONTAINER_HOSTNAME);
+    String runtime = environment.get(ENV_DOCKER_CONTAINER_DOCKER_RUNTIME);
     boolean useEntryPoint = checkUseEntryPoint(environment);
 
     if (imageName == null || imageName.isEmpty()) {
@@ -816,6 +840,8 @@ public void launchContainer(ContainerRuntimeContext ctx)
 
     validateImageName(imageName);
 
+    validateContainerRuntimeType(runtime);
+
     if (defaultImageUpdate) {
       pullImageFromRemote(containerIdStr, imageName);
     }
@@ -886,6 +912,9 @@ public void launchContainer(ContainerRuntimeContext ctx)
     }
 
     runCommand.setCapabilities(capabilities);
+    if (runtime != null && !runtime.isEmpty()) {
+      runCommand.addRuntime(runtime);
+    }
 
     runCommand.addAllReadWriteMountLocations(containerLogDirs);
     runCommand.addAllReadWriteMountLocations(applicationLocalDirs);

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java

@@ -2481,6 +2481,56 @@ public void testDockerContainerRelaunch()
         dockerCommands.get(counter));
   }
 
+  @Test
+  public void testLaunchContainersWithSpecificDockerRuntime()
+      throws ContainerExecutionException, PrivilegedOperationException,
+      IOException {
+    DockerLinuxContainerRuntime runtime = new DockerLinuxContainerRuntime(
+        mockExecutor, mockCGroupsHandler);
+    runtime.initialize(conf, nmContext);
+
+    env.put(DockerLinuxContainerRuntime
+            .ENV_DOCKER_CONTAINER_DOCKER_RUNTIME, "runc");
+    runtime.launchContainer(builder.build());
+    List<String> dockerCommands = readDockerCommands();
+    Assert.assertEquals(14, dockerCommands.size());
+    Assert.assertEquals("  runtime=runc", dockerCommands.get(11));
+  }
+
+  @Test
+  @SuppressWarnings("unchecked")
+  public void testContainerLaunchWithAllowedRuntimes()
+      throws ContainerExecutionException, IOException,
+      PrivilegedOperationException {
+    DockerLinuxContainerRuntime runtime =
+        new DockerLinuxContainerRuntime(mockExecutor, mockCGroupsHandler);
+    runtime.initialize(conf, nmContext);
+
+    String disallowedRuntime = "runc2";
+
+    try {
+      env.put(DockerLinuxContainerRuntime.ENV_DOCKER_CONTAINER_DOCKER_RUNTIME,
+          disallowedRuntime);
+      runtime.launchContainer(builder.build());
+      Assert.fail("Runtime was expected to be disallowed: " +
+          disallowedRuntime);
+    } catch (ContainerExecutionException e) {
+      LOG.info("Caught expected exception: " + e);
+    }
+
+    String allowedRuntime = "runc";
+    env.put(DockerLinuxContainerRuntime.ENV_DOCKER_CONTAINER_DOCKER_RUNTIME,
+        allowedRuntime);
+    //this should cause no failures.
+
+    runtime.launchContainer(builder.build());
+    List<String> dockerCommands = readDockerCommands();
+
+    //This is the expected docker invocation for this case
+    Assert.assertEquals(14, dockerCommands.size());
+    Assert.assertEquals("  runtime=runc", dockerCommands.get(11));
+  }
+
   private static void verifyStopCommand(List<String> dockerCommands,
       String signal) {
     Assert.assertEquals(4, dockerCommands.size());