[hadoop] LIHADOOP-76132 : Evolve the BundledTokenIdentifier to include SPIFFE tokens (apache#458)

Author: Raymond Lam

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/BundledTokenIdentifier.java

@@ -20,4 +20,7 @@ public interface BundledTokenIdentifier {
 
   // The bundled information for additional tokens.
   Token[] getInnerTokens();
+
+  // The SPIFFE tokens
+  Token[] getSPIFFETokens();
 }

hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/BundledDelegationTokenIdentifier.java

@@ -23,7 +23,11 @@ public class BundledDelegationTokenIdentifier extends DelegationTokenIdentifier
     implements BundledTokenIdentifier {
 
   // The current VERSION of the token.
-  private static final byte VERSION = 0;
+  private static final byte VERSION = 1;
+
+  // The original serde implementation of this identifier was un-versioned.
+  // This denotes the default version for un-versioned tokens.
+  private static final byte BASE_VERSION = 0;
 
   // The designated main password.
   // This will be empty (byte[0]) if deserialization fails to read any content
@@ -45,22 +49,28 @@ public class BundledDelegationTokenIdentifier extends DelegationTokenIdentifier
   // content for the tokenVersion.
   private byte tokenVersion;
 
-  // The original serde implementation of this identifier was un-versioned.
-  // This denotes the default version for un-versioned tokens.
-  private static final byte BASE_VERSION = 0;
+  // The list of SPIFFE tokens.
+  private Token[] spiffeTokens;
 
   public BundledDelegationTokenIdentifier() {
     super();
   }
 
   public BundledDelegationTokenIdentifier(Token<?> mainToken,
       Token<AbstractDelegationTokenIdentifier>[] tokens) throws IOException {
+    this(mainToken, tokens, new Token[0]);
+  }
+
+  public BundledDelegationTokenIdentifier(Token<?> mainToken,
+      Token<AbstractDelegationTokenIdentifier>[] tokens,
+      Token<SPIFFEDelegationTokenIdentifier>[] spiffeTokens) throws IOException {
     this((AbstractDelegationTokenIdentifier) mainToken.decodeIdentifier(),
-        mainToken.getPassword(), tokens);
+        mainToken.getPassword(), tokens, spiffeTokens);
   }
 
   public BundledDelegationTokenIdentifier(AbstractDelegationTokenIdentifier mainTokenId,
-      byte[] mainPassword, Token<AbstractDelegationTokenIdentifier>[] tokens) throws IOException {
+      byte[] mainPassword, Token<AbstractDelegationTokenIdentifier>[] tokens,
+      Token<SPIFFEDelegationTokenIdentifier>[] spiffeTokens) throws IOException {
     super(mainTokenId.getOwner(), mainTokenId.getRenewer(), mainTokenId.getRealUser());
     // Set the attributes from the main tokenIdentifier, which are expected by the
     // DelegationTokenIdentifier class.
@@ -81,6 +91,12 @@ public BundledDelegationTokenIdentifier(AbstractDelegationTokenIdentifier mainTo
     this.innerTokens = tokens;
 
     this.tokenVersion = VERSION;
+
+    if (spiffeTokens == null) {
+      spiffeTokens = new Token[0];
+    }
+
+    this.spiffeTokens = spiffeTokens;
   }
 
   @Override
@@ -98,6 +114,11 @@ public void write(DataOutput out) throws IOException {
     }
 
     out.writeByte(tokenVersion);
+
+    WritableUtils.writeVInt(out, this.spiffeTokens.length);
+    for (Token spiffeToken : this.spiffeTokens) {
+      spiffeToken.write(out);
+    }
   }
 
   @Override
@@ -138,6 +159,18 @@ public void readFields(DataInput in) throws IOException {
       this.innerTokens = new Token[0];
       this.tokenVersion = BASE_VERSION;
     }
+
+    if (this.tokenVersion < 1) {
+      this.spiffeTokens = new Token[0];
+    } else {
+      int spiffeTokenLength = WritableUtils.readVInt(in);
+      this.spiffeTokens = new Token[spiffeTokenLength];
+      for (int i = 0; i < spiffeTokenLength; ++i) {
+        Token spiffeToken = new Token();
+        spiffeToken.readFields(in);
+        this.spiffeTokens[i] = spiffeToken;
+      }
+    }
   }
 
   @Override
@@ -159,4 +192,9 @@ public byte[] getMainPassword() {
   public Token[] getInnerTokens() {
     return this.innerTokens;
   }
+
+  @Override
+  public Token[] getSPIFFETokens() {
+    return this.spiffeTokens;
+  }
 }

hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/security/token/delegation/TestBundledDelegationTokenIdentifier.java

@@ -15,28 +15,10 @@
 public class TestBundledDelegationTokenIdentifier {
   @Test
   public void testBundledTokenSerialization() throws Exception {
-    // Create tokenIdentifier with bundled tokens and write to stream.
-    BundledDelegationTokenIdentifier bundledTokenId1 = createTestBundledTokenId();
-    DataOutputBuffer out = new DataOutputBuffer();
-    bundledTokenId1.write(out);
-
-    // Read as bundledTokenIdentifier from stream.
-    BundledDelegationTokenIdentifier bundledTokenId2 = new BundledDelegationTokenIdentifier();
-    readDelegationTokenIdentifier(bundledTokenId2, out);
-
-    // Validate that DelegationTokenInformation attributes are deserialized correctly.
-    assertEquals(bundledTokenId1, bundledTokenId2);
-
-    Token[] innerTokens1 = bundledTokenId1.getInnerTokens();
-    Token[] innerTokens2 = bundledTokenId2.getInnerTokens();
-
-    // Validate that bundled token information was deserialized correctly. 
-    assertEquals(innerTokens1.length, innerTokens2.length);
-    for (int i = 0; i < innerTokens1.length; i++) {
-      assertEquals(innerTokens1[i].decodeIdentifier(), innerTokens2[i].decodeIdentifier());
-      assertArrayEquals(innerTokens1[i].getPassword(), innerTokens2[i].getPassword());
-      assertEquals(innerTokens1[i].getService(), innerTokens2[i].getService());
-    }
+    assertBundledDelegationTokensEquals(3, 0);
+    assertBundledDelegationTokensEquals(0, 1);
+    assertBundledDelegationTokensEquals(3, 1);
+    assertBundledDelegationTokensEquals(3, 3);
   }
 
   @Test
@@ -78,14 +60,29 @@ public void testBundledTokenForwardsCompatible() throws Exception {
   }
 
   private BundledDelegationTokenIdentifier createTestBundledTokenId() throws IOException {
-    int tokensLength = 3;
-    Token<AbstractDelegationTokenIdentifier>[] allTokens = new Token[tokensLength];
-    for (int i = 0; i < tokensLength; i++) {
+    return createTestBundledTokenId(3, 0);
+  }
+
+  private BundledDelegationTokenIdentifier createTestBundledTokenId(
+      int hdfsTokensLength,
+      int spiffeTokenLength) throws IOException {
+    Token<AbstractDelegationTokenIdentifier> mainToken = createTestToken(1);
+
+    Token<AbstractDelegationTokenIdentifier>[] allTokens = new Token[hdfsTokensLength];
+    for (int i = 0; i < hdfsTokensLength; i++) {
       Token<AbstractDelegationTokenIdentifier> token1 = createTestToken(i + 1);
       allTokens[i] = token1;
     }
 
-    return new BundledDelegationTokenIdentifier(allTokens[0], allTokens);
+    Token<SPIFFEDelegationTokenIdentifier>[] spiffeTokens = new Token[spiffeTokenLength];
+    for (int i = 0; i < spiffeTokenLength; ++i) {
+      Token<SPIFFEDelegationTokenIdentifier> spiffeToken =
+          createTestSPIFFEToken(i + 1);
+      spiffeTokens[i] = spiffeToken;
+    }
+
+    return new BundledDelegationTokenIdentifier(
+        mainToken, allTokens, spiffeTokens);
   }
 
   private Token<AbstractDelegationTokenIdentifier> createTestToken(int id) {
@@ -102,10 +99,54 @@ private Token<AbstractDelegationTokenIdentifier> createTestToken(int id) {
         new Text("HDFS_DELEGATION_TOKEN"), new Text("service_" + id));
   }
 
+  private Token<SPIFFEDelegationTokenIdentifier> createTestSPIFFEToken(int id) {
+    SPIFFEDelegationTokenIdentifier identifier =
+        new SPIFFEDelegationTokenIdentifier("header_" + id, "payload_" + id);
+    return new Token<>(identifier.getBytes(), ("password_" + id).getBytes(),
+        SPIFFEDelegationTokenIdentifier.SPIFFE_DELEGATION_KIND, null);
+  }
+
   private void readDelegationTokenIdentifier(DelegationTokenIdentifier tokenId,
       DataOutputBuffer out) throws IOException {
     DataInputBuffer in = new DataInputBuffer();
     in.reset(out.getData(), out.getLength());
     tokenId.readFields(in);
   }
+
+  private void assertBundledDelegationTokensEquals(
+      int expectedNumHdfsTokens, int expectedNumSpiffeTokens) throws IOException {
+
+    BundledDelegationTokenIdentifier bundledTokenId1 =
+        createTestBundledTokenId(expectedNumHdfsTokens, expectedNumSpiffeTokens);
+    DataOutputBuffer out = new DataOutputBuffer();
+    bundledTokenId1.write(out);
+
+    // Read as bundledTokenIdentifier from stream.
+    BundledDelegationTokenIdentifier bundledTokenId2 = new BundledDelegationTokenIdentifier();
+    readDelegationTokenIdentifier(bundledTokenId2, out);
+
+    // Validate that DelegationTokenInformation attributes are deserialized correctly.
+    assertEquals(bundledTokenId1, bundledTokenId2);
+
+    Token[] innerTokens1 = bundledTokenId1.getInnerTokens();
+    Token[] innerTokens2 = bundledTokenId2.getInnerTokens();
+
+    // Validate that bundled token information was deserialized correctly.
+    assertEquals(innerTokens1.length, innerTokens2.length);
+    for (int i = 0; i < innerTokens1.length; i++) {
+      assertEquals(innerTokens1[i].getKind(), innerTokens2[i].getKind());
+      assertEquals(innerTokens1[i].decodeIdentifier(), innerTokens2[i].decodeIdentifier());
+      assertArrayEquals(innerTokens1[i].getPassword(), innerTokens2[i].getPassword());
+      assertEquals(innerTokens1[i].getService(), innerTokens2[i].getService());
+    }
+
+    Token[] spiffeTokens1 = bundledTokenId1.getSPIFFETokens();
+    Token[] spiffeTokens2 = bundledTokenId2.getSPIFFETokens();
+
+    assertEquals(spiffeTokens1.length, spiffeTokens2.length);
+    for (int i = 0; i < spiffeTokens1.length; ++i) {
+      assertEquals(spiffeTokens1[i], spiffeTokens2[i]);
+      assertEquals(spiffeTokens1[i].decodeIdentifier(), spiffeTokens2[i].decodeIdentifier());
+    }
+  }
 }