NIFI-4942 [WIP] Added skeleton for secure hash handling in encrypt-co…

…nfig toolkit. Added test resource for Python scrypt implementation/verifier. Added unit tests.

NIFI-4942 [WIP] More unit tests passing.

NIFI-4942 All unit tests pass and test artifacts are cleaned up.

NIFI-4942 Added RAT exclusions.

NIFI-4942 Added Scrypt hash format checker. Added unit tests.

NIFI-4942 Added NiFi hash format checker. Added unit tests.

NIFI-4942 Added check for simultaneous use of -z/-y. Added logic to check hashed password/key. Added logic to retrieve secure hash from file to compare. Added unit tests (125/125).

NIFI-4942 Added new ExitCode. Added logic to return current hash params in JSON for Ambari to consume. Fixed typos in error messages. Added unit tests (129/129).

NIFI-4942 Added Scrypt hash format verification for hash check. Added unit tests.

NIFI-4942 Fixed RAT checks.

Signed-off-by: Yolanda Davis <ymdavis@apache.org>

This closes #2628

nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/util/crypto/scrypt/Scrypt.java

@@ -24,6 +24,7 @@
 import java.security.SecureRandom;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.regex.Pattern;
 import javax.crypto.Mac;
 import javax.crypto.spec.SecretKeySpec;
 import org.apache.commons.codec.binary.Base64;
@@ -51,6 +52,8 @@ public class Scrypt {
 
     private static final int DEFAULT_SALT_LENGTH = 16;
 
+    private static final Pattern SCRYPT_PATTERN = Pattern.compile("^\\$\\w{2}\\$\\w{5,}\\$[\\w\\/\\=\\+]{11,64}\\$[\\w\\/\\=\\+]{1,256}$");
+
     /**
      * Hash the supplied plaintext password and generate output in the format described
      * below:
@@ -158,12 +161,12 @@ public static boolean check(String password, String hashed) {
                 throw new IllegalArgumentException("Hash cannot be empty");
             }
 
-            String[] parts = hashed.split("\\$");
-
-            if (parts.length != 5 || !parts[1].equals("s0")) {
+            if (!verifyHashFormat(hashed)) {
                 throw new IllegalArgumentException("Hash is not properly formatted");
             }
 
+            String[] parts = hashed.split("\\$");
+
             List<Integer> splitParams = parseParameters(parts[2]);
             int n = splitParams.get(0);
             int r = splitParams.get(1);
@@ -188,6 +191,38 @@ public static boolean check(String password, String hashed) {
         }
     }
 
+    /**
+     * Returns true if the provided hash is a valid scrypt hash. Expected format:
+     * <p>
+     * {@code $s0$40801$ABCDEFGHIJKLMNOPQRSTUQ$hxU5g0eH6sRkBqcsiApI8jxvKRT+2QMCenV0GToiMQ8}
+     * <p>
+     * Components:
+     * <p>
+     * s0 -- version. Currently only "s0" is supported
+     * 40801 -- hex-encoded N, r, p parameters. {@see Scrypt#encodeParams()} for format
+     * ABCDEFGHIJKLMNOPQRSTUQ -- Base64-encoded (URL-safe, no padding) salt value.
+     * By default, 22 characters (16 bytes) but can be an arbitrary length between 11 and 64 characters (8 - 48 bytes) of random salt data
+     * hxU5g0eH6sRkBqcsiApI8jxvKRT+2QMCenV0GToiMQ8 -- the Base64-encoded (URL-safe, no padding)
+     * resulting hash component. By default, 43 characters (32 bytes) but can be an arbitrary length between 1 and MAX (depends on implementation, see RFC 7914)
+     *
+     * @param hash the hash to verify
+     * @return true if the format is acceptable
+     * @see Scrypt#formatSalt(byte[], int, int, int)
+     */
+    public static boolean verifyHashFormat(String hash) {
+        if (StringUtils.isBlank(hash)) {
+            return false;
+        }
+
+        // Currently, only version s0 is supported
+        if (!hash.startsWith("$s0$")) {
+            return false;
+        }
+
+        // Check against the pattern
+        return SCRYPT_PATTERN.matcher(hash).matches();
+    }
+
     /**
      * Parses the individual values from the encoded params value in the modified-mcrypt format for the salt & hash.
      * <p/>
@@ -285,7 +320,7 @@ protected static byte[] deriveScryptKey(byte[] password, byte[] salt, int n, int
             logger.warn("An empty salt was used for scrypt key derivation");
 //            throw new IllegalArgumentException("Salt cannot be empty");
             // as the Exception is not being thrown, prevent NPE if salt is null by setting it to empty array
-            if( salt == null ) salt = new byte[]{};
+            if (salt == null) salt = new byte[]{};
         }
 
         if (saltLength < 8 || saltLength > 32) {

nifi-commons/nifi-security-utils/src/test/groovy/org/apache/nifi/security/util/scrypt/ScryptGroovyTest.groovy

@@ -397,4 +397,60 @@ class ScryptGroovyTest {
             assert msg =~ "Hash cannot be empty|Hash is not properly formatted"
         }
     }
+
+    @Test
+    void testVerifyHashFormatShouldDetectValidHash() throws Exception {
+        // Arrange
+        final def VALID_HASHES = [
+                "\$s0\$40801\$AAAAAAAAAAAAAAAAAAAAAA\$gLSh7ChbHdOIMvZ74XGjV6qF65d9qvQ8n75FeGnM8YM",
+                "\$s0\$40801\$ABCDEFGHIJKLMNOPQRSTUQ\$hxU5g0eH6sRkBqcsiApI8jxvKRT+2QMCenV0GToiMQ8",
+                "\$s0\$40801\$eO+UUcKYL2gnpD51QCc+gnywQ7Eg9tZeLMlf0XXr2zc\$99aTTB39TJo69aZCONQmRdyWOgYsDi+1MI+8D0EgMNM",
+                "\$s0\$40801\$AAAAAAAAAAAAAAAAAAAAAA\$Gk7K9YmlsWbd8FS7e4RKVWnkg9vlsqYnlD593pJ71gg",
+                "\$s0\$40801\$ABCDEFGHIJKLMNOPQRSTUQ\$Ri78VZbrp2cCVmGh2a9Nbfdov8LPnFb49MYyzPCaXmE",
+                "\$s0\$40801\$eO+UUcKYL2gnpD51QCc+gnywQ7Eg9tZeLMlf0XXr2zc\$rZIrP2qdIY7LN4CZAMgbCzl3YhXz6WhaNyXJXqFIjaI",
+                "\$s0\$40801\$AAAAAAAAAAAAAAAAAAAAAA\$GxH68bGykmPDZ6gaPIGOONOT2omlZ7cd0xlcZ9UsY/0",
+                "\$s0\$40801\$ABCDEFGHIJKLMNOPQRSTUQ\$KLGZjWlo59sbCbtmTg5b4k0Nu+biWZRRzhPhN7K5kkI",
+                "\$s0\$40801\$eO+UUcKYL2gnpD51QCc+gnywQ7Eg9tZeLMlf0XXr2zc\$6Ql6Efd2ac44ERoV31CL3Q0J3LffNZKN4elyMHux99Y",
+                // Uncommon but technically valid
+                "\$s0\$F0801\$AAAAAAAAAAA\$A",
+                "\$s0\$40801\$ABCDEFGHIJKLMNOPABCDEFGHIJKLMNOPABCDEFGHIJKLMNOPABCDEFGHIJKLMNOP\$A",
+                "\$s0\$40801\$ABCDEFGHIJKLMNOPABCDEFGHIJKLMNOPABCDEFGHIJKLMNOPABCDEFGHIJKLMNOP\$ABCDEFGHIJKLMNOPABCDEFGHIJKLMNOPABCDEFGHIJKLMNOPABCDEFGHIJKLMNOP",
+                "\$s0\$40801\$ABCDEFGHIJKLMNOPABCDEFGHIJKLMNOPABCDEFGHIJKLMNOPABCDEFGHIJKLMNOP\$ABCDEFGHIJKLMNOPABCDEFGHIJKLMNOPABCDEFGHIJKLMNOPABCDEFGHIJKLMNOPABCDEFGHIJKLMNOPABCDEFGHIJKLMNOPABCDEFGHIJKLMNOPABCDEFGHIJKLMNOP",
+                "\$s0\$F0801\$AAAAAAAAAAA\$A",
+                "\$s0\$F0801\$AAAAAAAAAAA\$A",
+                "\$s0\$F0801\$AAAAAAAAAAA\$A",
+                "\$s0\$F0801\$AAAAAAAAAAA\$A",
+                "\$s0\$F0801\$AAAAAAAAAAA\$A",
+        ]
+
+        // Act
+        VALID_HASHES.each { String validHash ->
+            logger.info("Using hash: ${validHash}")
+
+            boolean isValidHash = Scrypt.verifyHashFormat(validHash)
+            logger.info("Hash is valid: ${isValidHash}")
+
+            // Assert
+            assert isValidHash
+        }
+    }
+
+    @Test
+    void testVerifyHashFormatShouldDetectInvalidHash() throws Exception {
+        // Arrange
+
+        // Even though the spec allows for empty salts, the JCE does not, so extend enforcement of that to the user boundary
+        final def INVALID_HASHES = ['', null, '$s0$a0801$', '$s0$a0801$abcdefghijklmnopqrstuv$']
+
+        // Act
+        INVALID_HASHES.each { String invalidHash ->
+            logger.info("Using hash: ${invalidHash}")
+
+            boolean isValidHash = Scrypt.verifyHashFormat(invalidHash)
+            logger.info("Hash is valid: ${isValidHash}")
+
+            // Assert
+            assert !isValidHash
+        }
+    }
 }

nifi-toolkit/nifi-toolkit-encrypt-config/pom.xml

@@ -13,7 +13,8 @@
   See the License for the specific language governing permissions and
   limitations under the License.
 -->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <parent>
         <groupId>org.apache.nifi</groupId>
         <artifactId>nifi-toolkit</artifactId>
@@ -162,7 +163,19 @@
                     </execution>
                 </executions>
             </plugin>
+            <plugin>
+                <groupId>org.apache.rat</groupId>
+                <artifactId>apache-rat-plugin</artifactId>
+                <configuration>
+                    <excludes combine.children="append">
+                        <exclude>src/test/resources/scrypt.py</exclude>
+                        <exclude>src/test/resources/secure_hash.key</exclude>
+                        <exclude>src/test/resources/secure_hash_128.key</exclude>
+                    </excludes>
+                </configuration>
+            </plugin>
         </plugins>
+
     </build>
 
 </project>