nix: make fetching Gradle dependencies more generic

Before what we did was essentially guess what files might exist for any
given package. This approach mostly works, but not entirely.
This is especially problematic when dealing with weird edge case
packages like `react-native`, which you can read about here:

react-native-community/discussions-and-proposals#508
https://github.com/react-native-community/discussions-and-proposals/blob/4a06fc64/proposals/0508-out-of-npm-artifacts.md#the-react-native-android-archive

In order to avoid as much the guessing aspect of fetching Gradle
dependencies we are using both HTML listsings of files and
`artifact-metadata.json` files that exist for more recent packages.

This way we can avoid having to add special edge cases that have been
found out when working on React Native 72 upgrade in:
#17062

Signed-off-by: Jakub Sokołowski <jakub@status.im>

nix/deps/gradle/url2json.sh

@@ -14,6 +14,10 @@ declare -a REPOS=(
   "https://jitpack.io"
 )
 
+# Ignore files which filenames contain these elements:
+FILENAMES_BLACKLIST='-(javadoc|sources|src|tests|adapters|bin|api)'
+FILETYPES_BLACKLIST='\.(md5|sha|pom|json|zip|module)$'
+
 function nix_prefetch() {
     nix store prefetch-file --json "${1}" 2>/dev/null
 }
@@ -39,6 +43,47 @@ function pom_has_nodeps_jar() {
         >/dev/null 2>&1 
 }
 
+function guess_pkg_files() {
+    # Some deps are just POMs, in which case there is no JAR to fetch.
+    [[ "${OBJ_TYPE}" == "" ]]        && echo "${PKG_NAME}.jar"
+    [[ "${OBJ_TYPE}" == "jar" ]]     && echo "${PKG_NAME}.jar"
+    [[ "${OBJ_TYPE}" == "bundle" ]]  && echo "${PKG_NAME}.jar"
+    [[ "${OBJ_TYPE}" =~ aar* ]]      && echo "${PKG_NAME}.aar"
+    [[ "${OBJ_TYPE}" == "aar.asc" ]] && echo "${PKG_NAME}.${OBJ_TYPE}"
+    pom_has_nodeps_jar "${POM_PATH}" && echo "${PKG_NAME}-nodeps.jar"
+}
+
+function get_pkg_files() {
+    REPO_URL="${1}"
+    PKG_PATH="${2}"
+    PKG_NAME="${3}"
+    # Google Maven repo doesn't have normal HTML directory listing.
+    if [[ "${REPO_URL}" == "https://dl.google.com/dl/android/maven2" ]]; then
+        FOUND=$(curl --fail -s "${REPO_URL}/${PKG_PATH}/artifact-metadata.json")
+        # Some older packages do not have artifacts-metadata.json.
+        if [[ "$?" -eq 0 ]]; then
+            FOUND=$(echo "${FOUND}" | jq -r '.artifacts[].name')
+        else
+            FOUND=''
+        fi
+    else
+        FOUND=$(
+            curl -s "${REPO_URL}/${PKG_PATH}/" \
+                | htmlq a -a href \
+                | grep -e "^${PKG_NAME}"
+        )
+    fi
+    if [[ "${FOUND}" == '' ]]; then
+        guess_pkg_files
+    else
+        # Filter out files we don't actually need for builds.
+        echo "${FOUND}" \
+            | grep -v -E \
+                -e "${FILENAMES_BLACKLIST}" \
+                -e "${FILETYPES_BLACKLIST}"
+    fi
+}
+
 function fetch_and_template_file() {
     local FILENAME="${1}"
     local OBJ_URL OBJ_NIX_FETCH_OUT OBJ_NAME OBJ_PATH
@@ -58,10 +103,6 @@ function fetch_and_template_file() {
       }"
 }
 
-function fetch_and_template_file_no_fail() {
-    fetch_and_template_file "${1}" 2>/dev/null || true
-}
-
 if [[ -z "${1}" ]]; then
     echo "Required POM URL argument not given!" >&2
     exit 1
@@ -77,7 +118,7 @@ echo -en "${CLR} - Nix entry for: ${1##*/}\r" >&2
 REPO_URL=$(match_repo_url "${PKG_URL_NO_EXT}")
 
 if [[ -z "${REPO_URL}" ]]; then
-    echo " ! Repo URL not found: %s" "${REPO_URL}" >&2
+    echo " ! Repo URL not found for: ${POM_URL}" >&2
     exit 1
 fi
 # Get the relative path without full URL
@@ -114,12 +155,8 @@ echo -ne "
         \"sha256\": \"${POM_SHA256}\"
       }"
 
-# Some deps are just POMs, in which case there is no JAR to fetch.
-[[ "${OBJ_TYPE}" == "" ]]        && fetch_and_template_file_no_fail "${PKG_NAME}.jar"
-[[ "${OBJ_TYPE}" == "jar" ]]     && fetch_and_template_file "${PKG_NAME}.jar"
-[[ "${OBJ_TYPE}" == "bundle" ]]  && fetch_and_template_file "${PKG_NAME}.jar"
-[[ "${OBJ_TYPE}" =~ aar* ]]      && fetch_and_template_file "${PKG_NAME}.aar"
-[[ "${OBJ_TYPE}" == "aar.asc" ]] && fetch_and_template_file "${PKG_NAME}.${OBJ_TYPE}"
-pom_has_nodeps_jar "${POM_PATH}" && fetch_and_template_file "${PKG_NAME}-nodeps.jar"
+for FILE in $(get_pkg_files "${REPO_URL}" "${PKG_PATH}" "${PKG_NAME}"); do
+    fetch_and_template_file "${FILE}"
+done
 
 echo -e '\n    }\n  },'

nix/shells.nix

@@ -38,7 +38,7 @@ let
 
     # for running gradle by hand
     gradle = mkShell {
-      buildInputs = with pkgs; [ gradle maven goMavenResolver ];
+      buildInputs = with pkgs; [ gradle maven goMavenResolver htmlq ];
       shellHook = ''
         export STATUS_GO_ANDROID_LIBDIR="DUMMY"
         export STATUS_NIX_MAVEN_REPO="${pkgs.deps.gradle}"