Multi-arch patching support

README.md

@@ -1,19 +1,38 @@
 # PolyMC Meta
+
 Scripts to generate jsons and jars that PolyMC will access.
 
-## Deployment
-It is recommended to use Docker to deploy the environment.
-
-- Clone this repo to a server
-- Make sure it's writable by the container later: `chown -R 1337:1337 .`
-- Configure `config/config_local.sh`
-  - The defaults should be fine (apart from committer email and name perhaps)
-- Put your SSH key (which has push access to meta-upstream and meta-polymc) at `config/deploy.key`
-- Pull meta- repos: `bash clone.sh`
-- Customize docker-compose.yaml
-- Run `docker-compose up -d --build`
-- Observe Cron logs using `docker-compose logs -f` (Runs hourly by default)
-- (Optional) Run once to fill caches: `docker-compose run meta update`
-
-For local development you can also use `docker-compose.local.yaml`. By default it uses `UID=1000` and `GID=1000`.
-Make sure it's the same as your host instance.
+## Configuration
+
+1. Clone this repo
+2. If you want to override any properties from `config.sh`, create a file `config/config_local.sh` and write them there
+3. If deploying, put your SSH key in `config/deploy.key` (should have push access to `meta-upstream` and `meta-polymc`)
+4. Pull the meta repos: `bash clone.sh`
+
+## Building
+
+## Local
+
+You can run the scripts locally if you want to quickly test or debug. To do so, you need to install [poetry](https://github.com/python-poetry/poetry):
+
+```bash
+pip install poetry
+```
+
+Then, simply run:
+
+```
+poetry install
+poetry run ./update.sh
+```
+
+## Docker
+
+It is recommended to use Docker if you want to deploy the environment.
+
+1. Customize `docker-compose.yaml`
+2. Run `docker-compose up -d --build`
+   - Observe Cron logs using `docker-compose logs -f` (Runs hourly by default)
+   - (Optional) Run once to fill caches: `docker-compose run meta update`
+
+For local development you can also use `docker-compose.local.yaml`. By default it uses `UID=1000` and `GID=1000`. Make sure it's the same as your host instance.

clone.sh

@@ -12,9 +12,9 @@ fi
 set -x
 
 if [ ! -d "${UPSTREAM_DIR}" ]; then
-    git clone "${UPSTREAM_REPO}" "${UPSTREAM_DIR}"
+    git clone --depth=1 "${UPSTREAM_REPO}" "${UPSTREAM_DIR}"
 fi
 
 if [ ! -d "${PMC_DIR}" ]; then
-    git clone "${PMC_REPO}" "${PMC_DIR}"
+    git clone --depth=1 "${PMC_REPO}" "${PMC_DIR}"
 fi

config.sh

@@ -1,12 +1,16 @@
 export UPSTREAM_DIR=upstream
-export UPSTREAM_REPO=git@github.com:PolyMC/meta-upstream.git
+export UPSTREAM_REPO=https://github.com/PolyMC/meta-upstream.git
 export PMC_DIR=polymc
-export PMC_REPO=git@github.com:PolyMC/meta-polymc.git
-export BRANCH_master=master
-export BRANCH_develop=develop
+export PMC_REPO=https://github.com/PolyMC/meta-polymc.git
+export BRANCH=master
 export DEPLOY_TO_S3=false
-export DEPLOY_TO_FOLDER=true
-export DEPLOY_FOLDER_master=/app/public/v1
-export DEPLOY_FOLDER_develop=/app/public/dev
+export DEPLOY_TO_FOLDER=false
+export DEPLOY_FOLDER=/app/public/v1
 export DEPLOY_FOLDER_USER=http
 export DEPLOY_FOLDER_GROUP=http
+export DEPLOY_TO_GIT=false
+# export GIT_SSH_COMMAND="ssh -i '${BASEDIR}/config/deploy.key'"
+# export GIT_AUTHOR_NAME="Herpington Derpson"
+# export GIT_AUTHOR_EMAIL="herpderp@derpmail.com"
+# export GIT_COMMITTER_NAME="$GIT_AUTHOR_NAME"
+# export GIT_COMMITTER_EMAIL="$GIT_AUTHOR_EMAIL"

config/readme.txt

@@ -5,4 +5,6 @@ These are expected:
 * meta-multimc.key.pub
 * meta-upstream.key
 * meta-upstream.key.pub
-* s3cmd.cfg
+* s3cmd.cfg
+
+You can also create 'config_local.sh' and put any property overrides from 'config.sh'.

generateMojang.py

@@ -4,13 +4,15 @@
 from collections import defaultdict, namedtuple
 from operator import attrgetter
 from pprint import pprint
-from typing import Optional
+import re
+from typing import Dict, List, Optional
+from libPatches import get_lib_patches
 
 from meta.common import ensure_component_dir, polymc_path, upstream_path, static_path
 from meta.common.mojang import VERSION_MANIFEST_FILE, MINECRAFT_COMPONENT, LWJGL3_COMPONENT, LWJGL_COMPONENT, \
     STATIC_OVERRIDES_FILE, VERSIONS_DIR
 from meta.model import MetaVersion, Library, GradleSpecifier, MojangLibraryDownloads, MojangArtifact, Dependency, \
-    MetaPackage, MojangRules
+    MetaPackage, MojangRule, MojangRules, OSRule
 from meta.model.mojang import MojangIndexWrap, MojangIndex, MojangVersion, LegacyOverrideIndex
 
 PMC_DIR = polymc_path()
@@ -56,37 +58,12 @@ def map_log4j_artifact(version):
 }
 
 
-# We want versions that contain natives for all platforms. If there are multiple, pick the latest one
-# LWJGL versions we want
-PASS_VARIANTS = [
-    "fcccb40326b2edc13dffbc49bc6fbafd7ca9b0cd",  # 3.3.1 (2022-05-04 14:41:35+00:00)
-    "ea4973ebc9eadf059f30f0958c89f330898bff51",  # 3.2.2 (2019-07-04 14:41:05+00:00)
-    "8e1f89b96c6f583a0e494949c75115ed13412ba1",  # 3.2.1 (2019-02-13 16:12:08+00:00)
-    "7ed2372097dbd635f5aef3137711141ce91c4ee9",  # 3.1.6 (2018-11-29 13:11:38+00:00)
-    "5a006b7c72a080ac673fff02b259f3127c376655",  # 3.1.2 (2018-06-21 12:57:11+00:00)
-    "a3f254df5a63a0a1635755733022029e8cfae1b3",  # 2.9.4-nightly-20150209 (2016-12-20 14:05:34+00:00)
-    "879be09c0bd0d4bafc2ea4ea3d2ab8607a0d976c",  # 2.9.3 (2015-01-30 11:58:24+00:00)
-    "8d4951d00253dfaa36a0faf1c8be541431861c30",  # 2.9.1 (2014-05-22 14:44:33+00:00)
-    "cf58c9f92fed06cb041a7244c6b4b667e6d544cc",  # 2.9.1-nightly-20131120 (2013-12-06 13:55:34+00:00)
-    "27dcadcba29a1a7127880ca1a77efa9ece866f24",  # 2.9.0 (2013-09-06 12:31:58+00:00)
-]
-
-# LWJGL versions we def. don't want!
-BAD_VARIANTS = [
-    "d986df9598fa2bcf4a5baab5edf044548e66d011",  # 3.2.2 (2021-12-10 03:36:38+00:00) only linux, windows
-    "4b73fccb9e5264c2068bdbc26f9651429abbf21a",  # 3.2.2 (2021-08-25 14:41:57+00:00) only linux, windows
-    "ab463e9ebc6a36abf22f2aa27b219dd372ff5069",  # 3.2.2 (2019-07-19 09:25:47+00:00) only linux, windows
-    "8bde129ef334023c365bd7f57512a4bf5e72a378",  # 3.2.1 (2019-04-18 11:05:19+00:00) only osx, windows
-    "65b2ce1f2b869bf98b8dd7ec0bc6956967d04811",  # 3.1.6 (2019-04-18 11:05:19+00:00) only linux
-    "f04052162b50fa1433f67e1a90bc79466c4ab776",  # 2.9.0 (2013-10-21 16:34:47+00:00) only linux, windows
-    "6442fc475f501fbd0fc4244fd1c38c02d9ebaf7e",  # 2.9.0 (2011-03-30 22:00:00+00:00) fine but newer variant available
-]
-
-
-def add_or_get_bucket(buckets, rules: Optional[MojangRules]) -> MetaVersion:
+def add_or_get_bucket(buckets, lib: Library) -> MetaVersion:
     rule_hash = None
-    if rules:
-        rule_hash = hash(rules.json())
+    # if there are custom rules used to exclude arch-specific patches, use a key of "None"
+    # to continue treating the library as common between all LWJGL versions
+    if lib.rules and not lib.arch_rules:
+            rule_hash = hash(lib.rules.json())
 
     if rule_hash in buckets:
         bucket = buckets[rule_hash]
@@ -204,13 +181,14 @@ def process_single_variant(lwjgl_variant: MetaVersion):
         filtered_libraries = list(filter(lambda l: l.name.artifact not in ["jutils", "jinput"], v.libraries))
         v.libraries = filtered_libraries
     else:
-        raise Exception("LWJGL version not recognized: %s" % v.version)
+        print("LWJGL version not recognized: %s" % v.version)
+        return False
 
     v.volatile = True
     v.order = -1
     good = True
     for lib in v.libraries:
-        if not lib.natives:
+        if not lib.natives or lib.arch_rules:
             continue
         checked_dict = {'linux', 'windows', 'osx'}
         if not checked_dict.issubset(lib.natives.keys()):
@@ -227,8 +205,22 @@ def process_single_variant(lwjgl_variant: MetaVersion):
                     break
     if good:
         v.write(filename)
+        return True
+    else:
+        return False
+
+
+# converts a full library name to a major version name
+# for example, "ca.weblite:java-objc-bridge:1.1.0" becomes "ca.weblite:java-objc-bridge:1"
+def get_major_name(name: str):
+    return re.match(r'.+:\d+', name).group(0)
+
+
+def add_or_append_arch_rule(lib: Library, action: str, arch: str):
+    if lib.arch_rules:
+        lib.arch_rules[action].append(arch)
     else:
-        print("Skipped LWJGL", v.version)
+        lib.arch_rules = {action: [arch]}
 
 
 def main():
@@ -237,6 +229,14 @@ def main():
 
     found_any_lwjgl3 = False
 
+    lib_patches: Dict[str, Dict[str, Library]] = {}
+    # remap patches to a Library indexed by its major version name
+    for k, v in get_lib_patches().items():
+        d = {}
+        for p in v:
+            d[get_major_name(p["name"])] = Library.parse_obj(p)
+        lib_patches[k] = d
+
     for filename in os.listdir(os.path.join(UPSTREAM_DIR, VERSIONS_DIR)):
         input_file = os.path.join(UPSTREAM_DIR, VERSIONS_DIR, filename)
         if not input_file.endswith(".json"):
@@ -252,6 +252,19 @@ def main():
         for lib in v.libraries:
             remove_paths_from_lib(lib)
             specifier = lib.name
+            new_libs: List[Library] = []
+
+            for name, patches in lib_patches.items():
+                lib_name = str(lib.name)
+                patch = patches.get(get_major_name(lib_name))
+                if patch == None:
+                    continue
+                print(f"Patching library {lib_name} for {name}")
+                new_lib = copy.deepcopy(patch)
+                add_or_append_arch_rule(new_lib, "allow", name)
+                new_libs.append(new_lib)
+                add_or_append_arch_rule(lib, "disallow", name)
+
             if specifier.is_lwjgl():
                 rules = None
                 if lib.rules:
@@ -260,7 +273,7 @@ def main():
                 if is_macos_only(rules):
                     print("Candidate library ", specifier, " is only for macOS and is therefore ignored.")
                     continue
-                bucket = add_or_get_bucket(buckets, rules)
+                bucket = add_or_get_bucket(buckets, lib)
                 if specifier.group == "org.lwjgl.lwjgl" and specifier.artifact == "lwjgl":
                     bucket.version = specifier.version
                 if specifier.group == "org.lwjgl" and specifier.artifact == "lwjgl":
@@ -270,6 +283,8 @@ def main():
                 if not bucket.libraries:
                     bucket.libraries = []
                 bucket.libraries.append(lib)
+                if len(new_libs) > 0:
+                    bucket.libraries.append(*new_libs)
                 bucket.release_time = v.release_time
             # FIXME: workaround for insane log4j nonsense from December 2021. Probably needs adjustment.
             elif lib.name.is_log4j():
@@ -294,6 +309,8 @@ def main():
                 ))
             else:
                 libs_minecraft.append(lib)
+                if len(new_libs) > 0:
+                    libs_minecraft.append(*new_libs)
         if len(buckets) == 1:
             for key in buckets:
                 lwjgl = buckets[key]
@@ -357,34 +374,18 @@ def main():
             override.apply_onto_meta_version(v)
         v.write(out_filename)
 
-    for lwjglVersionVariant in lwjglVersionVariants:
-        decided_variant = None
-        passed_variants = 0
-        unknown_variants = 0
-        print("%d variant(s) for LWJGL %s:" % (len(lwjglVersionVariants[lwjglVersionVariant]), lwjglVersionVariant))
-
-        for variant in lwjglVersionVariants[lwjglVersionVariant]:
-            if variant.sha1 in BAD_VARIANTS:
-                print("Variant %s ignored because it's marked as bad." % variant.sha1)
-                continue
-            if variant.sha1 in PASS_VARIANTS:
+    for version, variants in lwjglVersionVariants.items():
+        variants: List[LWJGLEntry]
+        print("%d variant(s) for LWJGL %s:" % (len(variants), version))
+        success = False
+        # try all variants until one works, starting from newest
+        for variant in sorted(variants, key=lambda v: v.version.release_time, reverse=True):
+            if process_single_variant(variant.version):
                 print("Variant %s accepted." % variant.sha1)
-                decided_variant = variant
-                passed_variants += 1
-                continue
-            # print natives classifiers to decide which variant to use
-            n = [x.natives.keys() for x in variant.version.libraries if x.natives is not None]
-            print(n)
-
-            print(f"    \"{variant.sha1}\",  # {lwjglVersionVariant} ({variant.version.release_time})")
-            unknown_variants += 1
-        print("")
-
-        if decided_variant and passed_variants == 1 and unknown_variants == 0:
-            process_single_variant(decided_variant.version)
-        else:
-            raise Exception("No variant decided for version %s out of %d possible ones and %d unknown ones." % (
-                lwjglVersionVariant, passed_variants, unknown_variants))
+                success = True
+                break
+        if not success:
+            raise Exception(f"No variant passed for version {version}.")
 
     lwjgl_package = MetaPackage(uid=LWJGL_COMPONENT, name='LWJGL 2')
     lwjgl_package.recommended = ['2.9.4-nightly-20150209']