tmp

Author: Remi-Gau

Dockerfile-core.template renamed to Dockerfile-core.mustache

@@ -1,25 +1,29 @@
 FROM bids/base_validator
 
 # Update system
-RUN apt-get -qq update && apt-get -qq install -y \
-    unzip \
-    xorg \
-    wget && \
+RUN apt-get -qq update -qq && \
+    apt-get -qq install -qq -y --no-install-recommends \
+        unzip \
+        xorg \
+        wget && \
+    apt-get clean && \
     rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
 # Install MATLAB MCR
-ENV MATLAB_VERSION %%MATLAB_VERSION%%
+ENV MATLAB_VERSION {{ MATLAB_VERSION }}
 RUN mkdir /opt/mcr_install && \
     mkdir /opt/mcr && \
-    wget --quiet -P /opt/mcr_install %%MCR_LINK%% && \
+    wget --quiet -P /opt/mcr_install {{ MCR_LINK }} && \
     unzip -q /opt/mcr_install/*${MATLAB_VERSION}*.zip -d /opt/mcr_install && \
     cd /opt/mcr_install && mkdir save && \
+{{#core_only}}
     for f in $(grep -E '(xml|enc)$' productdata/1000.txt) ; do  cp --parents archives/$f save/ ; done && \
     for f in $(grep -E '(xml|enc)$' productdata/35000.txt) ; do  cp --parents archives/$f save/ ; done && \
     for f in $(grep -E '(xml|enc)$' productdata/35010.txt) ; do  cp --parents archives/$f save/ ; done && \
     rm -rf archives && mv save/archives . && rmdir save && \
+{{/core_only}}
     /opt/mcr_install/install -destinationFolder /opt/mcr -agreeToLicense yes -mode silent && \
-    rm -rf /opt/mcr_install /tmp/* && \
+{{#core_only}}
     rm -rf /opt/mcr/*/cefclient && \
     rm -rf /opt/mcr/*/mcr/toolbox/matlab/maps && \
     rm -rf /opt/mcr/*/java/jarext && \
@@ -34,9 +38,11 @@ RUN mkdir /opt/mcr_install && \
     rm -rf /opt/mcr/*/bin/glnxa64/libQt* && \
     rm -rf /opt/mcr/*/bin/glnxa64/qtwebengine && \
     rm -rf /opt/mcr/*/bin/glnxa64/cef_resources
+{{/core_only}}
+    rm -rf /opt/mcr_install /tmp/*
 
 # Configure environment
-ENV MCR_VERSION %%MCR_VERSION%%
+ENV MCR_VERSION {{ MCR_VERSION }}
 ENV LD_LIBRARY_PATH /opt/mcr/${MCR_VERSION}/runtime/glnxa64:/opt/mcr/${MCR_VERSION}/bin/glnxa64:/opt/mcr/${MCR_VERSION}/sys/os/glnxa64:/opt/mcr/${MCR_VERSION}/sys/opengl/lib/glnxa64
 ENV MCR_INHIBIT_CTF_LOCK 1
 ENV MCR_HOME /opt/mcr/${MCR_VERSION}

Dockerfile-full.template renamed to Dockerfile-full.mustache

@@ -10,16 +10,16 @@ RUN apt-get -qq update -qq && \
     rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
 # Install MATLAB MCR
-ENV MATLAB_VERSION %%MATLAB_VERSION%%
+ENV MATLAB_VERSION {{ MATLAB_VERSION }}
 RUN mkdir /opt/mcr_install && \
     mkdir /opt/mcr && \
-    wget --quiet -P /opt/mcr_install %%MCR_LINK%% && \
-    unzip -q /opt/mcr_install/*.zip -d /opt/mcr_install && \
+    wget --quiet -P /opt/mcr_install {{ MCR_LINK }} && \
+    unzip -q /opt/mcr_install/*${MATLAB_VERSION}*.zip -d /opt/mcr_install && \
     /opt/mcr_install/install -destinationFolder /opt/mcr -agreeToLicense yes -mode silent && \
     rm -rf /opt/mcr_install /tmp/*
 
 # Configure environment
-ENV MCR_VERSION %%MCR_VERSION%%
+ENV MCR_VERSION {{ MCR_VERSION }}
 ENV LD_LIBRARY_PATH /opt/mcr/${MCR_VERSION}/runtime/glnxa64:/opt/mcr/${MCR_VERSION}/bin/glnxa64:/opt/mcr/${MCR_VERSION}/sys/os/glnxa64:/opt/mcr/${MCR_VERSION}/sys/opengl/lib/glnxa64
 ENV MCR_INHIBIT_CTF_LOCK 1
 

requirements.txt

@@ -0,0 +1,4 @@
+rich
+beautifulsoup4
+packaging
+chevron

update.py

@@ -23,91 +23,150 @@
 
 Update.py should be run often enough to catch individual Matlab release updates.
 """
+import pathlib
 import re
 from subprocess import DEVNULL
 from subprocess import run
 from urllib import request
 
+import chevron
 from bs4 import BeautifulSoup
 from packaging import version
+from rich import print
+
 
 REL_URL = "https://www.mathworks.com/products/compiler/matlab-runtime.html"
 VER_LIMIT = "9.3"  # release URLs get weird before that..
+DRY_RUN = True
+
+variants = [("Dockerfile-full.mustache", ""), ("Dockerfile-core.mustache", "-core")]
 
 
 def call(cmd, split=True):
     if split:
         cmd = cmd.split()
+    print(f"[green]{' '.join(cmd)}[/green]")
+    if DRY_RUN:
+        return True
     process = run(cmd, stdout=DEVNULL, stderr=DEVNULL)
     return process.returncode == 0
 
 
-with request.urlopen(REL_URL) as res:
-    if res.status != 200:
-        raise RuntimeError("Could not open matlab release URL")
-    html = res.read()
-
-soup = BeautifulSoup(html, "html.parser")
-ver_re = re.compile(r"(R2\d{3}.) \((\d\.\d)\)")
-rel_re = re.compile(r"Release/(\d+)/")
-
-dockers = []
-for row in soup.find_all("table")[0].find_all("tr"):
-    tds = row.find_all("td")
-    if len(tds) >= 4:
-        name = tds[0].text
-        match = ver_re.match(name)
-        if not match:
-            continue
-        mcr_name, mcr_ver = match.groups()
-        if version.parse(mcr_ver) <= version.parse(VER_LIMIT):
-            continue
-        try:
-            link = tds[2].a.get("href")
-        except (KeyError, ValueError):
-            raise RuntimeError("Error parsing matlab release page")
-        if "glnxa64" not in link:
-            raise RuntimeError("Error parsing matlab release page link")
-        if match := rel_re.search(link):
-            mcr_ver = f"{mcr_ver}.{match.groups()[0]}"
-        dockers.append((mcr_name, mcr_ver, link))
+def add_dockerfile_to_branch(new_tags, docker):
 
-
-variants = [("Dockerfile-full.template", ""), ("Dockerfile-core.template", "-core")]
-new_tags = []
-
-for docker in dockers:
     mcr_name, mcr_ver, link = docker
+
     if len(mcr_ver.split(".")) == 2:
         mcr_ver = f"{mcr_ver}.0"
     mcr_ver_maj = ".".join(mcr_ver.split(".")[:2])
     mcr_ver_dir = f'v{mcr_ver_maj.replace(".", "")}'
+
+    print(f"\n[blue]{mcr_name}[/blue]")
+
     if not call(f"git checkout {mcr_name}"):
         call(f"git checkout -b {mcr_name}")
+
     for (template, suffix) in variants:
+
         tag = f"{mcr_ver}{suffix}"
-        if call(f"git rev-parse --verify {tag}"):
-            print(f"Skipping {mcr_name}/{tag}, already present")
+
+        if not DRY_RUN and call(f"git rev-parse --verify {tag}"):
+            print(f"[red]Skipping {mcr_name}-{tag}, already present[/red]")
             continue
-        print(f"Adding {mcr_name}/{tag}")
+        print(f"\n[blue]Adding {mcr_name}-{tag}[/blue]")
+
         if not call("git merge master"):
             raise RuntimeError("Merging master failed, will not continue")
-        with open(template) as f:
-            lines = f.read()
-            lines = lines.replace("%%MATLAB_VERSION%%", mcr_name)
-            lines = lines.replace("%%MCR_VERSION%%", mcr_ver_dir)
-            lines = lines.replace("%%MCR_LINK%%", link)
+
+        lines = pathlib.Path(template).read_text()
+        lines = lines.replace("%%MATLAB_VERSION%%", mcr_name)
+        lines = lines.replace("%%MCR_VERSION%%", mcr_ver_dir)
+        lines = lines.replace("%%MCR_LINK%%", link)
+        if not DRY_RUN:
             with open("Dockerfile", "w+") as f2:
                 f2.write(lines)
-            call("git add Dockerfile")
-            # Tag X.Y.Z[-variant] - see circle CI for shared tag X.Y[-variant]
-            call(["git", "commit", "-m", "Auto-Update"], split=False)
-            call(f"git tag {tag}")
-            new_tags.append(tag)
+
+        with open(template) as f:
+            content = chevron.render(
+                f,
+                {
+                    "MATLAB_VERSION": mcr_name,
+                    "MCR_VERSION": mcr_ver_dir,
+                    "MCR_LINK": link,
+                },
+            )
+        with open("Dockerfile", "w+") as f2:
+            f2.write(content)
+
+        call("git add Dockerfile")
+        # Tag X.Y.Z[-variant] - see circle CI for shared tag X.Y[-variant]
+        call(["git", "commit", "-m", "Auto-Update"], split=False)
+
+        call(f"git tag {tag}")
+
+        new_tags.append(tag)
+
     call("git checkout master")
 
-if new_tags:
-    print("New tags have been added, verify and update to git with:")
-    print("git push --all")
-    for tag in reversed(new_tags):
-        print(f"git push origin {tag}")
+    return new_tags
+
+
+def list_mcr(soup):
+
+    ver_re = re.compile(r"(R2\d{3}.) \((\d\.\d+)\)")
+    rel_re = re.compile(r"Release/(\d+)/")
+
+    dockers = []
+    for row in soup.find_all("table")[0].find_all("tr"):
+
+        tds = row.find_all("td")
+
+        if len(tds) >= 4:
+            name = tds[0].text
+            match = ver_re.match(name)
+            if not match:
+                continue
+            mcr_name, mcr_ver = match.groups()
+
+            if version.parse(mcr_ver) <= version.parse(VER_LIMIT):
+                continue
+            try:
+                link = tds[2].a.get("href")
+            except (KeyError, ValueError) as e:
+                raise RuntimeError("Error parsing matlab release page") from e
+
+            if "glnxa64" not in link:
+                raise RuntimeError("Error parsing matlab release page link")
+
+            if match := rel_re.search(link):
+                mcr_ver = f"{mcr_ver}.{match.groups()[0]}"
+
+            dockers.append((mcr_name, mcr_ver, link))
+
+    return dockers
+
+
+def main():
+
+    with request.urlopen(REL_URL) as res:
+        if res.status != 200:
+            raise RuntimeError("Could not open matlab release URL")
+        html = res.read()
+
+    soup = BeautifulSoup(html, "html.parser")
+
+    dockers = list_mcr(soup)
+
+    new_tags = []
+    for docker in dockers:
+        new_tags = add_dockerfile_to_branch(new_tags, docker)
+
+    if new_tags:
+        print("\nNew tags have been added, verify and update to git with:")
+        print("git push --all")
+        for tag in reversed(new_tags):
+            print(f"git push origin {tag}")
+
+
+if __name__ == "__main__":
+    main()