chore: vendor get-pip, enable network isolation (#26101)

Verification build:
https://dev.azure.com/monacotools/Monaco/_build/results?buildId=462156&view=results

Fixes #26064

Author: rzhao271

.gitignore

@@ -24,7 +24,6 @@ cucumber-report.json
 port.txt
 precommit.hook
 python_files/lib/**
-python_files/get-pip.py
 debug_coverage*/**
 languageServer/**
 languageServer.*/**

.vscodeignore

@@ -63,6 +63,7 @@ out/test/**
 out/testMultiRootWkspc/**
 precommit.hook
 python_files/**/*.pyc
+python_files/download_get_pip.py
 python_files/lib/**/*.egg-info/**
 python_files/lib/jedilsp/bin/**
 python_files/lib/python/bin/**

build/azure-pipeline.pre-release.yml

@@ -87,6 +87,11 @@ extends:
           architecture: 'x64'
         displayName: Select Python version
 
+      - task: PipAuthenticate@1
+        inputs:
+          artifactFeeds: 'Monaco/vscode'
+        displayName: Authenticate to vscode feed
+
       - script: python -m pip install -U pip
         displayName: Upgrade pip
 
@@ -97,7 +102,7 @@ extends:
         displayName: Install NPM dependencies
 
       - script: nox --session install_python_libs
-        displayName: Install Jedi, get-pip, etc
+        displayName: Install Python extension dependencies
 
       - script: python ./build/update_package_file.py
         displayName: Update telemetry in package.json

build/azure-pipeline.stable.yml

@@ -81,6 +81,11 @@ extends:
           architecture: 'x64'
         displayName: Select Python version
 
+      - task: PipAuthenticate@1
+        inputs:
+          artifactFeeds: 'Monaco/vscode'
+        displayName: Authenticate to vscode feed
+
       - script: python -m pip install -U pip
         displayName: Upgrade pip
 
@@ -91,7 +96,7 @@ extends:
         displayName: Install NPM dependencies
 
       - script: nox --session install_python_libs
-        displayName: Install Jedi, get-pip, etc
+        displayName: Install Python extension dependencies
 
       - script: python ./build/update_package_file.py
         displayName: Update telemetry in package.json

build/build-install-requirements.txt

@@ -4,7 +4,7 @@
             "Component": {
                 "Other": {
                     "Name": "get-pip",
-                    "Version": "21.3.1",
+                    "Version": "26.2.1",
                     "DownloadUrl": "https://github.com/pypa/get-pip"
                 },
                 "Type": "other"

cgmanifest.json

@@ -55,13 +55,6 @@ def install_python_libs(session: nox.Session):
     session.install("packaging")
     session.install("debugpy")
 
-    # Download get-pip script
-    session.run(
-        "python",
-        "./python_files/download_get_pip.py",
-        env={"PYTHONPATH": "./python_files/lib/temp"},
-    )
-
     if pathlib.Path("./python_files/lib/temp").exists():
         shutil.rmtree("./python_files/lib/temp")
 

noxfile.py

@@ -1,59 +1,75 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # Licensed under the MIT License.
 
+# This file is now a standalone Python script that
+# updates the vendored get-pip.py and cgmanifest.json
+# to the latest pip release.
+
+import argparse
 import json
 import pathlib
-import urllib.request as url_lib
-
-from packaging.version import parse as version_parser
+import urllib.request
+from typing import Optional
 
 EXTENSION_ROOT = pathlib.Path(__file__).parent.parent
-GET_PIP_DEST = EXTENSION_ROOT / "python_files"
-PIP_PACKAGE = "pip"
-PIP_VERSION = "latest"  # Can be "latest", or specific version "23.1.2"
-
-
-def _get_package_data():
-    json_uri = f"https://pypi.org/pypi/{PIP_PACKAGE}/json"
-    # Response format: https://warehouse.readthedocs.io/api-reference/json/#project
-    # Release metadata format: https://github.com/pypa/interoperability-peps/blob/master/pep-0426-core-metadata.rst
-    with url_lib.urlopen(json_uri) as response:
-        return json.loads(response.read())
-
-
-def _download_and_save(root, version):
-    root = pathlib.Path.cwd() if root is None or root == "." else pathlib.Path(root)
-    url = f"https://raw.githubusercontent.com/pypa/get-pip/{version}/public/get-pip.py"
-    print(url)
-    with url_lib.urlopen(url) as response:
-        data = response.read()
-        get_pip_file = root / "get-pip.py"
-        get_pip_file.write_bytes(data)
-
-
-def main(root):
-    data = _get_package_data()
-
-    if PIP_VERSION == "latest":
-        # Pick latest 5 versions to try and get-pip
-        sorted_versions = sorted(data["releases"].keys(), key=version_parser, reverse=True)[:5]
-        downloaded = False
-        while sorted_versions:
-            use_version = sorted_versions.pop(0)
-            try:
-                print(f"Trying version: get-pip == {use_version}")
-                _download_and_save(root, use_version)
-                downloaded = True
-                break
-            except Exception as e:
-                print(f"Failed to download get-pip == {use_version}: {e}")
-                print(f"NExt attempt(s) with versions: {sorted_versions}")
-        if not downloaded:
-            raise Exception("Failed to download get-pip.py")
-    else:
-        use_version = PIP_VERSION
-        _download_and_save(root, use_version)
+GET_PIP_DEST = EXTENSION_ROOT / "python_files" / "get-pip.py"
+CGMANIFEST_PATH = EXTENSION_ROOT / "cgmanifest.json"
+PIP_METADATA_URL = "https://pypi.org/pypi/pip/json"
+GET_PIP_URL = "https://raw.githubusercontent.com/pypa/get-pip/{version}/public/get-pip.py"
+
+
+def _get_latest_version() -> str:
+    with urllib.request.urlopen(PIP_METADATA_URL) as response:
+        metadata = json.load(response)
+
+    version = metadata.get("info", {}).get("version")
+    if not isinstance(version, str) or not version:
+        raise ValueError(f"PyPI metadata from {PIP_METADATA_URL} did not contain a version")
+    return version
+
+
+def _download_get_pip(version: str) -> bytes:
+    url = GET_PIP_URL.format(version=version)
+    print(f"Downloading {url}")
+    with urllib.request.urlopen(url) as response:
+        get_pip = response.read()
+
+    expected_version = f"pip (version {version})".encode()
+    if expected_version not in get_pip[:1024]:
+        raise ValueError(f"Downloaded get-pip.py did not contain pip {version}")
+    return get_pip
+
+
+def _update_cgmanifest(version: str) -> str:
+    manifest = json.loads(CGMANIFEST_PATH.read_text(encoding="utf-8"))
+    for registration in manifest["Registrations"]:
+        component = registration.get("Component", {}).get("Other", {})
+        if component.get("Name") == "get-pip":
+            component["Version"] = version
+            return f"{json.dumps(manifest, indent=4)}\n"
+
+    raise ValueError(f"get-pip registration was not found in {CGMANIFEST_PATH}")
+
+
+def refresh_get_pip(version: Optional[str] = None) -> None:
+    version = version or _get_latest_version()
+    get_pip = _download_get_pip(version)
+    cgmanifest = _update_cgmanifest(version)
+
+    GET_PIP_DEST.write_bytes(get_pip)
+    CGMANIFEST_PATH.write_text(cgmanifest, encoding="utf-8")
+    print(f"Updated {GET_PIP_DEST} and {CGMANIFEST_PATH} to get-pip {version}")
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Refresh the vendored get-pip.py")
+    parser.add_argument(
+        "--version",
+        help="pip release to vendor; defaults to the latest release on PyPI",
+    )
+    args = parser.parse_args()
+    refresh_get_pip(args.version)
 
 
 if __name__ == "__main__":
-    main(GET_PIP_DEST)
+    main()