python_metadata.bzl

# After updating this file, make sure to run "bazel mod tidy"
load("@bazel_lib//lib:base64.bzl", "base64")
load("@bazel_lib//lib:strings.bzl", "chr")
load("//:build/python/packages_20240829_4.bzl", "PACKAGES_20240829_4")
load("//:build/python/packages_20250808.bzl", "PACKAGES_20250808")

def _chunk(data, length):
    return [data[i:i + length] for i in range(0, len(data), length)]

def hex_to_b64(hex):
    s = ""
    for chunk in _chunk(hex, 2):
        s += chr(int(chunk, 16))
    return "sha256-" + base64.encode(s)

PYODIDE_VERSIONS = [
    {
        "version": "0.26.0a2",
        "sha256": "fbda450a64093a8d246c872bb901ee172a57fe594c9f35bba61f36807c73300d",
    },
    {
        "version": "0.28.2",
        "sha256": "c9f6dd067d119e50850849f7428e3c636ecbc2684a0d2ff992f3bd48a1062b6c",
    },
]

# This is the list of all the package metadata that we use.
#
# IMPORTANT: packages that are present here should never be removed after the package version is
# released to the public. This is so that we don't break workers using those packages.
#
# ORDER MATTERS: the order of the keys in this dictionary matters, older package bundles should come
# first.
_package_lockfiles = [
    PACKAGES_20240829_4,
    PACKAGES_20250808,
]

# The below is a list of pyodide-lock.json files for each package bundle version that we support.
# Each of these gets embedded in the workerd and EW binary.
PYTHON_LOCKFILES = [meta["info"] for meta in _package_lockfiles]

# Used to generate the import tests, where we import each top level name from each package and check
# that it doesn't fail.
PYTHON_IMPORTS_TO_TEST = {meta["info"]["tag"]: meta["import_tests"] for meta in _package_lockfiles}

# Each new package bundle should contain the same packages as the previous. We verify this
# constraint here.
def verify_no_packages_were_removed():
    for curr_info, next_info in zip(_package_lockfiles[:-1], _package_lockfiles[1:]):
        curr_pkgs = curr_info["import_tests"]
        next_pkgs = next_info["import_tests"]
        missing_pkgs = [pkg for pkg in curr_pkgs if pkg not in next_pkgs]
        if missing_pkgs:
            fail("Some packages from version ", curr_info["info"]["tag"], " missing in version", next_info["info"]["tag"], ":\n", "   ", ", ".join(missing_pkgs), "\n\n")

verify_no_packages_were_removed()

def _bundle_id(*, pyodide_version, pyodide_date, backport, **_kwds):
    return "%s_%s_%s" % (pyodide_version, pyodide_date, backport)

def _add_integrity(entry):
    for key, value in entry.items():
        if not key.endswith("_hash"):
            continue
        newkey = key.removesuffix("_hash") + "_integrity"
        entry[newkey] = hex_to_b64(value)

def _make_vendored_packages(entry):
    if entry["name"] == "development":
        return
    vendor_tests = {}
    for e in entry["vendored_packages_for_tests"]:
        vendor_tests[e["name"]] = e
    entry["vendored_packages_for_tests"] = vendor_tests

def _check_pyodide_versions(version_info):
    pyodide_versions = {ver["version"]: 1 for ver in PYODIDE_VERSIONS}
    pyodide_versions["dev"] = 1
    for entry in version_info.values():
        if entry["pyodide_version"] not in pyodide_versions:
            fail("Version %s of Pyodide not in PYODIDE_VERSIONS" % entry["pyodide_version"])

def _make_bundle_version_info(versions):
    result = {}
    for entry in versions:
        name = entry["name"]
        if name != "development":
            entry["id"] = _bundle_id(**entry)
            entry["real_pyodide_version"] = entry["pyodide_version"]
        entry["feature_flags"] = [entry["flag"]]
        entry["feature_string_flags"] = [entry["enable_flag_name"]]
        if "packages" in entry:
            entry["packages"] = entry["packages"]["info"]["tag"]
        _add_integrity(entry)
        result[name] = entry
        _make_vendored_packages(entry)

    dev = result["development"]

    # Uncomment to test with development = 0.26.0a2
    # dev["real_pyodide_version"] = "0.26.0a2"
    result["development"] = result[dev["real_pyodide_version"]] | dev
    _check_pyodide_versions(result)
    return result

VENDORED_VERSION_INDEPENDENT = [
    {
        # Downloaded from https://pub-25a5b2f2f1b84655b185a505c7a3ad23.r2.dev/beautifulsoup4-vendored-for-ew-testing.zip
        "name": "beautifulsoup4",
        "abi": None,
        "sha256": "5aa09c5f549443969dda260a70e58e3ac8537bd3d29155b307a3d98b36eb70fd",
    },
    {
        "name": "pytest-asyncio",
        "abi": None,
        "sha256": "be25b788392d124cbdfbb9b3d13541da69a7b6b977bc9474dddaddeaab6421b4",
    },
    {
        "name": "python-workers-runtime-sdk",
        "abi": None,
        "sha256": "fc4fb50f73973c257277155b3cb113aa2cf68e9da8ef424ecb049b41bc463183",
    },
]

BUNDLE_VERSION_INFO = _make_bundle_version_info([
    {
        "name": "0.26.0a2",
        "released": True,
        "pyodide_version": "0.26.0a2",
        "pyodide_date": "2024-03-01",
        "packages": PACKAGES_20240829_4,
        "backport": "78",
        "integrity": "sha256-V1iAhkevGuXaTVQo+85FbC8CINU6F7F+H5uwnmfBHTU=",
        "flag": "pythonWorkers",
        "enable_flag_name": "python_workers",
        "emscripten_version": "3.1.52",
        "python_version": "3.12.1",
        "baseline_snapshot": "baseline-61eedf943.bin",
        "baseline_snapshot_hash": "61eedf9432d635bdf091b26efece020b3543429a609fad7af9e8d4de2ec44f47",
        "numpy_snapshot": "ew-py-package-snapshot_numpy-v2.bin",
        "numpy_snapshot_hash": "5055deb53f404afacba73642fd10e766b123e661847e8fdf4f1ec92d8ca624dc",
        "fastapi_snapshot": "ew-py-package-snapshot_fastapi-v2.bin",
        "fastapi_snapshot_hash": "d204956a074cd74f7fe72e029e9a82686fcb8a138b509f765e664a03bfdd50fb",
        "vendored_packages_for_tests": VENDORED_VERSION_INDEPENDENT + [
            {
                # Downloaded from https://pub-25a5b2f2f1b84655b185a505c7a3ad23.r2.dev/fastapi-312-vendored-for-ew-testing.zip
                "name": "fastapi",
                "abi": "3.12",
                "sha256": "5e6e21dbeda7c1eaadb99e6e52aa2ce45325b51e9a417198701e68e0cfd12a4c",
            },
            {
                "name": "scipy",
                "abi": "3.12",
                "sha256": "787e45be6969a5609093b3df9cc2dba2afec9e10bace977f5045697cc329aa7c",
            },
        ],
    },
    {
        "name": "0.28.2",
        "pyodide_version": "0.28.2",
        "pyodide_date": "2025-01-16",
        "packages": PACKAGES_20250808,
        "backport": "9",
        "integrity": "sha256-uauWhzzVDWY9UUDpMOCOYxVbSHS9oCnjFizu039PL6U=",
        "flag": "pythonWorkers20250116",
        "enable_flag_name": "python_workers_20250116",
        "emscripten_version": "4.0.9",
        "python_version": "3.13.2",
        "baseline_snapshot": "baseline-4569679fb.bin",
        "baseline_snapshot_hash": "4569679fb78a3c5c8dbfa73d57c61c6a5394617632fbac7b5873ba322c85463d",
        "numpy_snapshot": "package_snapshot_numpy-60c9cb28e.bin",
        "numpy_snapshot_hash": "60c9cb28e6dc1ea6ab38b25471ddaa315b667637c9dd6f94aceb2acc6519c623",
        "fastapi_snapshot": "package_snapshot_fastapi-a6ccb56fe.bin",
        "fastapi_snapshot_hash": "a6ccb56fe9eac265d139727d0134e8d6432c5fe25c8c0b8ec95252b13493b297",
        "dedicated_fastapi_snapshot": "snapshot_a6b652a95810783f5078b9a5dbd4a07c30718acb4ff724e82c25db7353dd7f2d.bin",
        "dedicated_fastapi_snapshot_hash": "4af6f012a5fb32f31a426e6f109e88ae85b18ee3dd131e1caaaad989cd962bbe",
        "vendored_packages_for_tests": VENDORED_VERSION_INDEPENDENT + [
            {
                "name": "fastapi",
                "abi": "3.13",
                "sha256": "955091f1bd2eb33255ff2633df990bedc96e2f6294e78f2b416078777394f942",
            },
            # {
            #     "name": "scipy",
            #     "abi": "3.13",
            #     "sha256": "4f1b6fc179bd5c6d3de68abc4aa9fca2aaecd09c5c8d357c2ecfedce7d621f3d",
            # },
            {
                "name": "shapely",
                "abi": "3.13",
                "sha256": "2e5c462cb32ee8697b3647dfc9d5c88dcdfd0702da34a2d7dc6b07b8090dd321",
            },
        ],
    },
    {
        "real_pyodide_version": "0.28.2",
        "name": "development",
        "pyodide_version": "dev",
        "pyodide_date": "dev",
        "id": "dev",
        "flag": "pythonWorkersDevPyodide",
        "enable_flag_name": "python_workers_development",
    },
])