Skip to content

Remove metadata backfill task #15526

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Mar 4, 2024
Merged

Remove metadata backfill task #15526

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
272 changes: 0 additions & 272 deletions tests/unit/packaging/test_tasks.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,33 +10,25 @@
# See the License for the specific language governing permissions and
# limitations under the License.

import builtins
import tempfile

from contextlib import contextmanager
from itertools import product
from pathlib import Path
from zipfile import BadZipFile

import pretend
import pytest

from google.cloud.bigquery import SchemaField
from pip._internal.exceptions import InvalidWheel, UnsupportedWheel
from wtforms import Field, Form, StringField

import warehouse.packaging.tasks

from warehouse.accounts.models import WebAuthn
from warehouse.metrics.interfaces import IMetricsService
from warehouse.packaging.interfaces import IFileStorage
from warehouse.packaging.models import Description
from warehouse.packaging.tasks import (
check_file_cache_tasks_outstanding,
compute_2fa_metrics,
compute_packaging_metrics,
metadata_backfill,
metadata_backfill_individual,
sync_bigquery_release_files,
sync_file_to_cache,
update_bigquery_release_files,
Expand Down Expand Up @@ -896,267 +888,3 @@ def test_compute_2fa_metrics(db_request, monkeypatch):
pretend.call("warehouse.2fa.total_users_with_webauthn_enabled", 1),
pretend.call("warehouse.2fa.total_users_with_two_factor_enabled", 2),
]


def test_metadata_backfill(db_request, monkeypatch, metrics):
project = ProjectFactory()
release1 = ReleaseFactory(project=project)
release2 = ReleaseFactory(project=project)
FileFactory(release=release1, packagetype="sdist")
FileFactory(
release=release1,
packagetype="bdist_wheel",
metadata_file_sha256_digest="d34db33f",
)
FileFactory(release=release2, packagetype="sdist")
backfillable_file = FileFactory(
release=release2, packagetype="bdist_wheel", metadata_file_sha256_digest=None
)

delay = pretend.call_recorder(lambda x: None)
db_request.task = pretend.call_recorder(lambda x: pretend.stub(delay=delay))
db_request.registry.settings["metadata_backfill.batch_size"] = 500

metadata_backfill(db_request)

assert db_request.task.calls == [pretend.call(metadata_backfill_individual)]
assert delay.calls == [pretend.call(backfillable_file.id)]

assert metrics.increment.calls == [
pretend.call("warehouse.packaging.metadata_backfill.tasks"),
]
assert metrics.gauge.calls == [
pretend.call("warehouse.packaging.metadata_backfill.remaining", 1)
]


@pytest.mark.parametrize(
"path, requested_path",
[
# Regular path
(
"8a/6a/19...3b/pip-24.0-py3-none-any.whl",
"8a/6a/19...3b/pip-24.0-py3-none-any.whl",
),
# Path with characters that need quoted
(
"da/5d/cc...0d/scalg-0.1.1#-py3-none-any.whl",
"da/5d/cc...0d/scalg-0.1.1%23-py3-none-any.whl",
),
],
)
def test_metadata_backfill_individual(
db_request, monkeypatch, metrics, path, requested_path
):
project = ProjectFactory()
release1 = ReleaseFactory(project=project)
release2 = ReleaseFactory(project=project)
FileFactory(release=release1, packagetype="sdist")
FileFactory(
release=release1,
packagetype="bdist_wheel",
metadata_file_sha256_digest="d34db33f",
)
FileFactory(release=release2, packagetype="sdist")
backfillable_file = FileFactory(
release=release2,
packagetype="bdist_wheel",
metadata_file_sha256_digest=None,
path=path,
)

metadata_contents = b"some\nmetadata\ncontents"
stub_dist = pretend.stub(
_dist=pretend.stub(_files={Path("METADATA"): metadata_contents})
)
stub_session = pretend.stub()
dist_from_wheel_url = pretend.call_recorder(
lambda project_name, file_url, session: stub_dist
)
monkeypatch.setattr(
warehouse.packaging.tasks, "dist_from_wheel_url", dist_from_wheel_url
)
monkeypatch.setattr(warehouse.packaging.tasks, "PipSession", lambda: stub_session)
archive_storage = pretend.stub(
store=pretend.call_recorder(lambda path_out, path_in, meta: None),
)
cache_storage = pretend.stub(
store=pretend.call_recorder(lambda path_out, path_in, meta: None),
)
db_request.find_service = pretend.call_recorder(
lambda iface, name=None, context=None: {
IFileStorage: {
"archive": archive_storage,
"cache": cache_storage,
},
IMetricsService: {None: metrics},
}[iface][name]
)

@contextmanager
def mock_temporary_directory():
yield "/tmp/wutang"

monkeypatch.setattr(tempfile, "TemporaryDirectory", mock_temporary_directory)

mock_write = pretend.call_recorder(lambda value: None)

@contextmanager
def mock_open(filename, perms):
yield pretend.stub(write=mock_write)

monkeypatch.setattr(builtins, "open", mock_open)

db_request.registry.settings["files.url"] = (
"https://files.example.com/packages/{path}"
)

metadata_backfill_individual(db_request, backfillable_file.id)

assert dist_from_wheel_url.calls == [
pretend.call(
project.normalized_name,
f"https://files.example.com/packages/{requested_path}",
stub_session,
)
]

assert backfillable_file.metadata_file_sha256_digest == (
"e85ce4c9e2d2eddba19c396ed04470efaa2a9c2a6b3c6463e6876a41e55d828d"
)
assert backfillable_file.metadata_file_blake2_256_digest == (
"39cc629504be4087d48889e8666392bd379b91e1826e269cd8467bb29298da82"
)
assert (
archive_storage.store.calls
== cache_storage.store.calls
== [
pretend.call(
backfillable_file.metadata_path,
f"/tmp/wutang/{backfillable_file.filename}.metadata",
meta={
"project": project.normalized_name,
"version": release2.version,
"package-type": backfillable_file.packagetype,
"python-version": backfillable_file.python_version,
},
),
]
)

assert metrics.increment.calls == [
pretend.call("warehouse.packaging.metadata_backfill.files"),
]


@pytest.mark.parametrize(
"exception", [InvalidWheel("foo", "bar"), UnsupportedWheel, BadZipFile, KeyError]
)
def test_metadata_backfill_file_invalid_wheel(
db_request, monkeypatch, metrics, exception
):
project = ProjectFactory()
release = ReleaseFactory(project=project)
backfillable_file = FileFactory(
release=release, packagetype="bdist_wheel", metadata_file_sha256_digest=None
)

stub_session = pretend.stub()
monkeypatch.setattr(warehouse.packaging.tasks, "PipSession", lambda: stub_session)
dist_from_wheel_url = pretend.raiser(exception)
monkeypatch.setattr(
warehouse.packaging.tasks, "dist_from_wheel_url", dist_from_wheel_url
)
db_request.find_service = pretend.call_recorder(
lambda iface, name=None, context=None: {
IFileStorage: {
"archive": pretend.stub(),
"cache": pretend.stub(),
},
IMetricsService: {None: metrics},
}[iface][name]
)
db_request.registry.settings["files.url"] = (
"https://files.example.com/packages/{path}"
)

assert backfillable_file.metadata_file_unbackfillable is False

metadata_backfill_individual(db_request, backfillable_file.id)

assert backfillable_file.metadata_file_unbackfillable is True
assert metrics.increment.calls == []


def test_metadata_backfill_file_oserror(db_request, monkeypatch, metrics):
project = ProjectFactory()
release = ReleaseFactory(project=project)
backfillable_file = FileFactory(
release=release, packagetype="bdist_wheel", metadata_file_sha256_digest=None
)

metadata_contents = b"some\nmetadata\ncontents"
stub_dist = pretend.stub(
_dist=pretend.stub(_files={Path("METADATA"): metadata_contents})
)
stub_session = pretend.stub()
dist_from_wheel_url = pretend.call_recorder(
lambda project_name, file_url, session: stub_dist
)
monkeypatch.setattr(
warehouse.packaging.tasks, "dist_from_wheel_url", dist_from_wheel_url
)
stub_session = pretend.stub()
monkeypatch.setattr(warehouse.packaging.tasks, "PipSession", lambda: stub_session)

@contextmanager
def mock_open(filename, perms):
raise OSError

monkeypatch.setattr(builtins, "open", mock_open)

db_request.find_service = pretend.call_recorder(
lambda iface, name=None, context=None: {
IFileStorage: {
"archive": pretend.stub(),
"cache": pretend.stub(),
},
IMetricsService: {None: metrics},
}[iface][name]
)
db_request.registry.settings["files.url"] = (
"https://files.example.com/packages/{path}"
)

assert backfillable_file.metadata_file_unbackfillable is False

metadata_backfill_individual(db_request, backfillable_file.id)

assert backfillable_file.metadata_file_unbackfillable is True
assert metrics.increment.calls == []


def test_metadata_backfill_file_delorted(db_request, monkeypatch, metrics):
stub_session = pretend.stub()
monkeypatch.setattr(warehouse.packaging.tasks, "PipSession", lambda: stub_session)
dist_from_wheel_url = pretend.call_recorder(lambda *a, **kw: None)
monkeypatch.setattr(
warehouse.packaging.tasks, "dist_from_wheel_url", dist_from_wheel_url
)
db_request.find_service = pretend.call_recorder(
lambda iface, name=None, context=None: {
IFileStorage: {
"archive": pretend.stub(),
"cache": pretend.stub(),
},
IMetricsService: {None: metrics},
}[iface][name]
)
db_request.registry.settings["files.url"] = (
"https://files.example.com/packages/{path}"
)

metadata_backfill_individual(db_request, "66642069-0000-0000-0000-000000000000")

assert dist_from_wheel_url.calls == []
assert metrics.increment.calls == []
4 changes: 0 additions & 4 deletions warehouse/packaging/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,6 @@
check_file_cache_tasks_outstanding,
compute_2fa_metrics,
compute_packaging_metrics,
metadata_backfill,
update_description_html,
)
from warehouse.rate_limiting import IRateLimiter, RateLimit
Expand Down Expand Up @@ -194,6 +193,3 @@ def includeme(config):
# TODO: restore this
# if config.get_settings().get("warehouse.release_files_table"):
# config.add_periodic_task(crontab(minute=0), sync_bigquery_release_files)

# Backfill wheel metadata
config.add_periodic_task(crontab(minute="*/5"), metadata_backfill)
Loading