Skip to content

Commit d9f34e7

Browse files
authored
Fixup eachdist.py handling of package names (#5417)
* scripts/eachdist.py: tighter regexp for packages While at it also introduce tests * scripts: proper check of package name where to bupm version Checking for substrings is not precise enough, let's check for the metadata we have in pyproject.toml. * update workflows * Add missing opentelemetry-exporter-otlp-proto-common * Remove non existant packages * Add test to double check that all packages are accounted for
1 parent d10f472 commit d9f34e7

6 files changed

Lines changed: 214 additions & 8 deletions

File tree

.github/workflows/misc.yml

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,25 @@ env:
2929

3030
jobs:
3131

32+
test-scripts:
33+
name: test-scripts
34+
runs-on: ubuntu-latest
35+
timeout-minutes: 30
36+
steps:
37+
- name: Checkout repo @ SHA - ${{ github.sha }}
38+
uses: actions/checkout@v4
39+
40+
- name: Set up Python 3.11
41+
uses: actions/setup-python@v5
42+
with:
43+
python-version: "3.11"
44+
45+
- name: Install tox
46+
run: pip install tox-uv
47+
48+
- name: Run tests
49+
run: tox -e test-scripts
50+
3251
spellcheck:
3352
name: spellcheck
3453
runs-on: ubuntu-latest

eachdist.ini

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ packages=
2121
opentelemetry-exporter-zipkin-proto-http
2222
opentelemetry-exporter-zipkin-json
2323
opentelemetry-exporter-zipkin
24+
opentelemetry-exporter-otlp-proto-common
2425
opentelemetry-exporter-otlp-proto-grpc
2526
opentelemetry-exporter-otlp-proto-http
2627
opentelemetry-exporter-otlp
@@ -39,11 +40,9 @@ packages=
3940
opentelemetry-exporter-otlp-json-file
4041
opentelemetry-exporter-otlp-common
4142
opentelemetry-configuration
42-
opentelemetry-distro
4343
opentelemetry-proto-json
4444
opentelemetry-semantic-conventions
4545
opentelemetry-test-utils
46-
tests
4746

4847
[lintroots]
4948
extraroots=examples/*,scripts/

scripts/eachdist.py

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -561,12 +561,15 @@ def find(name, path):
561561

562562

563563
def filter_packages(targets, packages):
564+
package_names = set(packages)
564565
filtered_packages = []
565566
for target in targets:
566-
for pkg in packages:
567-
if pkg in str(target):
568-
filtered_packages.append(target)
569-
break
567+
pyproject = target.joinpath("pyproject.toml")
568+
if not pyproject.exists():
569+
continue
570+
target_name = load(pyproject).get("project", {}).get("name")
571+
if target_name in package_names:
572+
filtered_packages.append(target)
570573
return filtered_packages
571574

572575

@@ -601,7 +604,7 @@ def update_dependencies(targets, version, packages):
601604
operators_pattern = "|".join(re.escape(op) for op in operators)
602605

603606
for pkg in packages:
604-
search = rf"({basename(pkg)}[^,]*)({operators_pattern})(.*\.dev)"
607+
search = rf"({basename(pkg)}(?:\[[^\]]+\])?\s*)({operators_pattern})(.*\.dev)"
605608
replace = r"\1\2 " + version
606609
update_files(
607610
targets,
@@ -618,7 +621,10 @@ def update_patch_dependencies(targets, version, prev_version, packages):
618621
operators_pattern = "|".join(re.escape(op) for op in operators)
619622

620623
for pkg in packages:
621-
search = rf"({basename(pkg)}[^,]*?)(\s?({operators_pattern})\s?)(.*{prev_version})"
624+
search = (
625+
rf"({basename(pkg)}(?:\[[^\]]+\])?\s*)"
626+
rf"(\s?({operators_pattern})\s?)(.*{re.escape(prev_version)})"
627+
)
622628
replace = r"\g<1>\g<2>" + version
623629
print(f"{search=}\t{replace=}\t{pkg=}")
624630
update_files(
Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
pytest==8.3.4
2+
toml==0.10.2

scripts/tests/test_eachdist.py

Lines changed: 175 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,175 @@
1+
# Copyright The OpenTelemetry Authors
2+
# SPDX-License-Identifier: Apache-2.0
3+
4+
import importlib.util
5+
from configparser import ConfigParser
6+
from pathlib import Path
7+
from textwrap import dedent
8+
9+
from toml import load
10+
11+
PACKAGE_ROOT_GLOBS = (
12+
"opentelemetry-*/pyproject.toml",
13+
"exporter/opentelemetry-*/pyproject.toml",
14+
"propagator/opentelemetry-*/pyproject.toml",
15+
"shim/opentelemetry-*/pyproject.toml",
16+
"tests/opentelemetry-*/pyproject.toml",
17+
"codegen/opentelemetry-*/pyproject.toml",
18+
)
19+
PRIVATE_PACKAGE_CLASSIFIER = "Private :: Do Not Upload"
20+
21+
22+
def load_eachdist():
23+
script = Path(__file__).parents[1] / "eachdist.py"
24+
spec = importlib.util.spec_from_file_location("eachdist", script)
25+
module = importlib.util.module_from_spec(spec)
26+
spec.loader.exec_module(module)
27+
return module
28+
29+
30+
def write_pyproject(target, dependencies):
31+
target.mkdir()
32+
target.joinpath("pyproject.toml").write_text(
33+
dedent(
34+
f"""
35+
[project]
36+
dependencies = [
37+
{dependencies}
38+
]
39+
"""
40+
),
41+
encoding="utf-8",
42+
)
43+
44+
45+
def write_versioned_project(target, name, version):
46+
version_file = target / "src" / "opentelemetry" / "version" / "__init__.py"
47+
version_file.parent.mkdir(parents=True)
48+
version_file.write_text(
49+
f'__version__ = "{version}"\n',
50+
encoding="utf-8",
51+
)
52+
target.mkdir(exist_ok=True)
53+
target.joinpath("pyproject.toml").write_text(
54+
dedent(
55+
f"""
56+
[project]
57+
name = "{name}"
58+
59+
[tool.hatch.version]
60+
path = "src/opentelemetry/version/__init__.py"
61+
"""
62+
),
63+
encoding="utf-8",
64+
)
65+
66+
67+
def project_root():
68+
return Path(__file__).parents[2]
69+
70+
71+
def package_name(pyproject):
72+
project = load(pyproject)["project"]
73+
if PRIVATE_PACKAGE_CLASSIFIER in project.get("classifiers", ()):
74+
return None
75+
return project["name"]
76+
77+
78+
def test_all_release_packages_are_listed_in_eachdist():
79+
root = project_root()
80+
releasable_package_names = {
81+
name
82+
for package_glob in PACKAGE_ROOT_GLOBS
83+
for pyproject in root.glob(package_glob)
84+
if (name := package_name(pyproject)) is not None
85+
}
86+
87+
config = ConfigParser()
88+
config.read(root / "eachdist.ini")
89+
eachdist_package_names = set(config["stable"]["packages"].split()) | set(
90+
config["prerelease"]["packages"].split()
91+
)
92+
93+
missing_package_names = sorted(
94+
releasable_package_names - eachdist_package_names
95+
)
96+
assert not missing_package_names, (
97+
"packages missing from eachdist.ini: "
98+
f"{', '.join(missing_package_names)}"
99+
)
100+
101+
102+
def test_update_dependencies_matches_exact_package_name(tmp_path):
103+
eachdist = load_eachdist()
104+
target = tmp_path / "target"
105+
write_pyproject(
106+
target,
107+
'"opentelemetry-proto == 1.44.0.dev",\n'
108+
' "opentelemetry-proto-json == 0.65b0.dev",',
109+
)
110+
111+
eachdist.update_dependencies(
112+
[target],
113+
"1.44.0",
114+
["opentelemetry-proto"],
115+
)
116+
117+
pyproject = target.joinpath("pyproject.toml").read_text(encoding="utf-8")
118+
assert '"opentelemetry-proto == 1.44.0",' in pyproject
119+
assert '"opentelemetry-proto-json == 0.65b0.dev",' in pyproject
120+
121+
122+
def test_update_patch_dependencies_matches_exact_package_name(tmp_path):
123+
eachdist = load_eachdist()
124+
target = tmp_path / "target"
125+
write_pyproject(
126+
target,
127+
'"opentelemetry-proto == 1.43.0",\n'
128+
' "opentelemetry-proto-json == 1.43.0",',
129+
)
130+
131+
eachdist.update_patch_dependencies(
132+
[target],
133+
"1.43.1",
134+
"1.43.0",
135+
["opentelemetry-proto"],
136+
)
137+
138+
pyproject = target.joinpath("pyproject.toml").read_text(encoding="utf-8")
139+
assert '"opentelemetry-proto == 1.43.1",' in pyproject
140+
assert '"opentelemetry-proto-json == 1.43.0",' in pyproject
141+
142+
143+
def test_update_version_files_matches_exact_project_name(tmp_path):
144+
eachdist = load_eachdist()
145+
proto = tmp_path / "opentelemetry-proto"
146+
proto_json = tmp_path / "opentelemetry-proto-json"
147+
write_versioned_project(
148+
proto,
149+
"opentelemetry-proto",
150+
"1.44.0.dev",
151+
)
152+
write_versioned_project(
153+
proto_json,
154+
"opentelemetry-proto-json",
155+
"0.65b0.dev",
156+
)
157+
158+
eachdist.update_version_files(
159+
[proto, proto_json],
160+
"1.45.0.dev",
161+
["opentelemetry-proto"],
162+
)
163+
164+
assert (
165+
proto.joinpath("src/opentelemetry/version/__init__.py").read_text(
166+
encoding="utf-8"
167+
)
168+
== '__version__ = "1.45.0.dev"\n'
169+
)
170+
assert (
171+
proto_json.joinpath("src/opentelemetry/version/__init__.py").read_text(
172+
encoding="utf-8"
173+
)
174+
== '__version__ = "0.65b0.dev"\n'
175+
)

tox.ini

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -118,6 +118,8 @@ envlist =
118118
pypy3-test-opentelemetry-test-utils
119119
lint-opentelemetry-test-utils
120120

121+
test-scripts
122+
121123
lint-license-header-check
122124
spellcheck
123125
tracecontext
@@ -194,6 +196,7 @@ deps =
194196

195197
propagator-jaeger: -r {toxinidir}/propagator/opentelemetry-propagator-jaeger/test-requirements.txt
196198

199+
test-scripts: -r {toxinidir}/scripts/tests/test-requirements.txt
197200

198201
getting-started: -r {toxinidir}/docs/getting_started/tests/requirements.txt
199202
getting-started: {env:CONTRIB_REPO_UTIL_HTTP}
@@ -309,6 +312,8 @@ commands =
309312

310313
test-opentelemetry-test-utils: pytest {toxinidir}/tests/opentelemetry-test-utils/tests {posargs}
311314

315+
test-scripts: pytest {toxinidir}/scripts/tests {posargs}
316+
312317
coverage: {toxinidir}/scripts/coverage.sh
313318

314319
[testenv:spellcheck]

0 commit comments

Comments
 (0)