Skip to content

Commit 6e5479f

Browse files
authored
Optimize flattening in apache airflow workspace (#11313)
## Motivation No-op `uv lock` in apache airflow (891c67f210ab7c877d1f00ea6ea3d3cdbb0e96ef) is slow, which makes `uv run` slow, too. Reference project: ``` $ hyperfine "uv run python -c \"print('hi')\"" Benchmark 1: uv run python -c "print('hi')" Time (mean ± σ): 16.3 ms ± 1.5 ms [User: 9.8 ms, System: 6.4 ms] Range (min … max): 13.0 ms … 20.0 ms 186 runs ``` Apache airflow before: ``` $ hyperfine "uv run python -c \"print('hi')\"" Benchmark 1: uv run python -c "print('hi')" Time (mean ± σ): 161.0 ms ± 5.2 ms [User: 135.3 ms, System: 24.1 ms] Range (min … max): 155.0 ms … 176.3 ms 18 runs ``` ## Optimization `FlatRequiresDist::from_requirements` is taking 50% of main thread runtime. Before: ![image](https://github.com/user-attachments/assets/10ea76eb-d1e9-477c-b400-39e653eb8f3a) After both commits: ![image](https://github.com/user-attachments/assets/5c578ff6-f80b-46bb-9b5f-8be8435c3d85) Apache airflow after the first commit: ``` $ hyperfine "uv-profiling run python -c \"print('hi')\"" Benchmark 1: uv-profiling run python -c "print('hi')" Time (mean ± σ): 122.3 ms ± 5.4 ms [User: 96.1 ms, System: 24.7 ms] Range (min … max): 114.0 ms … 133.2 ms 23 runs ``` Apache airflow after the second commit: ``` $ hyperfine "uv-profiling run python -c \"print('hi')\"" Benchmark 1: uv-profiling run python -c "print('hi')" Time (mean ± σ): 108.5 ms ± 3.4 ms [User: 83.2 ms, System: 24.2 ms] Range (min … max): 103.6 ms … 119.9 ms 28 runs ```
1 parent 711766e commit 6e5479f

File tree

4 files changed

+50
-34
lines changed

4 files changed

+50
-34
lines changed

crates/uv-distribution/src/metadata/build_requires.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -106,7 +106,7 @@ impl BuildRequires {
106106
project_workspace.project_root(),
107107
project_sources,
108108
project_indexes,
109-
extra.as_ref(),
109+
extra.as_deref(),
110110
group,
111111
locations,
112112
project_workspace.workspace(),
@@ -181,7 +181,7 @@ impl BuildRequires {
181181
workspace.install_path(),
182182
project_sources,
183183
project_indexes,
184-
extra.as_ref(),
184+
extra.as_deref(),
185185
group,
186186
locations,
187187
workspace,

crates/uv-distribution/src/metadata/lowering.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -362,7 +362,7 @@ impl LoweredRequirement {
362362
requirement
363363
.marker
364364
.top_level_extra_name()
365-
.is_some_and(|extra| extra == *target)
365+
.is_some_and(|extra| &*extra == target)
366366
})
367367
})
368368
.cloned()

crates/uv-distribution/src/metadata/requires_dist.rs

Lines changed: 35 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -209,7 +209,7 @@ impl RequiresDist {
209209
project_workspace.project_root(),
210210
project_sources,
211211
project_indexes,
212-
extra.as_ref(),
212+
extra.as_deref(),
213213
group,
214214
locations,
215215
project_workspace.workspace(),
@@ -262,7 +262,7 @@ impl RequiresDist {
262262
// If there is no such requirement with the extra, error.
263263
if !metadata.requires_dist.iter().any(|requirement| {
264264
requirement.name == *name
265-
&& requirement.marker.top_level_extra_name().as_ref() == Some(extra)
265+
&& requirement.marker.top_level_extra_name().as_deref() == Some(extra)
266266
}) {
267267
return Err(MetadataError::IncompleteSourceExtra(
268268
name.clone(),
@@ -370,6 +370,12 @@ impl FlatRequiresDist {
370370
return Self(requirements);
371371
}
372372

373+
// Memoize the top level extras, in the same order as `requirements`
374+
let top_level_extras: Vec<_> = requirements
375+
.iter()
376+
.map(|req| req.marker.top_level_extra_name())
377+
.collect();
378+
373379
// Transitively process all extras that are recursively included.
374380
let mut flattened = requirements.clone();
375381
let mut seen = FxHashSet::<(ExtraName, MarkerTree)>::default();
@@ -384,33 +390,34 @@ impl FlatRequiresDist {
384390
}
385391

386392
// Find the requirements for the extra.
387-
for requirement in &requirements {
388-
if requirement.marker.top_level_extra_name().as_ref() == Some(&extra) {
389-
let requirement = {
390-
let mut marker = marker;
391-
marker.and(requirement.marker);
392-
uv_pypi_types::Requirement {
393-
name: requirement.name.clone(),
394-
extras: requirement.extras.clone(),
395-
groups: requirement.groups.clone(),
396-
source: requirement.source.clone(),
397-
origin: requirement.origin.clone(),
398-
marker: marker.simplify_extras(slice::from_ref(&extra)),
399-
}
400-
};
401-
if requirement.name == *name {
402-
// Add each transitively included extra.
403-
queue.extend(
404-
requirement
405-
.extras
406-
.iter()
407-
.cloned()
408-
.map(|extra| (extra, requirement.marker)),
409-
);
410-
} else {
411-
// Add the requirements for that extra.
412-
flattened.push(requirement);
393+
for (requirement, top_level_extra) in requirements.iter().zip(top_level_extras.iter()) {
394+
if top_level_extra.as_deref() != Some(&extra) {
395+
continue;
396+
}
397+
let requirement = {
398+
let mut marker = marker;
399+
marker.and(requirement.marker);
400+
uv_pypi_types::Requirement {
401+
name: requirement.name.clone(),
402+
extras: requirement.extras.clone(),
403+
groups: requirement.groups.clone(),
404+
source: requirement.source.clone(),
405+
origin: requirement.origin.clone(),
406+
marker: marker.simplify_extras(slice::from_ref(&extra)),
413407
}
408+
};
409+
if requirement.name == *name {
410+
// Add each transitively included extra.
411+
queue.extend(
412+
requirement
413+
.extras
414+
.iter()
415+
.cloned()
416+
.map(|extra| (extra, requirement.marker)),
417+
);
418+
} else {
419+
// Add the requirements for that extra.
420+
flattened.push(requirement);
414421
}
415422
}
416423
}

crates/uv-pep508/src/marker/tree.rs

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
use std::borrow::Cow;
12
use std::cmp::Ordering;
23
use std::fmt::{self, Display, Formatter};
34
use std::ops::{Bound, Deref};
@@ -772,7 +773,7 @@ impl MarkerTree {
772773
}
773774

774775
/// Returns the underlying [`MarkerTreeKind`] of the root node.
775-
pub fn kind(&self) -> MarkerTreeKind<'_> {
776+
pub fn kind(self) -> MarkerTreeKind<'static> {
776777
if self.is_true() {
777778
return MarkerTreeKind::True;
778779
}
@@ -1002,11 +1003,19 @@ impl MarkerTree {
10021003
///
10031004
/// ASSUMPTION: There is one `extra = "..."`, and it's either the only marker or part of the
10041005
/// main conjunction.
1005-
pub fn top_level_extra_name(self) -> Option<ExtraName> {
1006+
pub fn top_level_extra_name(self) -> Option<Cow<'static, ExtraName>> {
1007+
// Fast path: The marker is only a `extra == "..."`.
1008+
if let MarkerTreeKind::Extra(marker) = self.kind() {
1009+
if marker.edge(true).is_true() {
1010+
let CanonicalMarkerValueExtra::Extra(extra) = marker.name;
1011+
return Some(Cow::Borrowed(extra));
1012+
}
1013+
}
1014+
10061015
let extra_expression = self.top_level_extra()?;
10071016

10081017
match extra_expression {
1009-
MarkerExpression::Extra { name, .. } => name.into_extra(),
1018+
MarkerExpression::Extra { name, .. } => name.into_extra().map(Cow::Owned),
10101019
_ => unreachable!(),
10111020
}
10121021
}

0 commit comments

Comments
 (0)