Skip to content

Commit 72ba525

Browse files
leandrobbragantBre
andauthored
[ruff] Detect duplicate entries in __all__ (RUF068) (#22114)
Hello, This MR adds a new rule and its fix, `RUF069`, `DuplicateEntryInDunderAll`. I'm using `RUF069` because we already have [RUF068](astral-sh/ruff#20585) and [RUF069](astral-sh/ruff#21079 (comment)) in the works. The rule job is to prevent users from accidentally adding duplicate entries to `__all__`, which, for example, can result from copy-paste mistakes. It deals with the following syntaxes: ```python __all__: list[str] = ["a", "a"] __all__: typing.Any = ("a", "a") __all__.extend(["a", "a"]) __all__ += ["a", "a"] ``` But it does not keep track of `__all__` contents, meaning the following code snippet is a false negative: ```python class A: ... __all__ = ["A"] __all__.extend(["A"]) ``` ## Violation Example ```console RUF069 `__all__` contains duplicate entries --> RUF069.py:2:17 | 1 | __all__ = ["A", "A", "B"] | ^^^ help: Remove duplicate entries from `__all__` 1 | __all__ = ["A", "B"] - __all__ = ["A", "A", "B"] ``` ## Ecosystem Report The `ruff-ecosystem` results contain seven violations in four projects, all of them seem like true positives, with one instance appearing to be an actual bug. This [code snippet](https://github.com/python/typeshed/blob/90d855985be5aae9bc76e77b0f3d4b6738c38347/stubs/reportlab/reportlab/lib/rltempfile.pyi#L4) from `reportlab` contains the same entry twice instead of exporting both functions. ```python def get_rl_tempdir(*subdirs: str) -> str: ... def get_rl_tempfile(fn: str | None = None) -> str: ... __all__ = ("get_rl_tempdir", "get_rl_tempdir") ``` Closes [#21945](astral-sh/ruff#21945) --------- Co-authored-by: Brent Westbrook <brentrwestbrook@gmail.com>
1 parent a267ed7 commit 72ba525

11 files changed

Lines changed: 546 additions & 44 deletions

File tree

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
import typing
2+
3+
4+
class A: ...
5+
6+
7+
class B: ...
8+
9+
10+
# Good
11+
__all__ = "A" + "B"
12+
__all__: list[str] = ["A", "B"]
13+
__all__: typing.Any = ("A", "B")
14+
__all__ = ["A", "B"]
15+
__all__ = [A, "B", "B"]
16+
__all__ += ["A", "B"]
17+
__all__.extend(["A", "B"])
18+
19+
# Bad
20+
__all__: list[str] = ["A", "B", "A"]
21+
__all__: typing.Any = ("A", "B", "B")
22+
__all__ = ["A", "B", "A"]
23+
__all__ = ["A", "A", "B", "B"]
24+
__all__ = [
25+
"A",
26+
"A",
27+
"B",
28+
"B"
29+
]
30+
__all__ += ["B", "B"]
31+
__all__.extend(["B", "B"])
32+
33+
# Bad, unsafe
34+
__all__ = [
35+
"A",
36+
"A",
37+
"B",
38+
# Comment
39+
"B", # 2
40+
# 3
41+
]

crates/ruff_linter/src/checkers/ast/analyze/expression.rs

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1245,6 +1245,9 @@ pub(crate) fn expression(expr: &Expr, checker: &Checker) {
12451245
if checker.is_rule_enabled(Rule::UnsortedDunderAll) {
12461246
ruff::rules::sort_dunder_all_extend_call(checker, call);
12471247
}
1248+
if checker.is_rule_enabled(Rule::DuplicateEntryInDunderAll) {
1249+
ruff::rules::duplicate_entry_in_dunder_all_extend_call(checker, call);
1250+
}
12481251
if checker.is_rule_enabled(Rule::DefaultFactoryKwarg) {
12491252
ruff::rules::default_factory_kwarg(checker, call);
12501253
}

crates/ruff_linter/src/checkers/ast/analyze/statement.rs

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -966,6 +966,9 @@ pub(crate) fn statement(stmt: &Stmt, checker: &mut Checker) {
966966
if checker.is_rule_enabled(Rule::UnsortedDunderAll) {
967967
ruff::rules::sort_dunder_all_aug_assign(checker, aug_assign);
968968
}
969+
if checker.is_rule_enabled(Rule::DuplicateEntryInDunderAll) {
970+
ruff::rules::duplicate_entry_in_dunder_all_aug_assign(checker, aug_assign);
971+
}
969972
}
970973
Stmt::If(
971974
if_ @ ast::StmtIf {
@@ -1434,6 +1437,9 @@ pub(crate) fn statement(stmt: &Stmt, checker: &mut Checker) {
14341437
if checker.is_rule_enabled(Rule::UnsortedDunderAll) {
14351438
ruff::rules::sort_dunder_all_assign(checker, assign);
14361439
}
1440+
if checker.is_rule_enabled(Rule::DuplicateEntryInDunderAll) {
1441+
ruff::rules::duplicate_entry_in_dunder_all_assign(checker, assign);
1442+
}
14371443
if checker.source_type.is_stub() {
14381444
if checker.any_rule_enabled(&[
14391445
Rule::UnprefixedTypeParam,
@@ -1525,6 +1531,9 @@ pub(crate) fn statement(stmt: &Stmt, checker: &mut Checker) {
15251531
if checker.is_rule_enabled(Rule::UnsortedDunderAll) {
15261532
ruff::rules::sort_dunder_all_ann_assign(checker, assign_stmt);
15271533
}
1534+
if checker.is_rule_enabled(Rule::DuplicateEntryInDunderAll) {
1535+
ruff::rules::duplicate_entry_in_dunder_all_ann_assign(checker, assign_stmt);
1536+
}
15281537
if checker.source_type.is_stub() {
15291538
if let Some(value) = value {
15301539
if checker.is_rule_enabled(Rule::AssignmentDefaultInStub) {

crates/ruff_linter/src/codes.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1061,6 +1061,7 @@ pub fn code_to_rule(linter: Linter, code: &str) -> Option<(RuleGroup, Rule)> {
10611061
(Ruff, "065") => rules::ruff::rules::LoggingEagerConversion,
10621062
(Ruff, "066") => rules::ruff::rules::PropertyWithoutReturn,
10631063
(Ruff, "067") => rules::ruff::rules::NonEmptyInitModule,
1064+
(Ruff, "068") => rules::ruff::rules::DuplicateEntryInDunderAll,
10641065

10651066
(Ruff, "100") => rules::ruff::rules::UnusedNOQA,
10661067
(Ruff, "101") => rules::ruff::rules::RedirectedNOQA,

crates/ruff_linter/src/fix/edits.rs

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -284,6 +284,46 @@ pub(crate) fn add_argument(argument: &str, arguments: &Arguments, tokens: &Token
284284
}
285285
}
286286

287+
/// Remove the member at the given index from a sequence of expressions.
288+
pub(crate) fn remove_member(elts: &[ast::Expr], index: usize, source: &str) -> Result<Edit> {
289+
if index < elts.len() - 1 {
290+
// Case 1: the expression is _not_ the last node, so delete from the start of the
291+
// expression to the end of the subsequent comma.
292+
// Ex) Delete `"a"` in `{"a", "b", "c"}`.
293+
let mut tokenizer = SimpleTokenizer::starts_at(elts[index].end(), source);
294+
295+
// Find the trailing comma.
296+
tokenizer
297+
.find(|token| token.kind == SimpleTokenKind::Comma)
298+
.context("Unable to find trailing comma")?;
299+
300+
// Find the next non-whitespace token.
301+
let next = tokenizer
302+
.find(|token| {
303+
token.kind != SimpleTokenKind::Whitespace && token.kind != SimpleTokenKind::Newline
304+
})
305+
.context("Unable to find next token")?;
306+
307+
Ok(Edit::deletion(elts[index].start(), next.start()))
308+
} else if index > 0 {
309+
// Case 2: the expression is the last node, but not the _only_ node, so delete from the
310+
// start of the previous comma to the end of the expression.
311+
// Ex) Delete `"c"` in `{"a", "b", "c"}`.
312+
let mut tokenizer = SimpleTokenizer::starts_at(elts[index - 1].end(), source);
313+
314+
// Find the trailing comma.
315+
let comma = tokenizer
316+
.find(|token| token.kind == SimpleTokenKind::Comma)
317+
.context("Unable to find trailing comma")?;
318+
319+
Ok(Edit::deletion(comma.start(), elts[index].end()))
320+
} else {
321+
// Case 3: expression is the only node, so delete it.
322+
// Ex) Delete `"a"` in `{"a"}`.
323+
Ok(Edit::range_deletion(elts[index].range()))
324+
}
325+
}
326+
287327
/// Generic function to add a (regular) parameter to a function definition.
288328
pub(crate) fn add_parameter(parameter: &str, parameters: &Parameters, source: &str) -> Edit {
289329
if let Some(last) = parameters.args.iter().rfind(|arg| arg.default.is_none()) {

crates/ruff_linter/src/rules/flake8_bugbear/rules/duplicate_value.rs

Lines changed: 5 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,15 @@
1-
use anyhow::{Context, Result};
1+
use ruff_diagnostics::Fix;
22
use rustc_hash::FxHashMap;
33

44
use ruff_macros::{ViolationMetadata, derive_message_formats};
55
use ruff_python_ast as ast;
66
use ruff_python_ast::Expr;
77
use ruff_python_ast::comparable::HashableExpr;
8-
use ruff_python_trivia::{SimpleTokenKind, SimpleTokenizer};
98
use ruff_text_size::Ranged;
109

1110
use crate::checkers::ast::Checker;
12-
use crate::{Edit, Fix, FixAvailability, Violation};
11+
use crate::fix::edits;
12+
use crate::{FixAvailability, Violation};
1313

1414
/// ## What it does
1515
/// Checks for set literals that contain duplicate items.
@@ -70,49 +70,10 @@ pub(crate) fn duplicate_value(checker: &Checker, set: &ast::ExprSet) {
7070
);
7171

7272
diagnostic.try_set_fix(|| {
73-
remove_member(set, index, checker.locator().contents()).map(Fix::safe_edit)
73+
edits::remove_member(&set.elts, index, checker.locator().contents())
74+
.map(Fix::safe_edit)
7475
});
7576
}
7677
}
7778
}
7879
}
79-
80-
/// Remove the member at the given index from the [`ast::ExprSet`].
81-
fn remove_member(set: &ast::ExprSet, index: usize, source: &str) -> Result<Edit> {
82-
if index < set.len() - 1 {
83-
// Case 1: the expression is _not_ the last node, so delete from the start of the
84-
// expression to the end of the subsequent comma.
85-
// Ex) Delete `"a"` in `{"a", "b", "c"}`.
86-
let mut tokenizer = SimpleTokenizer::starts_at(set.elts[index].end(), source);
87-
88-
// Find the trailing comma.
89-
tokenizer
90-
.find(|token| token.kind == SimpleTokenKind::Comma)
91-
.context("Unable to find trailing comma")?;
92-
93-
// Find the next non-whitespace token.
94-
let next = tokenizer
95-
.find(|token| {
96-
token.kind != SimpleTokenKind::Whitespace && token.kind != SimpleTokenKind::Newline
97-
})
98-
.context("Unable to find next token")?;
99-
100-
Ok(Edit::deletion(set.elts[index].start(), next.start()))
101-
} else if index > 0 {
102-
// Case 2: the expression is the last node, but not the _only_ node, so delete from the
103-
// start of the previous comma to the end of the expression.
104-
// Ex) Delete `"c"` in `{"a", "b", "c"}`.
105-
let mut tokenizer = SimpleTokenizer::starts_at(set.elts[index - 1].end(), source);
106-
107-
// Find the trailing comma.
108-
let comma = tokenizer
109-
.find(|token| token.kind == SimpleTokenKind::Comma)
110-
.context("Unable to find trailing comma")?;
111-
112-
Ok(Edit::deletion(comma.start(), set.elts[index].end()))
113-
} else {
114-
// Case 3: expression is the only node, so delete it.
115-
// Ex) Delete `"a"` in `{"a"}`.
116-
Ok(Edit::range_deletion(set.elts[index].range()))
117-
}
118-
}

crates/ruff_linter/src/rules/ruff/mod.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -117,6 +117,7 @@ mod tests {
117117
#[test_case(Rule::LoggingEagerConversion, Path::new("RUF065_0.py"))]
118118
#[test_case(Rule::LoggingEagerConversion, Path::new("RUF065_1.py"))]
119119
#[test_case(Rule::PropertyWithoutReturn, Path::new("RUF066.py"))]
120+
#[test_case(Rule::DuplicateEntryInDunderAll, Path::new("RUF068.py"))]
120121
#[test_case(Rule::RedirectedNOQA, Path::new("RUF101_0.py"))]
121122
#[test_case(Rule::RedirectedNOQA, Path::new("RUF101_1.py"))]
122123
#[test_case(Rule::InvalidRuleCode, Path::new("RUF102.py"))]
Lines changed: 180 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,180 @@
1+
use rustc_hash::{FxBuildHasher, FxHashMap};
2+
3+
use ruff_diagnostics::{Applicability, Fix};
4+
use ruff_macros::{ViolationMetadata, derive_message_formats};
5+
use ruff_python_ast as ast;
6+
use ruff_text_size::Ranged;
7+
8+
use crate::checkers::ast::Checker;
9+
use crate::fix::edits;
10+
use crate::{FixAvailability, Violation};
11+
12+
/// ## What it does
13+
/// Detects duplicate elements in `__all__` definitions.
14+
///
15+
/// ## Why is this bad?
16+
/// Duplicate elements in `__all__` serve no purpose and can indicate copy-paste errors or
17+
/// incomplete refactoring.
18+
///
19+
/// ## Example
20+
/// ```python
21+
/// __all__ = [
22+
/// "DatabaseConnection",
23+
/// "Product",
24+
/// "User",
25+
/// "DatabaseConnection", # Duplicate
26+
/// ]
27+
/// ```
28+
///
29+
/// Use instead:
30+
/// ```python
31+
/// __all__ = [
32+
/// "DatabaseConnection",
33+
/// "Product",
34+
/// "User",
35+
/// ]
36+
/// ```
37+
///
38+
/// ## Fix Safety
39+
/// This rule's fix is marked as unsafe if the replacement would remove comments attached to the
40+
/// original expression, potentially losing important context or documentation.
41+
///
42+
/// For example:
43+
/// ```python
44+
/// __all__ = [
45+
/// "PublicAPI",
46+
/// # TODO: Remove this in v2.0
47+
/// "PublicAPI", # Deprecated alias
48+
/// ]
49+
/// ```
50+
#[derive(ViolationMetadata)]
51+
#[violation_metadata(preview_since = "0.14.14")]
52+
pub(crate) struct DuplicateEntryInDunderAll;
53+
54+
impl Violation for DuplicateEntryInDunderAll {
55+
const FIX_AVAILABILITY: FixAvailability = FixAvailability::Sometimes;
56+
57+
#[derive_message_formats]
58+
fn message(&self) -> String {
59+
"`__all__` contains duplicate entries".to_string()
60+
}
61+
62+
fn fix_title(&self) -> Option<String> {
63+
Some("Remove duplicate entries from `__all__`".to_string())
64+
}
65+
}
66+
67+
/// Apply RUF068 to `StmtAssign` AST node. For example: `__all__ = ["a", "b", "a"]`.
68+
pub(crate) fn duplicate_entry_in_dunder_all_assign(
69+
checker: &Checker,
70+
ast::StmtAssign { value, targets, .. }: &ast::StmtAssign,
71+
) {
72+
if let [expr] = targets.as_slice() {
73+
duplicate_entry_in_dunder_all(checker, expr, value);
74+
}
75+
}
76+
77+
/// Apply RUF068 to `StmtAugAssign` AST node. For example: `__all__ += ["a", "b", "a"]`.
78+
pub(crate) fn duplicate_entry_in_dunder_all_aug_assign(
79+
checker: &Checker,
80+
node: &ast::StmtAugAssign,
81+
) {
82+
if node.op.is_add() {
83+
duplicate_entry_in_dunder_all(checker, &node.target, &node.value);
84+
}
85+
}
86+
87+
/// Apply RUF068 to `__all__.extend()`.
88+
pub(crate) fn duplicate_entry_in_dunder_all_extend_call(
89+
checker: &Checker,
90+
ast::ExprCall {
91+
func,
92+
arguments: ast::Arguments { args, keywords, .. },
93+
..
94+
}: &ast::ExprCall,
95+
) {
96+
let ([value_passed], []) = (&**args, &**keywords) else {
97+
return;
98+
};
99+
let ast::Expr::Attribute(ast::ExprAttribute { value, attr, .. }) = &**func else {
100+
return;
101+
};
102+
if attr == "extend" {
103+
duplicate_entry_in_dunder_all(checker, value, value_passed);
104+
}
105+
}
106+
107+
/// Apply RUF068 to a `StmtAnnAssign` AST node.
108+
/// For example: `__all__: list[str] = ["a", "b", "a"]`.
109+
pub(crate) fn duplicate_entry_in_dunder_all_ann_assign(
110+
checker: &Checker,
111+
node: &ast::StmtAnnAssign,
112+
) {
113+
if let Some(value) = &node.value {
114+
duplicate_entry_in_dunder_all(checker, &node.target, value);
115+
}
116+
}
117+
118+
/// RUF068
119+
/// This routine checks whether `__all__` contains duplicated entries, and emits
120+
/// a violation if it does.
121+
fn duplicate_entry_in_dunder_all(checker: &Checker, target: &ast::Expr, value: &ast::Expr) {
122+
let ast::Expr::Name(ast::ExprName { id, .. }) = target else {
123+
return;
124+
};
125+
126+
if id != "__all__" {
127+
return;
128+
}
129+
130+
// We're only interested in `__all__` in the global scope
131+
if !checker.semantic().current_scope().kind.is_module() {
132+
return;
133+
}
134+
135+
let elts = match value {
136+
ast::Expr::List(ast::ExprList { elts, .. }) => elts,
137+
ast::Expr::Tuple(ast::ExprTuple { elts, .. }) => elts,
138+
_ => return,
139+
};
140+
141+
// It's impossible to have duplicates if there is one or no element
142+
if elts.len() <= 1 {
143+
return;
144+
}
145+
146+
let mut deduplicated_elts = FxHashMap::with_capacity_and_hasher(elts.len(), FxBuildHasher);
147+
let source = checker.locator().contents();
148+
149+
for (index, expr) in elts.iter().enumerate() {
150+
let Some(string_value) = expr.as_string_literal_expr() else {
151+
// In the example below we're ignoring `foo`:
152+
// __all__ = [foo, "bar", "bar"]
153+
continue;
154+
};
155+
156+
let name = string_value.value.to_str();
157+
158+
if let Some(previous_expr) = deduplicated_elts.insert(name, expr) {
159+
let mut diagnostic = checker.report_diagnostic(DuplicateEntryInDunderAll, expr.range());
160+
161+
diagnostic.secondary_annotation(
162+
format_args!("previous occurrence of `{name}` here"),
163+
previous_expr,
164+
);
165+
166+
diagnostic.set_primary_message(format_args!("`{name}` duplicated here"));
167+
168+
diagnostic.try_set_fix(|| {
169+
edits::remove_member(elts, index, source).map(|edit| {
170+
let applicability = if checker.comment_ranges().intersects(edit.range()) {
171+
Applicability::Unsafe
172+
} else {
173+
Applicability::Safe
174+
};
175+
Fix::applicable_edit(edit, applicability)
176+
})
177+
});
178+
}
179+
}
180+
}

0 commit comments

Comments
 (0)