Build CommentRanges outside the parser

crates/ruff_benchmark/Cargo.toml

@@ -45,6 +45,7 @@ ruff_linter = { workspace = true }
 ruff_python_ast = { workspace = true }
 ruff_python_formatter = { workspace = true }
 ruff_python_parser = { workspace = true }
+ruff_python_trivia = { workspace = true }
 
 [lints]
 workspace = true

crates/ruff_benchmark/benches/formatter.rs

@@ -6,6 +6,7 @@ use ruff_benchmark::criterion::{
 use ruff_benchmark::{TestCase, TestFile, TestFileDownloadError};
 use ruff_python_formatter::{format_module_ast, PreviewMode, PyFormatOptions};
 use ruff_python_parser::{parse, Mode};
+use ruff_python_trivia::CommentRanges;
 
 #[cfg(target_os = "windows")]
 #[global_allocator]
@@ -54,11 +55,14 @@ fn benchmark_formatter(criterion: &mut Criterion) {
                 let parsed =
                     parse(case.code(), Mode::Module).expect("Input should be a valid Python code");
 
+                let comment_ranges = CommentRanges::from(parsed.tokens());
+
                 b.iter(|| {
                     let options = PyFormatOptions::from_extension(Path::new(case.name()))
                         .with_preview(PreviewMode::Enabled);
-                    let formatted = format_module_ast(&parsed, case.code(), options)
-                        .expect("Formatting to succeed");
+                    let formatted =
+                        format_module_ast(&parsed, &comment_ranges, case.code(), options)
+                            .expect("Formatting to succeed");
 
                     formatted.print().expect("Printing to succeed")
                 });

crates/ruff_linter/src/checkers/ast/mod.rs

@@ -329,7 +329,7 @@ impl<'a> Checker<'a> {
 
     /// Returns the [`CommentRanges`] for the parsed source code.
     pub(crate) fn comment_ranges(&self) -> &'a CommentRanges {
-        self.parsed.comment_ranges()
+        self.indexer.comment_ranges()
     }
 
     /// Returns the [`Tokens`] for the parsed type annotation if the checker is in a typing context

crates/ruff_linter/src/checkers/imports.rs

@@ -51,7 +51,7 @@ pub(crate) fn check_imports(
                     settings,
                     package,
                     source_type,
-                    parsed,
+                    parsed.tokens(),
                 ) {
                     diagnostics.push(diagnostic);
                 }

crates/ruff_linter/src/checkers/physical_lines.rs

@@ -3,7 +3,6 @@
 use ruff_diagnostics::Diagnostic;
 use ruff_python_codegen::Stylist;
 use ruff_python_index::Indexer;
-use ruff_python_trivia::CommentRanges;
 use ruff_source_file::{Locator, UniversalNewlines};
 use ruff_text_size::TextSize;
 
@@ -20,7 +19,6 @@ pub(crate) fn check_physical_lines(
     locator: &Locator,
     stylist: &Stylist,
     indexer: &Indexer,
-    comment_ranges: &CommentRanges,
     doc_lines: &[TextSize],
     settings: &LinterSettings,
 ) -> Vec<Diagnostic> {
@@ -37,6 +35,7 @@ pub(crate) fn check_physical_lines(
     let enforce_copyright_notice = settings.rules.enabled(Rule::MissingCopyrightNotice);
 
     let mut doc_lines_iter = doc_lines.iter().peekable();
+    let comment_ranges = indexer.comment_ranges();
 
     for line in locator.contents().universal_newlines() {
         while doc_lines_iter
@@ -115,7 +114,6 @@ mod tests {
                 &locator,
                 &stylist,
                 &indexer,
-                parsed.comment_ranges(),
                 &[],
                 &LinterSettings {
                     pycodestyle: pycodestyle::settings::Settings {

crates/ruff_linter/src/checkers/tokens.rs

@@ -3,12 +3,12 @@
 use std::path::Path;
 
 use ruff_notebook::CellOffsets;
-use ruff_python_ast::{ModModule, PySourceType};
+use ruff_python_ast::PySourceType;
 use ruff_python_codegen::Stylist;
 
 use ruff_diagnostics::Diagnostic;
 use ruff_python_index::Indexer;
-use ruff_python_parser::Parsed;
+use ruff_python_parser::Tokens;
 use ruff_source_file::Locator;
 use ruff_text_size::Ranged;
 
@@ -23,7 +23,7 @@ use crate::settings::LinterSettings;
 
 #[allow(clippy::too_many_arguments)]
 pub(crate) fn check_tokens(
-    parsed: &Parsed<ModModule>,
+    tokens: &Tokens,
     path: &Path,
     locator: &Locator,
     indexer: &Indexer,
@@ -33,9 +33,7 @@ pub(crate) fn check_tokens(
     cell_offsets: Option<&CellOffsets>,
 ) -> Vec<Diagnostic> {
     let mut diagnostics: Vec<Diagnostic> = vec![];
-
-    let tokens = parsed.tokens();
-    let comment_ranges = parsed.comment_ranges();
+    let comment_ranges = indexer.comment_ranges();
 
     if settings.rules.any_enabled(&[
         Rule::BlankLineBetweenMethods,

crates/ruff_linter/src/directives.rs

@@ -4,8 +4,7 @@ use std::iter::Peekable;
 use std::str::FromStr;
 
 use bitflags::bitflags;
-use ruff_python_ast::ModModule;
-use ruff_python_parser::{Parsed, TokenKind, Tokens};
+use ruff_python_parser::{TokenKind, Tokens};
 use ruff_python_trivia::CommentRanges;
 use ruff_text_size::{Ranged, TextLen, TextRange, TextSize};
 
@@ -52,19 +51,19 @@ pub struct Directives {
 }
 
 pub fn extract_directives(
-    parsed: &Parsed<ModModule>,
+    tokens: &Tokens,
     flags: Flags,
     locator: &Locator,
     indexer: &Indexer,
 ) -> Directives {
     Directives {
         noqa_line_for: if flags.intersects(Flags::NOQA) {
-            extract_noqa_line_for(parsed.tokens(), locator, indexer)
+            extract_noqa_line_for(tokens, locator, indexer)
         } else {
             NoqaMapping::default()
         },
         isort: if flags.intersects(Flags::ISORT) {
-            extract_isort_directives(locator, parsed.comment_ranges())
+            extract_isort_directives(locator, indexer.comment_ranges())
         } else {
             IsortDirectives::default()
         },
@@ -380,6 +379,7 @@ impl TodoDirectiveKind {
 #[cfg(test)]
 mod tests {
     use ruff_python_parser::parse_module;
+    use ruff_python_trivia::CommentRanges;
     use ruff_text_size::{TextLen, TextRange, TextSize};
 
     use ruff_python_index::Indexer;
@@ -570,7 +570,8 @@ assert foo, \
     fn isort_directives(contents: &str) -> IsortDirectives {
         let parsed = parse_module(contents).unwrap();
         let locator = Locator::new(contents);
-        extract_isort_directives(&locator, parsed.comment_ranges())
+        let comment_ranges = CommentRanges::from(parsed.tokens());
+        extract_isort_directives(&locator, &comment_ranges)
     }
 
     #[test]

crates/ruff_linter/src/linter.rs

@@ -88,7 +88,7 @@ pub fn check_path(
     let mut error = None;
 
     let tokens = parsed.tokens();
-    let comment_ranges = parsed.comment_ranges();
+    let comment_ranges = indexer.comment_ranges();
 
     // Collect doc lines. This requires a rare mix of tokens (for comments) and AST
     // (for docstrings), which demands special-casing at this level.
@@ -105,7 +105,7 @@ pub fn check_path(
         .any(|rule_code| rule_code.lint_source().is_tokens())
     {
         diagnostics.extend(check_tokens(
-            parsed,
+            tokens,
             path,
             locator,
             indexer,
@@ -218,12 +218,7 @@ pub fn check_path(
         .any(|rule_code| rule_code.lint_source().is_physical_lines())
     {
         diagnostics.extend(check_physical_lines(
-            locator,
-            stylist,
-            indexer,
-            comment_ranges,
-            &doc_lines,
-            settings,
+            locator, stylist, indexer, &doc_lines, settings,
         ));
     }
 
@@ -385,7 +380,7 @@ pub fn add_noqa_to_path(
 
     // Extract the `# noqa` and `# isort: skip` directives from the source.
     let directives = directives::extract_directives(
-        &parsed,
+        parsed.tokens(),
         directives::Flags::from_settings(settings),
         &locator,
         &indexer,
@@ -428,7 +423,7 @@ pub fn add_noqa_to_path(
         path,
         &diagnostics,
         &locator,
-        parsed.comment_ranges(),
+        indexer.comment_ranges(),
         &settings.external,
         &directives.noqa_line_for,
         stylist.line_ending(),
@@ -459,7 +454,7 @@ pub fn lint_only(
 
     // Extract the `# noqa` and `# isort: skip` directives from the source.
     let directives = directives::extract_directives(
-        &parsed,
+        parsed.tokens(),
         directives::Flags::from_settings(settings),
         &locator,
         &indexer,
@@ -550,7 +545,7 @@ pub fn lint_fix<'a>(
 
         // Extract the `# noqa` and `# isort: skip` directives from the source.
         let directives = directives::extract_directives(
-            &parsed,
+            parsed.tokens(),
             directives::Flags::from_settings(settings),
             &locator,
             &indexer,

crates/ruff_linter/src/rules/isort/rules/organize_imports.rs

@@ -5,10 +5,10 @@ use itertools::{EitherOrBoth, Itertools};
 use ruff_diagnostics::{Diagnostic, Edit, Fix, FixAvailability, Violation};
 use ruff_macros::{derive_message_formats, violation};
 use ruff_python_ast::whitespace::trailing_lines_end;
-use ruff_python_ast::{ModModule, PySourceType, Stmt};
+use ruff_python_ast::{PySourceType, Stmt};
 use ruff_python_codegen::Stylist;
 use ruff_python_index::Indexer;
-use ruff_python_parser::Parsed;
+use ruff_python_parser::Tokens;
 use ruff_python_trivia::{leading_indentation, textwrap::indent, PythonWhitespace};
 use ruff_source_file::{Locator, UniversalNewlines};
 use ruff_text_size::{Ranged, TextRange};
@@ -89,7 +89,7 @@ pub(crate) fn organize_imports(
     settings: &LinterSettings,
     package: Option<&Path>,
     source_type: PySourceType,
-    parsed: &Parsed<ModModule>,
+    tokens: &Tokens,
 ) -> Option<Diagnostic> {
     let indentation = locator.slice(extract_indentation_range(&block.imports, locator));
     let indentation = leading_indentation(indentation);
@@ -108,7 +108,7 @@ pub(crate) fn organize_imports(
     let comments = comments::collect_comments(
         TextRange::new(range.start(), locator.full_line_end(range.end())),
         locator,
-        parsed.comment_ranges(),
+        indexer.comment_ranges(),
     );
 
     let trailing_line_end = if block.trailer.is_none() {
@@ -130,7 +130,7 @@ pub(crate) fn organize_imports(
         source_type,
         settings.target_version,
         &settings.isort,
-        parsed.tokens(),
+        tokens,
     );
 
     // Expand the span the entire range, including leading and trailing space.

crates/ruff_linter/src/rules/pyflakes/mod.rs

@@ -644,7 +644,7 @@ mod tests {
         let stylist = Stylist::from_tokens(parsed.tokens(), &locator);
         let indexer = Indexer::from_tokens(parsed.tokens(), &locator);
         let directives = directives::extract_directives(
-            &parsed,
+            parsed.tokens(),
             directives::Flags::from_settings(&settings),
             &locator,
             &indexer,

crates/ruff_linter/src/test.rs

@@ -114,7 +114,7 @@ pub(crate) fn test_contents<'a>(
     let stylist = Stylist::from_tokens(parsed.tokens(), &locator);
     let indexer = Indexer::from_tokens(parsed.tokens(), &locator);
     let directives = directives::extract_directives(
-        &parsed,
+        parsed.tokens(),
         directives::Flags::from_settings(settings),
         &locator,
         &indexer,
@@ -180,7 +180,7 @@ pub(crate) fn test_contents<'a>(
             let stylist = Stylist::from_tokens(parsed.tokens(), &locator);
             let indexer = Indexer::from_tokens(parsed.tokens(), &locator);
             let directives = directives::extract_directives(
-                &parsed,
+                parsed.tokens(),
                 directives::Flags::from_settings(settings),
                 &locator,
                 &indexer,

crates/ruff_python_formatter/src/cli.rs

@@ -8,6 +8,7 @@ use clap::{command, Parser, ValueEnum};
 use ruff_formatter::SourceCode;
 use ruff_python_ast::PySourceType;
 use ruff_python_parser::{parse, AsMode};
+use ruff_python_trivia::CommentRanges;
 use ruff_text_size::Ranged;
 
 use crate::comments::collect_comments;
@@ -62,14 +63,15 @@ pub fn format_and_debug_print(source: &str, cli: &Cli, source_path: &Path) -> Re
         });
 
     let source_code = SourceCode::new(source);
-    let formatted = format_module_ast(&parsed, source, options).context("Failed to format node")?;
+    let comment_ranges = CommentRanges::from(parsed.tokens());
+    let formatted = format_module_ast(&parsed, &comment_ranges, source, options)
+        .context("Failed to format node")?;
     if cli.print_ir {
         println!("{}", formatted.document().display(source_code));
     }
     if cli.print_comments {
         // Print preceding, following and enclosing nodes
-        let decorated_comments =
-            collect_comments(parsed.syntax(), source_code, parsed.comment_ranges());
+        let decorated_comments = collect_comments(parsed.syntax(), source_code, &comment_ranges);
         if !decorated_comments.is_empty() {
             println!("# Comment decoration: Range, Preceding, Following, Enclosing, Comment");
         }

crates/ruff_python_formatter/src/comments/mod.rs

@@ -482,11 +482,13 @@ mod tests {
     use ruff_formatter::SourceCode;
     use ruff_python_ast::{Mod, PySourceType};
     use ruff_python_parser::{parse, AsMode, Parsed};
+    use ruff_python_trivia::CommentRanges;
 
     use crate::comments::Comments;
 
     struct CommentsTestCase<'a> {
         parsed: Parsed<Mod>,
+        comment_ranges: CommentRanges,
         source_code: SourceCode<'a>,
     }
 
@@ -496,19 +498,17 @@ mod tests {
             let source_type = PySourceType::Python;
             let parsed =
                 parse(source, source_type.as_mode()).expect("Expect source to be valid Python");
+            let comment_ranges = CommentRanges::from(parsed.tokens());
 
             CommentsTestCase {
                 parsed,
+                comment_ranges,
                 source_code,
             }
         }
 
         fn to_comments(&self) -> Comments {
-            Comments::from_ast(
-                self.parsed.syntax(),
-                self.source_code,
-                self.parsed.comment_ranges(),
-            )
+            Comments::from_ast(self.parsed.syntax(), self.source_code, &self.comment_ranges)
         }
     }