Pull in RustPython parser

Cargo.toml

@@ -51,12 +51,6 @@ wsl = { version = "0.1.0" }
 # v1.0.1
 libcst = { git = "https://github.com/Instagram/LibCST.git", rev = "3cacca1a1029f05707e50703b49fe3dd860aa839", default-features = false }
 
-ruff_text_size = { git = "https://github.com/astral-sh/RustPython-Parser.git", rev = "593b46be5e0336fe01917f8ef400bd12d81df8c1" }
-rustpython-ast = { git = "https://github.com/astral-sh/RustPython-Parser.git", rev = "593b46be5e0336fe01917f8ef400bd12d81df8c1" }
-rustpython-format = { git = "https://github.com/astral-sh/RustPython-Parser.git", rev = "593b46be5e0336fe01917f8ef400bd12d81df8c1" }
-rustpython-literal = { git = "https://github.com/astral-sh/RustPython-Parser.git", rev = "593b46be5e0336fe01917f8ef400bd12d81df8c1" }
-rustpython-parser = { git = "https://github.com/astral-sh/RustPython-Parser.git", rev = "593b46be5e0336fe01917f8ef400bd12d81df8c1" }
-
 [profile.release]
 lto = "fat"
 codegen-units = 1
@@ -69,7 +63,7 @@ opt-level = 3
 
 # Reduce complexity of a parser function that would trigger a locals limit in a wasm tool.
 # https://github.com/bytecodealliance/wasm-tools/blob/b5c3d98e40590512a3b12470ef358d5c7b983b15/crates/wasmparser/src/limits.rs#L29
-[profile.dev.package.rustpython-parser]
+[profile.dev.package.ruff_python_parser]
 opt-level = 1
 
 # Use the `--profile release-debug` flag to show symbols in release mode.

crates/ruff/Cargo.toml

@@ -22,12 +22,13 @@ ruff_macros = { path = "../ruff_macros" }
 ruff_python_ast = { path = "../ruff_python_ast", features = ["serde"] }
 ruff_python_codegen = { path = "../ruff_python_codegen" }
 ruff_python_index = { path = "../ruff_python_index" }
+ruff_python_literal = { path = "../ruff_python_literal" }
 ruff_python_semantic = { path = "../ruff_python_semantic" }
 ruff_python_stdlib = { path = "../ruff_python_stdlib" }
 ruff_python_trivia = { path = "../ruff_python_trivia" }
 ruff_python_parser = { path = "../ruff_python_parser" }
 ruff_source_file = { path = "../ruff_source_file", features = ["serde"] }
-ruff_text_size = { workspace = true }
+ruff_text_size = { path = "../ruff_text_size" }
 
 annotate-snippets = { version = "0.9.1", features = ["color"] }
 anyhow = { workspace = true }
@@ -61,9 +62,8 @@ quick-junit = { version = "0.3.2" }
 regex = { workspace = true }
 result-like = { version = "0.4.6" }
 rustc-hash = { workspace = true }
-rustpython-format = { workspace = true }
-rustpython-parser = { workspace = true }
-rustpython-ast = { workspace = true }
+
+
 schemars = { workspace = true, optional = true }
 semver = { version = "1.0.16" }
 serde = { workspace = true }

crates/ruff/src/autofix/codemods.rs

@@ -4,7 +4,7 @@ use anyhow::{bail, Result};
 use libcst_native::{
     Codegen, CodegenState, ImportNames, ParenthesizableWhitespace, SmallStatement, Statement,
 };
-use rustpython_ast::{Ranged, Stmt};
+use ruff_python_ast::{Ranged, Stmt};
 
 use ruff_python_codegen::Stylist;
 use ruff_source_file::Locator;

crates/ruff/src/autofix/edits.rs

@@ -1,8 +1,8 @@
 //! Interface for generating autofix edits from higher-level actions (e.g., "remove an argument").
 use anyhow::{bail, Result};
+use ruff_python_ast::{self as ast, ExceptHandler, Expr, Keyword, Ranged, Stmt};
+use ruff_python_parser::{lexer, Mode};
 use ruff_text_size::{TextLen, TextRange, TextSize};
-use rustpython_ast::{self as ast, ExceptHandler, Expr, Keyword, Ranged, Stmt};
-use rustpython_parser::{lexer, Mode};
 
 use ruff_diagnostics::Edit;
 use ruff_python_codegen::Stylist;
@@ -294,9 +294,9 @@ fn next_stmt_break(semicolon: TextSize, locator: &Locator) -> TextSize {
 #[cfg(test)]
 mod tests {
     use anyhow::Result;
+    use ruff_python_ast::{Ranged, Suite};
+    use ruff_python_parser::Parse;
     use ruff_text_size::TextSize;
-    use rustpython_ast::{Ranged, Suite};
-    use rustpython_parser::Parse;
 
     use ruff_source_file::Locator;
 

crates/ruff/src/checkers/ast/analyze/argument.rs

@@ -1,4 +1,4 @@
-use rustpython_ast::{Arg, Ranged};
+use ruff_python_ast::{Arg, Ranged};
 
 use crate::checkers::ast::Checker;
 use crate::codes::Rule;

crates/ruff/src/checkers/ast/analyze/arguments.rs

@@ -1,4 +1,4 @@
-use rustpython_ast::Arguments;
+use ruff_python_ast::Arguments;
 
 use crate::checkers::ast::Checker;
 use crate::codes::Rule;

crates/ruff/src/checkers/ast/analyze/comprehension.rs

@@ -1,4 +1,4 @@
-use rustpython_ast::Comprehension;
+use ruff_python_ast::Comprehension;
 
 use crate::checkers::ast::Checker;
 use crate::codes::Rule;

crates/ruff/src/checkers/ast/analyze/deferred_for_loops.rs

@@ -1,4 +1,4 @@
-use rustpython_ast::{self as ast, Stmt};
+use ruff_python_ast::{self as ast, Stmt};
 
 use crate::checkers::ast::Checker;
 use crate::codes::Rule;

crates/ruff/src/checkers/ast/analyze/definitions.rs

@@ -1,6 +1,6 @@
 use ruff_python_ast::str::raw_contents_range;
+use ruff_python_ast::Ranged;
 use ruff_text_size::TextRange;
-use rustpython_ast::Ranged;
 
 use ruff_python_semantic::{BindingKind, ContextualizedDefinition, Export};
 

crates/ruff/src/checkers/ast/analyze/except_handler.rs

@@ -1,4 +1,4 @@
-use rustpython_ast::{self as ast, ExceptHandler, Ranged};
+use ruff_python_ast::{self as ast, ExceptHandler, Ranged};
 
 use crate::checkers::ast::Checker;
 use crate::registry::Rule;

crates/ruff/src/checkers/ast/analyze/expression.rs

@@ -1,5 +1,5 @@
-use rustpython_ast::{self as ast, Constant, Expr, ExprContext, Operator, Ranged};
-use rustpython_format::cformat::{CFormatError, CFormatErrorType};
+use ruff_python_ast::{self as ast, Constant, Expr, ExprContext, Operator, Ranged};
+use ruff_python_literal::cformat::{CFormatError, CFormatErrorType};
 
 use ruff_diagnostics::Diagnostic;
 

crates/ruff/src/checkers/ast/analyze/module.rs

@@ -1,4 +1,4 @@
-use rustpython_ast::Suite;
+use ruff_python_ast::Suite;
 
 use crate::checkers::ast::Checker;
 use crate::codes::Rule;

crates/ruff/src/checkers/ast/analyze/statement.rs

@@ -1,4 +1,4 @@
-use rustpython_ast::{self as ast, Expr, Ranged, Stmt};
+use ruff_python_ast::{self as ast, Expr, Ranged, Stmt};
 
 use ruff_diagnostics::Diagnostic;
 use ruff_python_ast::helpers;

crates/ruff/src/checkers/ast/analyze/suite.rs

@@ -1,4 +1,4 @@
-use rustpython_ast::Stmt;
+use ruff_python_ast::Stmt;
 
 use crate::checkers::ast::Checker;
 use crate::codes::Rule;

crates/ruff/src/checkers/ast/deferred.rs

@@ -1,5 +1,5 @@
+use ruff_python_ast::Expr;
 use ruff_text_size::TextRange;
-use rustpython_ast::Expr;
 
 use ruff_python_semantic::{ScopeId, Snapshot};
 

crates/ruff/src/checkers/ast/mod.rs

@@ -30,11 +30,11 @@ use std::path::Path;
 
 use itertools::Itertools;
 use log::error;
-use ruff_text_size::{TextRange, TextSize};
-use rustpython_ast::{
+use ruff_python_ast::{
     self as ast, Arg, ArgWithDefault, Arguments, Comprehension, Constant, ElifElseClause,
     ExceptHandler, Expr, ExprContext, Keyword, Pattern, Ranged, Stmt, Suite, UnaryOp,
 };
+use ruff_text_size::{TextRange, TextSize};
 
 use ruff_diagnostics::{Diagnostic, IsolationLevel};
 use ruff_python_ast::all::{extract_all_names, DunderAllFlags};

crates/ruff/src/checkers/imports.rs

@@ -2,7 +2,7 @@
 use std::borrow::Cow;
 use std::path::Path;
 
-use rustpython_ast::{self as ast, Ranged, Stmt, Suite};
+use ruff_python_ast::{self as ast, Ranged, Stmt, Suite};
 
 use ruff_diagnostics::Diagnostic;
 use ruff_python_ast::helpers::to_module_path;

crates/ruff/src/checkers/logical_lines.rs

@@ -1,9 +1,9 @@
+use ruff_python_parser::lexer::LexResult;
 use ruff_text_size::TextRange;
-use rustpython_parser::lexer::LexResult;
 
 use ruff_diagnostics::{Diagnostic, DiagnosticKind};
 use ruff_python_codegen::Stylist;
-use ruff_python_parser::token_kind::TokenKind;
+use ruff_python_parser::TokenKind;
 use ruff_source_file::Locator;
 
 use crate::registry::{AsRule, Rule};

crates/ruff/src/checkers/noqa.rs

@@ -3,8 +3,8 @@
 use std::path::Path;
 
 use itertools::Itertools;
+use ruff_python_ast::Ranged;
 use ruff_text_size::{TextLen, TextRange, TextSize};
-use rustpython_ast::Ranged;
 
 use ruff_diagnostics::{Diagnostic, Edit, Fix};
 use ruff_source_file::Locator;

crates/ruff/src/checkers/physical_lines.rs

@@ -99,8 +99,8 @@ pub(crate) fn check_physical_lines(
 
 #[cfg(test)]
 mod tests {
-    use rustpython_parser::lexer::lex;
-    use rustpython_parser::Mode;
+    use ruff_python_parser::lexer::lex;
+    use ruff_python_parser::Mode;
 
     use ruff_python_codegen::Stylist;
     use ruff_python_index::Indexer;

crates/ruff/src/checkers/tokens.rs

@@ -2,8 +2,8 @@
 
 use std::path::Path;
 
-use rustpython_parser::lexer::LexResult;
-use rustpython_parser::Tok;
+use ruff_python_parser::lexer::LexResult;
+use ruff_python_parser::Tok;
 
 use ruff_diagnostics::Diagnostic;
 use ruff_python_index::Indexer;

crates/ruff/src/directives.rs

@@ -3,9 +3,9 @@
 use std::str::FromStr;
 
 use bitflags::bitflags;
+use ruff_python_parser::lexer::LexResult;
+use ruff_python_parser::Tok;
 use ruff_text_size::{TextLen, TextRange, TextSize};
-use rustpython_parser::lexer::LexResult;
-use rustpython_parser::Tok;
 
 use ruff_python_index::Indexer;
 use ruff_source_file::Locator;
@@ -349,9 +349,9 @@ impl TodoDirectiveKind {
 
 #[cfg(test)]
 mod tests {
+    use ruff_python_parser::lexer::LexResult;
+    use ruff_python_parser::{lexer, Mode};
     use ruff_text_size::{TextLen, TextRange, TextSize};
-    use rustpython_parser::lexer::LexResult;
-    use rustpython_parser::{lexer, Mode};
 
     use ruff_python_index::Indexer;
     use ruff_source_file::Locator;

crates/ruff/src/doc_lines.rs

@@ -3,10 +3,10 @@
 
 use std::iter::FusedIterator;
 
+use ruff_python_ast::{self as ast, Constant, Expr, Ranged, Stmt, Suite};
+use ruff_python_parser::lexer::LexResult;
+use ruff_python_parser::Tok;
 use ruff_text_size::TextSize;
-use rustpython_ast::{self as ast, Constant, Expr, Ranged, Stmt, Suite};
-use rustpython_parser::lexer::LexResult;
-use rustpython_parser::Tok;
 
 use ruff_python_ast::statement_visitor::{walk_stmt, StatementVisitor};
 use ruff_source_file::{Locator, UniversalNewlineIterator};

crates/ruff/src/docstrings/extraction.rs

@@ -1,6 +1,6 @@
 //! Extract docstrings from an AST.
 
-use rustpython_ast::{self as ast, Constant, Expr, Stmt};
+use ruff_python_ast::{self as ast, Constant, Expr, Stmt};
 
 use ruff_python_semantic::{Definition, DefinitionId, Definitions, Member, MemberKind};
 

crates/ruff/src/docstrings/mod.rs

@@ -1,8 +1,8 @@
 use std::fmt::{Debug, Formatter};
 use std::ops::Deref;
 
+use ruff_python_ast::{Expr, Ranged};
 use ruff_text_size::{TextRange, TextSize};
-use rustpython_ast::{Expr, Ranged};
 
 use ruff_python_semantic::Definition;
 

crates/ruff/src/importer/insertion.rs

@@ -1,9 +1,9 @@
 //! Insert statements into Python code.
 use std::ops::Add;
 
+use ruff_python_ast::{Ranged, Stmt};
+use ruff_python_parser::{lexer, Mode, Tok};
 use ruff_text_size::TextSize;
-use rustpython_ast::{Ranged, Stmt};
-use rustpython_parser::{lexer, Mode, Tok};
 
 use ruff_diagnostics::Edit;
 use ruff_python_ast::helpers::is_docstring_stmt;
@@ -299,10 +299,10 @@ fn match_leading_semicolon(s: &str) -> Option<TextSize> {
 #[cfg(test)]
 mod tests {
     use anyhow::Result;
+    use ruff_python_ast::Suite;
+    use ruff_python_parser::lexer::LexResult;
+    use ruff_python_parser::Parse;
     use ruff_text_size::TextSize;
-    use rustpython_ast::Suite;
-    use rustpython_parser::lexer::LexResult;
-    use rustpython_parser::Parse;
 
     use ruff_python_codegen::Stylist;
     use ruff_source_file::{LineEnding, Locator};

crates/ruff/src/importer/mod.rs

@@ -7,8 +7,8 @@ use std::error::Error;
 
 use anyhow::Result;
 use libcst_native::{ImportAlias, Name, NameOrAttribute};
+use ruff_python_ast::{self as ast, Ranged, Stmt, Suite};
 use ruff_text_size::TextSize;
-use rustpython_ast::{self as ast, Ranged, Stmt, Suite};
 
 use ruff_diagnostics::Edit;
 use ruff_python_ast::imports::{AnyImport, Import, ImportFrom};

crates/ruff/src/jupyter/notebook.rs

@@ -10,10 +10,10 @@ use serde::Serialize;
 use serde_json::error::Category;
 
 use ruff_diagnostics::Diagnostic;
+use ruff_python_parser::lexer::lex;
+use ruff_python_parser::Mode;
 use ruff_source_file::{NewlineWithTrailingNewline, UniversalNewlineIterator};
 use ruff_text_size::{TextRange, TextSize};
-use rustpython_parser::lexer::lex;
-use rustpython_parser::Mode;
 
 use crate::autofix::source_map::{SourceMap, SourceMarker};
 use crate::jupyter::index::JupyterIndex;

crates/ruff/src/lex/docstring_detection.rs

@@ -4,7 +4,7 @@
 //!
 //! TODO(charlie): Consolidate with the existing AST-based docstring extraction.
 
-use rustpython_parser::Tok;
+use ruff_python_parser::Tok;
 
 #[derive(Default, Copy, Clone)]
 enum State {

crates/ruff/src/linter.rs

@@ -6,9 +6,9 @@ use anyhow::{anyhow, Result};
 use colored::Colorize;
 use itertools::Itertools;
 use log::error;
+use ruff_python_parser::lexer::LexResult;
+use ruff_python_parser::ParseError;
 use rustc_hash::FxHashMap;
-use rustpython_parser::lexer::LexResult;
-use rustpython_parser::ParseError;
 
 use ruff_diagnostics::Diagnostic;
 use ruff_python_ast::imports::ImportMap;

crates/ruff/src/logging.rs

@@ -7,7 +7,7 @@ use colored::Colorize;
 use fern;
 use log::Level;
 use once_cell::sync::Lazy;
-use rustpython_parser::{ParseError, ParseErrorType};
+use ruff_python_parser::{ParseError, ParseErrorType};
 
 use ruff_source_file::{OneIndexed, SourceCode, SourceLocation};
 

crates/ruff/src/noqa.rs

@@ -8,8 +8,8 @@ use std::path::Path;
 use anyhow::Result;
 use itertools::Itertools;
 use log::warn;
+use ruff_python_ast::Ranged;
 use ruff_text_size::{TextLen, TextRange, TextSize};
-use rustpython_ast::Ranged;
 
 use ruff_diagnostics::Diagnostic;
 use ruff_source_file::{LineEnding, Locator};

crates/ruff/src/rules/airflow/rules/task_variable_name.rs

@@ -1,10 +1,10 @@
-use rustpython_ast::{Expr, Ranged};
-use rustpython_parser::ast;
+use ruff_python_ast as ast;
+use ruff_python_ast::{Expr, Ranged};
 
 use ruff_diagnostics::{Diagnostic, Violation};
 use ruff_macros::{derive_message_formats, violation};
 use ruff_python_ast::helpers::find_keyword;
-use rustpython_ast::Constant;
+use ruff_python_ast::Constant;
 
 use crate::checkers::ast::Checker;
 

crates/ruff/src/rules/eradicate/detection.rs

@@ -1,8 +1,8 @@
 /// See: [eradicate.py](https://github.com/myint/eradicate/blob/98f199940979c94447a461d50d27862b118b282d/eradicate.py)
 use once_cell::sync::Lazy;
 use regex::Regex;
-use rustpython_ast::Suite;
-use rustpython_parser::Parse;
+use ruff_python_ast::Suite;
+use ruff_python_parser::Parse;
 
 static ALLOWLIST_REGEX: Lazy<Regex> = Lazy::new(|| {
     Regex::new(

crates/ruff/src/rules/flake8_2020/helpers.rs

@@ -1,4 +1,4 @@
-use rustpython_ast::Expr;
+use ruff_python_ast::Expr;
 
 use ruff_python_semantic::SemanticModel;