Skip to content

Python location transform and related refactoring #13

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 7 commits into from
May 10, 2023
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Let located only for python located stuff
  • Loading branch information
youknowone committed May 10, 2023
commit 1d366d52ab96b6fb01923f38e02e34ae9ec8444c
14 changes: 7 additions & 7 deletions ast/asdl_rs.py
Original file line number Diff line number Diff line change
Expand Up @@ -391,9 +391,9 @@ def visitModule(self, mod, depth):
)
self.emit(
"""
fn map_located<T>(&mut self, located: Attributed<T, U>) -> Result<Attributed<T, Self::TargetU>, Self::Error> {
let custom = self.map_user(located.custom)?;
Ok(Attributed { range: located.range, custom, node: located.node })
fn map_attributed<T>(&mut self, attributed: Attributed<T, U>) -> Result<Attributed<T, Self::TargetU>, Self::Error> {
let custom = self.map_user(attributed.custom)?;
Ok(Attributed { range: attributed.range, custom, node: attributed.node })
}""",
depth + 1,
)
Expand Down Expand Up @@ -423,11 +423,11 @@ def visitType(self, type, depth):
class FoldImplVisitor(EmitVisitor):
def visitModule(self, mod, depth):
self.emit(
"fn fold_located<U, F: Fold<U> + ?Sized, T, MT>(folder: &mut F, node: Attributed<T, U>, f: impl FnOnce(&mut F, T) -> Result<MT, F::Error>) -> Result<Attributed<MT, F::TargetU>, F::Error> {",
"fn fold_attributed<U, F: Fold<U> + ?Sized, T, MT>(folder: &mut F, node: Attributed<T, U>, f: impl FnOnce(&mut F, T) -> Result<MT, F::Error>) -> Result<Attributed<MT, F::TargetU>, F::Error> {",
depth,
)
self.emit(
"let node = folder.map_located(node)?; Ok(Attributed { custom: node.custom, range: node.range, node: f(folder, node.node)? })",
"let node = folder.map_attributed(node)?; Ok(Attributed { custom: node.custom, range: node.range, node: f(folder, node.node)? })",
depth + 1,
)
self.emit("}", depth)
Expand Down Expand Up @@ -459,7 +459,7 @@ def visitSum(self, sum, name, depth):
depth,
)
if typeinfo.has_attributes:
self.emit("fold_located(folder, node, |folder, node| {", depth)
self.emit("fold_attributed(folder, node, |folder, node| {", depth)

self.emit("match node {", depth + 1)
for cons in sum.types:
Expand Down Expand Up @@ -501,7 +501,7 @@ def visitProduct(self, product, name, depth):
depth,
)
if has_attributes:
self.emit("fold_located(folder, node, |folder, node| {", depth)
self.emit("fold_attributed(folder, node, |folder, node| {", depth)
rustname = structname + "Data"
else:
rustname = structname
Expand Down
28 changes: 14 additions & 14 deletions ast/src/gen/generic.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1020,15 +1020,15 @@ pub mod fold {
type Error;
fn map_user(&mut self, user: U) -> Result<Self::TargetU, Self::Error>;

fn map_located<T>(
fn map_attributed<T>(
&mut self,
located: Attributed<T, U>,
attributed: Attributed<T, U>,
) -> Result<Attributed<T, Self::TargetU>, Self::Error> {
let custom = self.map_user(located.custom)?;
let custom = self.map_user(attributed.custom)?;
Ok(Attributed {
range: located.range,
range: attributed.range,
custom,
node: located.node,
node: attributed.node,
})
}

Expand Down Expand Up @@ -1114,12 +1114,12 @@ pub mod fold {
fold_type_ignore(self, node)
}
}
fn fold_located<U, F: Fold<U> + ?Sized, T, MT>(
fn fold_attributed<U, F: Fold<U> + ?Sized, T, MT>(
folder: &mut F,
node: Attributed<T, U>,
f: impl FnOnce(&mut F, T) -> Result<MT, F::Error>,
) -> Result<Attributed<MT, F::TargetU>, F::Error> {
let node = folder.map_located(node)?;
let node = folder.map_attributed(node)?;
Ok(Attributed {
custom: node.custom,
range: node.range,
Expand Down Expand Up @@ -1171,7 +1171,7 @@ pub mod fold {
#[allow(unused)] folder: &mut F,
node: Stmt<U>,
) -> Result<Stmt<F::TargetU>, F::Error> {
fold_located(folder, node, |folder, node| match node {
fold_attributed(folder, node, |folder, node| match node {
StmtKind::FunctionDef(StmtFunctionDef {
name,
args,
Expand Down Expand Up @@ -1375,7 +1375,7 @@ pub mod fold {
#[allow(unused)] folder: &mut F,
node: Expr<U>,
) -> Result<Expr<F::TargetU>, F::Error> {
fold_located(folder, node, |folder, node| match node {
fold_attributed(folder, node, |folder, node| match node {
ExprKind::BoolOp(ExprBoolOp { op, values }) => Ok(ExprKind::BoolOp(ExprBoolOp {
op: Foldable::fold(op, folder)?,
values: Foldable::fold(values, folder)?,
Expand Down Expand Up @@ -1675,7 +1675,7 @@ pub mod fold {
#[allow(unused)] folder: &mut F,
node: Excepthandler<U>,
) -> Result<Excepthandler<F::TargetU>, F::Error> {
fold_located(folder, node, |folder, node| match node {
fold_attributed(folder, node, |folder, node| match node {
ExcepthandlerKind::ExceptHandler(ExcepthandlerExceptHandler { type_, name, body }) => {
Ok(ExcepthandlerKind::ExceptHandler(
ExcepthandlerExceptHandler {
Expand Down Expand Up @@ -1732,7 +1732,7 @@ pub mod fold {
#[allow(unused)] folder: &mut F,
node: Arg<U>,
) -> Result<Arg<F::TargetU>, F::Error> {
fold_located(folder, node, |folder, node| {
fold_attributed(folder, node, |folder, node| {
let ArgData {
arg,
annotation,
Expand All @@ -1758,7 +1758,7 @@ pub mod fold {
#[allow(unused)] folder: &mut F,
node: Keyword<U>,
) -> Result<Keyword<F::TargetU>, F::Error> {
fold_located(folder, node, |folder, node| {
fold_attributed(folder, node, |folder, node| {
let KeywordData { arg, value } = node;
Ok(KeywordData {
arg: Foldable::fold(arg, folder)?,
Expand All @@ -1779,7 +1779,7 @@ pub mod fold {
#[allow(unused)] folder: &mut F,
node: Alias<U>,
) -> Result<Alias<F::TargetU>, F::Error> {
fold_located(folder, node, |folder, node| {
fold_attributed(folder, node, |folder, node| {
let AliasData { name, asname } = node;
Ok(AliasData {
name: Foldable::fold(name, folder)?,
Expand Down Expand Up @@ -1846,7 +1846,7 @@ pub mod fold {
#[allow(unused)] folder: &mut F,
node: Pattern<U>,
) -> Result<Pattern<F::TargetU>, F::Error> {
fold_located(folder, node, |folder, node| match node {
fold_attributed(folder, node, |folder, node| match node {
PatternKind::MatchValue(PatternMatchValue { value }) => {
Ok(PatternKind::MatchValue(PatternMatchValue {
value: Foldable::fold(value, folder)?,
Expand Down
4 changes: 2 additions & 2 deletions ast/src/source_locator.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,10 @@ impl crate::fold::Fold<()> for SourceLocator<'_> {

#[cold]
fn map_user(&mut self, _user: ()) -> Result<Self::TargetU, Self::Error> {
unreachable!("implemented map_located");
unreachable!("implemented map_attributed");
}

fn map_located<T>(
fn map_attributed<T>(
&mut self,
node: Attributed<T, ()>,
) -> Result<Attributed<T, Self::TargetU>, Self::Error> {
Expand Down
4 changes: 2 additions & 2 deletions parser/src/lexer.rs
Original file line number Diff line number Diff line change
Expand Up @@ -203,12 +203,12 @@ pub type LexResult = Result<Spanned, LexicalError>;
/// ```
#[inline]
pub fn lex(source: &str, mode: Mode) -> impl Iterator<Item = LexResult> + '_ {
lex_located(source, mode, TextSize::default())
lex_starts_at(source, mode, TextSize::default())
}

/// Create a new lexer from a source string, starting at a given location.
/// You probably want to use [`lex`] instead.
pub fn lex_located(
pub fn lex_starts_at(
source: &str,
mode: Mode,
start_offset: TextSize,
Expand Down
2 changes: 1 addition & 1 deletion parser/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,7 @@ mod string;
mod token;

pub use parser::{
parse, parse_expression, parse_expression_located, parse_located, parse_program, parse_tokens,
parse, parse_expression, parse_expression_at, parse_program, parse_starts_at, parse_tokens,
ParseError, ParseErrorType,
};
pub use string::FStringErrorType;
Expand Down
20 changes: 10 additions & 10 deletions parser/src/parser.rs
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ pub fn parse_program(source: &str, source_path: &str) -> Result<ast::Suite, Pars
///
/// ```
pub fn parse_expression(source: &str, path: &str) -> Result<ast::Expr, ParseError> {
parse_expression_located(source, path, TextSize::default())
parse_expression_at(source, path, TextSize::default())
}

/// Parses a Python expression from a given location.
Expand All @@ -84,17 +84,17 @@ pub fn parse_expression(source: &str, path: &str) -> Result<ast::Expr, ParseErro
/// somewhat silly, location:
///
/// ```
/// use rustpython_parser::{text_size::TextSize, parse_expression_located};
/// use rustpython_parser::{text_size::TextSize, parse_expression_at};
///
/// let expr = parse_expression_located("1 + 2", "<embedded>", TextSize::from(400));
/// let expr = parse_expression_at("1 + 2", "<embedded>", TextSize::from(400));
/// assert!(expr.is_ok());
/// ```
pub fn parse_expression_located(
pub fn parse_expression_at(
source: &str,
path: &str,
offset: TextSize,
) -> Result<ast::Expr, ParseError> {
parse_located(source, Mode::Expression, path, offset).map(|top| match top {
parse_starts_at(source, Mode::Expression, path, offset).map(|top| match top {
ast::Mod::Expression(ast::ModExpression { body }) => *body,
_ => unreachable!(),
})
Expand Down Expand Up @@ -132,7 +132,7 @@ pub fn parse_expression_located(
/// assert!(program.is_ok());
/// ```
pub fn parse(source: &str, mode: Mode, source_path: &str) -> Result<ast::Mod, ParseError> {
parse_located(source, mode, source_path, TextSize::default())
parse_starts_at(source, mode, source_path, TextSize::default())
}

/// Parse the given Python source code using the specified [`Mode`] and [`Location`].
Expand All @@ -143,7 +143,7 @@ pub fn parse(source: &str, mode: Mode, source_path: &str) -> Result<ast::Mod, Pa
/// # Example
///
/// ```
/// use rustpython_parser::{text_size::TextSize, Mode, parse_located};
/// use rustpython_parser::{text_size::TextSize, Mode, parse_starts_at};
///
/// let source = r#"
/// def fib(i):
Expand All @@ -154,16 +154,16 @@ pub fn parse(source: &str, mode: Mode, source_path: &str) -> Result<ast::Mod, Pa
///
/// print(fib(42))
/// "#;
/// let program = parse_located(source, Mode::Module, "<embedded>", TextSize::from(0));
/// let program = parse_starts_at(source, Mode::Module, "<embedded>", TextSize::from(0));
/// assert!(program.is_ok());
/// ```
pub fn parse_located(
pub fn parse_starts_at(
source: &str,
mode: Mode,
source_path: &str,
offset: TextSize,
) -> Result<ast::Mod, ParseError> {
let lxr = lexer::lex_located(source, mode, offset);
let lxr = lexer::lex_starts_at(source, mode, offset);
parse_tokens(lxr, mode, source_path)
}

Expand Down
4 changes: 2 additions & 2 deletions parser/src/string.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
use crate::{
ast::{self, Constant, Expr, ExprKind},
lexer::{LexicalError, LexicalErrorType},
parser::{parse_expression_located, LalrpopError, ParseError, ParseErrorType},
parser::{parse_expression_at, LalrpopError, ParseError, ParseErrorType},
token::{StringKind, Tok},
};
use itertools::Itertools;
Expand Down Expand Up @@ -575,7 +575,7 @@ impl<'a> StringParser<'a> {
fn parse_fstring_expr(source: &str, location: TextSize) -> Result<Expr, ParseError> {
let fstring_body = format!("({source})");
let start = location - TextSize::from(1);
parse_expression_located(&fstring_body, "<fstring>", start)
parse_expression_at(&fstring_body, "<fstring>", start)
}

fn parse_string(
Expand Down