numerous refactoring

- Split parser core and compiler core. Fix #14
- AST int type to `u32`
- Updated asdl_rs.py and update_asdl.sh fix #6
- Use `ruff_python_ast::SourceLocation` for Python source location. Deleted our own Location.
- Renamed ast::Located to ast::Attributed to distinguish terms for TextSize and SourceLocation
- `ast::<Node>`s for TextSize located ast. `ast::located::<Node>` for Python source located ast.
- And also strictly renaming `located` to refer only python location related interfaces.
- `SourceLocator` to convert locations.
- New `source-code` features of to disable python locations when unnecessary.
- Also including fully merging astral-sh/RustPython#4 closes #9

Cargo.toml

@@ -16,11 +16,12 @@ members = [
 ]
 
 [workspace.dependencies]
+rustpython-ast = { path = "ast", version = "0.2.0" }
+rustpython-parser-core = { path = "core", version = "0.2.0" }
+rustpython-literal = { path = "literal", version = "0.2.0" }
+
 ahash = "0.7.6"
 anyhow = "1.0.45"
-ascii = "1.0"
-bitflags = "1.3.2"
-bstr = "0.2.17"
 cfg-if = "1.0"
 insta = "1.14.0"
 itertools = "0.10.3"
@@ -32,7 +33,7 @@ rand = "0.8.5"
 serde = "1.0"
 static_assertions = "1.1"
 unicode_names2 = { version = "0.6.0", git = "https://github.com/youknowone/unicode_names2.git", rev = "4ce16aa85cbcdd9cc830410f1a72ef9a235f2fde" }
-ruff_python_ast = { git = "https://github.com/youknowone/ruff.git", rev = "583df5c1fa43b2732896219f8ab425116c140c80" }
+ruff_python_ast = { git = "https://github.com/youknowone/ruff.git", rev = "088958e8fda2f74f1ebf315c75db13c232409b13" }
 # ruff_python_ast = { path = "../ruff/crates/ruff_python_ast" }
 
 [profile.dev.package."*"]

ast/Cargo.toml

@@ -8,14 +8,14 @@ repository = "https://github.com/RustPython/RustPython"
 license = "MIT"
 
 [features]
-default = ["constant-optimization", "fold", "location"]
+default = ["constant-optimization", "fold", "source-code"]
 constant-optimization = ["fold"]
-location = []
+source-code = ["fold"]
 fold = []
 unparse = ["rustpython-literal"]
 
 [dependencies]
-rustpython-compiler-core = { path = "../core", version = "0.2.0" }
-rustpython-literal = { path = "../literal", version = "0.2.0", optional = true }
+rustpython-parser-core = { workspace = true }
+rustpython-literal = { workspace = true, optional = true }
 
 num-bigint = { workspace = true }

ast/asdl_rs.py

@@ -8,7 +8,6 @@
 from argparse import ArgumentParser
 from pathlib import Path
 from typing import Optional, Dict
-from attr import dataclass
 
 import asdl
 
@@ -18,7 +17,7 @@
 builtin_type_mapping = {
     "identifier": "Ident",
     "string": "String",
-    "int": "usize",
+    "int": "u32",
     "constant": "Constant",
 }
 assert builtin_type_mapping.keys() == asdl.builtin_types
@@ -391,7 +390,18 @@ def visitModule(self, mod, depth):
             depth + 1,
         )
         self.emit(
-            "fn map_located<T>(&mut self, located: Attributed<T, U>) -> Result<Attributed<T, Self::TargetU>, Self::Error> { let custom = self.map_user(located.custom)?; Ok(Attributed { range: located.range, custom, node: located.node }) }",
+            """
+            fn map_located<T>(&mut self, located: Attributed<T, U>) -> Result<Attributed<T, Self::TargetU>, Self::Error> {
+                let custom = self.map_user(located.custom)?;
+                Ok(Attributed { range: located.range, custom, node: located.node })
+            }""",
+            depth + 1,
+        )
+        self.emit(
+            """
+            fn fold<X: Foldable<U, Self::TargetU>>(&mut self, node: X) -> Result<X::Mapped, Self::Error> {
+                node.fold(self)
+            }""",
             depth + 1,
         )
         for dfn in mod.dfns:
@@ -715,8 +725,8 @@ def make_pattern(self, fields):
         return ",".join(rust_field(f.name) for f in fields)
 
     def gen_sum_fromobj(self, sum, sumname, enumname, rustname, depth):
-        if sum.attributes:
-            self.extract_location(sumname, depth)
+        # if sum.attributes:
+        #     self.extract_location(sumname, depth)
 
         self.emit("let _cls = _object.class();", depth)
         self.emit("Ok(", depth)
@@ -739,8 +749,8 @@ def gen_sum_fromobj(self, sum, sumname, enumname, rustname, depth):
         self.emit("})", depth)
 
     def gen_product_fromobj(self, product, prodname, structname, depth):
-        if product.attributes:
-            self.extract_location(prodname, depth)
+        # if product.attributes:
+        #     self.extract_location(prodname, depth)
 
         self.emit("Ok(", depth)
         self.gen_construction(structname, product, prodname, depth + 1)
@@ -761,11 +771,15 @@ def gen_construction(self, cons_path, cons, name, depth):
     def extract_location(self, typename, depth):
         row = self.decode_field(asdl.Field("int", "lineno"), typename)
         column = self.decode_field(asdl.Field("int", "col_offset"), typename)
-        self.emit(f"""let _location = {{
-        let row = try_location_field({row}, _vm)?;
-        let column = try_location_field({column}, _vm)?;
-        SourceLocation {{ row, column }}
-        }};""", depth)
+        self.emit(
+            f"""
+            let _location = {{
+                let row = {row};
+                let column = {column};
+                try_location(row, column)
+            }};""",
+            depth,
+        )
 
     def decode_field(self, field, typename):
         name = json.dumps(field.name)
@@ -805,7 +819,7 @@ def write_located_def(typeinfo, f):
     f.write(
         textwrap.dedent(
             """
-            use crate::location::SourceRange;
+            use rustpython_parser_core::source_code::SourceRange;
 
             pub type Located<T> = super::generic::Attributed<T, SourceRange>;
             """

ast/src/attributed.rs

@@ -1,5 +1,7 @@
-use crate::location::{SourceLocation, SourceRange};
-use rustpython_compiler_core::text_size::{TextRange, TextSize};
+use rustpython_parser_core::{
+    source_code::{SourceLocation, SourceRange},
+    text_size::{TextRange, TextSize},
+};
 
 #[derive(Clone, Debug, PartialEq)]
 pub struct Attributed<T, U = ()> {

ast/src/constant.rs

@@ -1,5 +1,4 @@
 use num_bigint::BigInt;
-pub use rustpython_compiler_core::ConversionFlag;
 
 #[derive(Clone, Debug, PartialEq)]
 pub enum Constant {
@@ -137,7 +136,7 @@ impl<U> crate::fold::Fold<U> for ConstantOptimizer {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use rustpython_compiler_core::text_size::TextRange;
+    use rustpython_parser_core::text_size::TextRange;
 
     #[cfg(feature = "constant-optimization")]
     #[test]

ast/src/fold_helpers.rs

@@ -62,4 +62,4 @@ macro_rules! simple_fold {
     };
 }
 
-simple_fold!(usize, String, bool, constant::Constant);
+simple_fold!(u32, String, bool, constant::Constant);

ast/src/gen/generic.rs

@@ -158,7 +158,7 @@ pub struct StmtAnnAssign<U = ()> {
     pub target: Box<Expr<U>>,
     pub annotation: Box<Expr<U>>,
     pub value: Option<Box<Expr<U>>>,
-    pub simple: usize,
+    pub simple: u32,
 }
 
 impl<U> From<StmtAnnAssign<U>> for StmtKind<U> {
@@ -328,7 +328,7 @@ impl<U> From<StmtImport<U>> for StmtKind<U> {
 pub struct StmtImportFrom<U = ()> {
     pub module: Option<Ident>,
     pub names: Vec<Alias<U>>,
-    pub level: Option<usize>,
+    pub level: Option<u32>,
 }
 
 impl<U> From<StmtImportFrom<U>> for StmtKind<U> {
@@ -610,7 +610,7 @@ impl<U> From<ExprCall<U>> for ExprKind<U> {
 #[derive(Clone, Debug, PartialEq)]
 pub struct ExprFormattedValue<U = ()> {
     pub value: Box<Expr<U>>,
-    pub conversion: usize,
+    pub conversion: u32,
     pub format_spec: Option<Box<Expr<U>>>,
 }
 
@@ -819,7 +819,7 @@ pub struct Comprehension<U = ()> {
     pub target: Expr<U>,
     pub iter: Expr<U>,
     pub ifs: Vec<Expr<U>>,
-    pub is_async: usize,
+    pub is_async: u32,
 }
 
 #[derive(Clone, Debug, PartialEq)]
@@ -996,7 +996,7 @@ pub type Pattern<U = ()> = Attributed<PatternKind<U>, U>;
 
 #[derive(Clone, Debug, PartialEq)]
 pub struct TypeIgnoreTypeIgnore {
-    pub lineno: usize,
+    pub lineno: u32,
     pub tag: String,
 }
 
@@ -1019,6 +1019,7 @@ pub mod fold {
         type TargetU;
         type Error;
         fn map_user(&mut self, user: U) -> Result<Self::TargetU, Self::Error>;
+
         fn map_located<T>(
             &mut self,
             located: Attributed<T, U>,
@@ -1030,6 +1031,13 @@ pub mod fold {
                 node: located.node,
             })
         }
+
+        fn fold<X: Foldable<U, Self::TargetU>>(
+            &mut self,
+            node: X,
+        ) -> Result<X::Mapped, Self::Error> {
+            node.fold(self)
+        }
         fn fold_mod(&mut self, node: Mod<U>) -> Result<Mod<Self::TargetU>, Self::Error> {
             fold_mod(self, node)
         }

ast/src/gen/located.rs

@@ -1,6 +1,6 @@
 // File automatically generated by ast/asdl_rs.py.
 
-use crate::location::SourceRange;
+use rustpython_parser_core::source_code::SourceRange;
 
 pub type Located<T> = super::generic::Attributed<T, SourceRange>;
 pub type Mod = super::generic::Mod<SourceRange>;

ast/src/lib.rs

@@ -7,53 +7,21 @@ mod generic {
     include!("gen/generic.rs");
 }
 mod impls;
-#[cfg(feature = "location")]
-pub mod located {
-    include!("gen/located.rs");
-}
-#[cfg(feature = "location")]
-mod locator;
-#[cfg(feature = "location")]
-pub use crate::locator::locate;
-#[cfg(feature = "location")]
-pub use rustpython_compiler_core::SourceLocator;
-
+#[cfg(feature = "source-code")]
+mod source_locator;
 #[cfg(feature = "unparse")]
 mod unparse;
 
 pub use attributed::Attributed;
-pub use constant::{Constant, ConversionFlag};
+pub use constant::Constant;
 pub use generic::*;
+pub use rustpython_parser_core::{text_size, ConversionFlag};
 
 pub type Suite<U = ()> = Vec<Stmt<U>>;
 
-pub mod location {
-    pub use rustpython_compiler_core::source_code::{OneIndexed, SourceLocation};
-
-    #[derive(Debug)]
-    pub struct SourceRange {
-        pub start: SourceLocation,
-        pub end: Option<SourceLocation>,
-    }
-
-    impl SourceRange {
-        pub fn new(start: SourceLocation, end: SourceLocation) -> Self {
-            Self {
-                start,
-                end: Some(end),
-            }
-        }
-        pub fn unwrap_end(&self) -> SourceLocation {
-            self.end.unwrap()
-        }
-    }
-
-    impl From<std::ops::Range<SourceLocation>> for SourceRange {
-        fn from(value: std::ops::Range<SourceLocation>) -> Self {
-            Self {
-                start: value.start,
-                end: Some(value.end),
-            }
-        }
-    }
+#[cfg(feature = "source-code")]
+pub mod located {
+    include!("gen/located.rs");
 }
+
+pub use rustpython_parser_core::source_code;

ast/src/locator.rs → ast/src/source_locator.rs

@@ -1,11 +1,5 @@
 use crate::attributed::Attributed;
-use crate::fold_helpers::Foldable;
-use crate::location::SourceRange;
-use rustpython_compiler_core::SourceLocator;
-
-pub fn locate<X: Foldable<(), SourceRange>>(locator: &mut SourceLocator, ast: X) -> X::Mapped {
-    ast.fold(locator).unwrap()
-}
+use rustpython_parser_core::source_code::{SourceLocator, SourceRange};
 
 impl crate::fold::Fold<()> for SourceLocator<'_> {
     type TargetU = SourceRange;

ast/src/unparse.rs

@@ -1,7 +1,5 @@
-use crate::{
-    Arg, Arguments, Boolop, Cmpop, Comprehension, Constant, ConversionFlag, Expr, ExprKind,
-    Operator,
-};
+use crate::ConversionFlag;
+use crate::{Arg, Arguments, Boolop, Cmpop, Comprehension, Constant, Expr, ExprKind, Operator};
 use std::fmt;
 
 mod precedence {
@@ -452,7 +450,7 @@ impl<'a> Unparser<'a> {
     fn unparse_formatted<U>(
         &mut self,
         val: &Expr<U>,
-        conversion: usize,
+        conversion: u32,
         spec: Option<&Expr<U>>,
     ) -> fmt::Result {
         let buffered = to_string_fmt(|f| Unparser::new(f).unparse_expr(val, precedence::TEST + 1));
@@ -466,7 +464,7 @@ impl<'a> Unparser<'a> {
         self.p(&buffered)?;
         drop(buffered);
 
-        if conversion != ConversionFlag::None as usize {
+        if conversion != ConversionFlag::None as u32 {
             self.p("!")?;
             let buf = &[conversion as u8];
             let c = std::str::from_utf8(buf).unwrap();

core/Cargo.toml

@@ -1,14 +1,13 @@
 [package]
-name = "rustpython-compiler-core"
-description = "RustPython specific bytecode."
+name = "rustpython-parser-core"
+description = "RustPython parser data types."
 version = "0.2.0"
 authors = ["RustPython Team"]
 edition = "2021"
 repository = "https://github.com/RustPython/RustPython"
 license = "MIT"
 
 [dependencies]
-bitflags = { workspace = true }
 itertools = { workspace = true }
 num-bigint = { workspace = true }
 num-complex = { workspace = true }
@@ -18,3 +17,6 @@ ruff_python_ast = { workspace = true }
 
 lz4_flex = "0.9.2"
 
+[features]
+default = ["source-code"]
+source-code = []