Replace row/column based Location with byte-offsets.

parser/src/parser.rs

@@ -13,11 +13,12 @@
 //! [`Mode`]: crate::mode
 
 use crate::{
-    ast::{self, Location},
+    ast::{self},
     lexer::{self, LexResult, LexicalError, LexicalErrorType},
     mode::Mode,
     python,
     token::Tok,
+    Location,
 };
 use itertools::Itertools;
 use std::iter;
@@ -69,7 +70,7 @@ pub fn parse_program(source: &str, source_path: &str) -> Result<ast::Suite, Pars
 ///
 /// ```
 pub fn parse_expression(source: &str, path: &str) -> Result<ast::Expr, ParseError> {
-    parse_expression_located(source, path, Location::new(1, 0))
+    parse_expression_located(source, path, Location::default())
 }
 
 /// Parses a Python expression from a given location.
@@ -83,9 +84,10 @@ pub fn parse_expression(source: &str, path: &str) -> Result<ast::Expr, ParseErro
 /// somewhat silly, location:
 ///
 /// ```
-/// use rustpython_parser::{ast::Location, parse_expression_located};
+/// use ruff_text_size::TextSize;
+/// use rustpython_parser::{parse_expression_located};
 ///
-/// let expr = parse_expression_located("1 + 2", "<embedded>", Location::new(5, 20));
+/// let expr = parse_expression_located("1 + 2", "<embedded>", TextSize::from(400));
 /// assert!(expr.is_ok());
 /// ```
 pub fn parse_expression_located(
@@ -131,7 +133,7 @@ pub fn parse_expression_located(
 /// assert!(program.is_ok());
 /// ```
 pub fn parse(source: &str, mode: Mode, source_path: &str) -> Result<ast::Mod, ParseError> {
-    parse_located(source, mode, source_path, Location::new(1, 0))
+    parse_located(source, mode, source_path, Location::default())
 }
 
 /// Parse the given Python source code using the specified [`Mode`] and [`Location`].
@@ -142,7 +144,8 @@ pub fn parse(source: &str, mode: Mode, source_path: &str) -> Result<ast::Mod, Pa
 /// # Example
 ///
 /// ```
-/// use rustpython_parser::{ast::Location, Mode, parse_located};
+/// use ruff_text_size::TextSize;
+/// use rustpython_parser::{Mode, parse_located};
 ///
 /// let source = r#"
 /// def fib(i):
@@ -153,7 +156,7 @@ pub fn parse(source: &str, mode: Mode, source_path: &str) -> Result<ast::Mod, Pa
 ///
 /// print(fib(42))
 /// "#;
-/// let program = parse_located(source, Mode::Module, "<embedded>", Location::new(1, 0));
+/// let program = parse_located(source, Mode::Module, "<embedded>", TextSize::from(0));
 /// assert!(program.is_ok());
 /// ```
 pub fn parse_located(
@@ -186,12 +189,16 @@ pub fn parse_tokens(
     mode: Mode,
     source_path: &str,
 ) -> Result<ast::Mod, ParseError> {
-    let marker_token = (Default::default(), mode.to_marker(), Default::default());
+    let marker_token = (mode.to_marker(), Default::default());
     let lexer = iter::once(Ok(marker_token))
         .chain(lxr)
-        .filter_ok(|(_, tok, _)| !matches!(tok, Tok::Comment { .. } | Tok::NonLogicalNewline));
+        .filter_ok(|(tok, _)| !matches!(tok, Tok::Comment { .. } | Tok::NonLogicalNewline));
     python::TopParser::new()
-        .parse(lexer.into_iter())
+        .parse(
+            lexer
+                .into_iter()
+                .map_ok(|(t, range)| (range.start(), t, range.end())),
+        )
         .map_err(|e| parse_error_from_lalrpop(e, source_path))
 }
 
@@ -223,6 +230,7 @@ fn parse_error_from_lalrpop(
     source_path: &str,
 ) -> ParseError {
     let source_path = source_path.to_owned();
+
     match err {
         // TODO: Are there cases where this isn't an EOF?
         LalrpopError::InvalidToken { location } => ParseError {
@@ -246,7 +254,7 @@ fn parse_error_from_lalrpop(
             let expected = (expected.len() == 1).then(|| expected[0].clone());
             ParseError {
                 error: ParseErrorType::UnrecognizedToken(token.1, expected),
-                location: token.0.with_col_offset(1),
+                location: token.0,
                 source_path,
             }
         }
@@ -576,9 +584,9 @@ except* OSError as e:
     fn test_modes() {
         let source = "a[0][1][2][3][4]";
 
-        assert!(parse(&source, Mode::Expression, "<embedded>").is_ok());
-        assert!(parse(&source, Mode::Module, "<embedded>").is_ok());
-        assert!(parse(&source, Mode::Interactive, "<embedded>").is_ok());
+        assert!(parse(source, Mode::Expression, "<embedded>").is_ok());
+        assert!(parse(source, Mode::Module, "<embedded>").is_ok());
+        assert!(parse(source, Mode::Interactive, "<embedded>").is_ok());
     }
 
     #[test]