mirrors/ruff
1
0
Fork 0
mirror of https://github.com/astral-sh/ruff.git synced 2025-07-24 05:25:17 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Move RustPython vendored and helper code into its own crate (#3171)

Browse source
This commit is contained in:
Charlie Marsh 2023-02-23 09:14:16 -05:00 • committed by GitHub
parent 0f04aa2a5f
commit 095f005bf4
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
19 changed files with 58 additions and 57 deletions
Download patch file Download diff file Expand all files Collapse all files

1
crates/ruff_python_formatter/src/core/mod.rs
View file

@ -1,4 +1,3 @@
pub mod locator;
pub mod rustpython_helpers;
pub mod types;
pub mod visitor;

28
crates/ruff_python_formatter/src/core/rustpython_helpers.rs
View file

@ -1,28 +0,0 @@
use rustpython_parser as parser;
use rustpython_parser::ast::{Mod, Suite};
use rustpython_parser::lexer::LexResult;
use rustpython_parser::{lexer, Mode, ParseError};
/// Collect tokens up to and including the first error.
pub fn tokenize(contents: &str) -> Vec<LexResult> {
let mut tokens: Vec<LexResult> = vec![];
for tok in lexer::lex(contents, Mode::Module) {
let is_err = tok.is_err();
tokens.push(tok);
if is_err {
break;
}
}
tokens
}
/// Parse a full Python program from its tokens.
pub(crate) fn parse_program_tokens(
lxr: Vec<LexResult>,
source_path: &str,
) -> anyhow::Result<Suite, ParseError> {
parser::parse_tokens(lxr, Mode::Module, source_path).map(|top| match top {
Mod::Module { body, .. } => body,
_ => unreachable!(),
})
}

5
crates/ruff_python_formatter/src/lib.rs
View file

@ -5,7 +5,6 @@ use rustpython_parser::lexer::LexResult;
use crate::attachment::attach;
use crate::context::ASTFormatContext;
use crate::core::locator::Locator;
use crate::core::rustpython_helpers;
use crate::cst::Stmt;
use crate::newlines::normalize_newlines;
use crate::parentheses::normalize_parentheses;
@ -24,13 +23,13 @@ pub mod trivia;
pub fn fmt(contents: &str) -> Result<Formatted<ASTFormatContext>> {
// Tokenize once.
let tokens: Vec<LexResult> = rustpython_helpers::tokenize(contents);
let tokens: Vec<LexResult> = ruff_rustpython::tokenize(contents);
// Extract trivia.
let trivia = trivia::extract_trivia_tokens(&tokens);
// Parse the AST.
let python_ast = rustpython_helpers::parse_program_tokens(tokens, "<filename>")?;
let python_ast = ruff_rustpython::parse_program_tokens(tokens, "<filename>")?;
// Convert to a CST.
let mut python_cst: Vec<Stmt> = python_ast.into_iter().map(Into::into).collect();