mirror of
https://github.com/astral-sh/ruff.git
synced 2025-09-28 12:55:05 +00:00
81160320de
## Summary I look at the token stream a lot, not specifically in the playground but in the terminal output and it's annoying to scroll a lot to find specific location. Most of the information is also redundant. The final format we end up with is: `<kind> <range> (flags = ...)` e.g., `String 0..4 (flags = BYTE_STRING)` where the flags part is only populated if there are any flags set.
32 lines
918 B
Rust
32 lines
918 B
Rust
//! Print the token stream for a given Python file.
|
|
#![allow(clippy::print_stdout, clippy::print_stderr)]
|
|
|
|
use std::path::PathBuf;
|
|
|
|
use anyhow::Result;
|
|
|
|
use ruff_linter::source_kind::SourceKind;
|
|
use ruff_python_ast::PySourceType;
|
|
use ruff_python_parser::parse_unchecked_source;
|
|
|
|
#[derive(clap::Args)]
|
|
pub(crate) struct Args {
|
|
/// Python file for which to generate the AST.
|
|
#[arg(required = true)]
|
|
file: PathBuf,
|
|
}
|
|
|
|
pub(crate) fn main(args: &Args) -> Result<()> {
|
|
let source_type = PySourceType::from(&args.file);
|
|
let source_kind = SourceKind::from_path(&args.file, source_type)?.ok_or_else(|| {
|
|
anyhow::anyhow!(
|
|
"Could not determine source kind for file: {}",
|
|
args.file.display()
|
|
)
|
|
})?;
|
|
let parsed = parse_unchecked_source(source_kind.source_code(), source_type);
|
|
for token in parsed.tokens() {
|
|
println!("{token:#?}");
|
|
}
|
|
Ok(())
|
|
}
|