mirrors/ruff
1
0
Fork 0
mirror of https://github.com/astral-sh/ruff.git synced 2025-11-01 20:30:49 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 4

Use cursor based lexer (#6012)

Browse source
This commit is contained in:
Micha Reiser 2023-07-26 11:32:26 +02:00 • committed by GitHub
parent 025fa4eba8
commit 16e1737d1b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 17 additions and 23 deletions
Download patch file Download diff file Expand all files Collapse all files

18
Cargo.lock generated
View file

@ -2102,7 +2102,6 @@ dependencies = [
"insta", "insta",
"is-macro", "is-macro",
"itertools", "itertools",
"log",
"memchr", "memchr",
"num-bigint", "num-bigint",
"num-traits", "num-traits",
@ -2210,7 +2209,7 @@ dependencies = [
[[package]] [[package]]
name = "ruff_text_size" name = "ruff_text_size"
version = "0.0.0" version = "0.0.0"
source = "git+https://github.com/astral-sh/RustPython-Parser.git?rev=5ef4ccd6322a8b7c5e8ead0ae3c89426d7be7d72#5ef4ccd6322a8b7c5e8ead0ae3c89426d7be7d72" source = "git+https://github.com/astral-sh/RustPython-Parser.git?rev=593b46be5e0336fe01917f8ef400bd12d81df8c1#593b46be5e0336fe01917f8ef400bd12d81df8c1"
dependencies = [ dependencies = [
"schemars", "schemars",
"serde", "serde",
@ -2312,7 +2311,7 @@ dependencies = [
[[package]] [[package]]
name = "rustpython-ast" name = "rustpython-ast"
version = "0.2.0" version = "0.2.0"
source = "git+https://github.com/astral-sh/RustPython-Parser.git?rev=5ef4ccd6322a8b7c5e8ead0ae3c89426d7be7d72#5ef4ccd6322a8b7c5e8ead0ae3c89426d7be7d72" source = "git+https://github.com/astral-sh/RustPython-Parser.git?rev=593b46be5e0336fe01917f8ef400bd12d81df8c1#593b46be5e0336fe01917f8ef400bd12d81df8c1"
dependencies = [ dependencies = [
"is-macro", "is-macro",
"num-bigint", "num-bigint",
@ -2323,7 +2322,7 @@ dependencies = [
[[package]] [[package]]
name = "rustpython-format" name = "rustpython-format"
version = "0.2.0" version = "0.2.0"
source = "git+https://github.com/astral-sh/RustPython-Parser.git?rev=5ef4ccd6322a8b7c5e8ead0ae3c89426d7be7d72#5ef4ccd6322a8b7c5e8ead0ae3c89426d7be7d72" source = "git+https://github.com/astral-sh/RustPython-Parser.git?rev=593b46be5e0336fe01917f8ef400bd12d81df8c1#593b46be5e0336fe01917f8ef400bd12d81df8c1"
dependencies = [ dependencies = [
"bitflags 2.3.3", "bitflags 2.3.3",
"itertools", "itertools",
@ -2335,7 +2334,7 @@ dependencies = [
[[package]] [[package]]
name = "rustpython-literal" name = "rustpython-literal"
version = "0.2.0" version = "0.2.0"
source = "git+https://github.com/astral-sh/RustPython-Parser.git?rev=5ef4ccd6322a8b7c5e8ead0ae3c89426d7be7d72#5ef4ccd6322a8b7c5e8ead0ae3c89426d7be7d72" source = "git+https://github.com/astral-sh/RustPython-Parser.git?rev=593b46be5e0336fe01917f8ef400bd12d81df8c1#593b46be5e0336fe01917f8ef400bd12d81df8c1"
dependencies = [ dependencies = [
"hexf-parse", "hexf-parse",
"is-macro", "is-macro",
@ -2347,20 +2346,18 @@ dependencies = [
[[package]] [[package]]
name = "rustpython-parser" name = "rustpython-parser"
version = "0.2.0" version = "0.2.0"
source = "git+https://github.com/astral-sh/RustPython-Parser.git?rev=5ef4ccd6322a8b7c5e8ead0ae3c89426d7be7d72#5ef4ccd6322a8b7c5e8ead0ae3c89426d7be7d72" source = "git+https://github.com/astral-sh/RustPython-Parser.git?rev=593b46be5e0336fe01917f8ef400bd12d81df8c1#593b46be5e0336fe01917f8ef400bd12d81df8c1"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"is-macro", "is-macro",
"itertools", "itertools",
"lalrpop-util", "lalrpop-util",
"log",
"num-bigint", "num-bigint",
"num-traits", "num-traits",
"phf",
"phf_codegen",
"rustc-hash", "rustc-hash",
"rustpython-ast", "rustpython-ast",
"rustpython-parser-core", "rustpython-parser-core",
"static_assertions",
"tiny-keccak", "tiny-keccak",
"unic-emoji-char", "unic-emoji-char",
"unic-ucd-ident", "unic-ucd-ident",
@ -2370,10 +2367,9 @@ dependencies = [
[[package]] [[package]]
name = "rustpython-parser-core" name = "rustpython-parser-core"
version = "0.2.0" version = "0.2.0"
source = "git+https://github.com/astral-sh/RustPython-Parser.git?rev=5ef4ccd6322a8b7c5e8ead0ae3c89426d7be7d72#5ef4ccd6322a8b7c5e8ead0ae3c89426d7be7d72" source = "git+https://github.com/astral-sh/RustPython-Parser.git?rev=593b46be5e0336fe01917f8ef400bd12d81df8c1#593b46be5e0336fe01917f8ef400bd12d81df8c1"
dependencies = [ dependencies = [
"is-macro", "is-macro",
"memchr",
"ruff_text_size", "ruff_text_size",
] ]

10
Cargo.toml
View file

@ -51,11 +51,11 @@ wsl = { version = "0.1.0" }
# v1.0.1 # v1.0.1
libcst = { git = "https://github.com/Instagram/LibCST.git", rev = "3cacca1a1029f05707e50703b49fe3dd860aa839", default-features = false } libcst = { git = "https://github.com/Instagram/LibCST.git", rev = "3cacca1a1029f05707e50703b49fe3dd860aa839", default-features = false }
ruff_text_size = { git = "https://github.com/astral-sh/RustPython-Parser.git", rev = "5ef4ccd6322a8b7c5e8ead0ae3c89426d7be7d72" } ruff_text_size = { git = "https://github.com/astral-sh/RustPython-Parser.git", rev = "593b46be5e0336fe01917f8ef400bd12d81df8c1" }
rustpython-ast = { git = "https://github.com/astral-sh/RustPython-Parser.git", rev = "5ef4ccd6322a8b7c5e8ead0ae3c89426d7be7d72" , default-features = false, features = ["num-bigint"]} rustpython-ast = { git = "https://github.com/astral-sh/RustPython-Parser.git", rev = "593b46be5e0336fe01917f8ef400bd12d81df8c1" }
rustpython-format = { git = "https://github.com/astral-sh/RustPython-Parser.git", rev = "5ef4ccd6322a8b7c5e8ead0ae3c89426d7be7d72", default-features = false, features = ["num-bigint"] } rustpython-format = { git = "https://github.com/astral-sh/RustPython-Parser.git", rev = "593b46be5e0336fe01917f8ef400bd12d81df8c1" }
rustpython-literal = { git = "https://github.com/astral-sh/RustPython-Parser.git", rev = "5ef4ccd6322a8b7c5e8ead0ae3c89426d7be7d72", default-features = false } rustpython-literal = { git = "https://github.com/astral-sh/RustPython-Parser.git", rev = "593b46be5e0336fe01917f8ef400bd12d81df8c1" }
rustpython-parser = { git = "https://github.com/astral-sh/RustPython-Parser.git", rev = "5ef4ccd6322a8b7c5e8ead0ae3c89426d7be7d72" , default-features = false, features = ["full-lexer", "num-bigint"] } rustpython-parser = { git = "https://github.com/astral-sh/RustPython-Parser.git", rev = "593b46be5e0336fe01917f8ef400bd12d81df8c1" }
[profile.release] [profile.release]
lto = "fat" lto = "fat"

2
crates/ruff/resources/test/fixtures/jupyter/not_json.ipynb vendored
View file

@ -1 +1 @@
broken "§=($/=(") broken "§=($/=()

7
crates/ruff/src/jupyter/notebook.rs
View file

@ -12,6 +12,8 @@ use serde_json::error::Category;
use ruff_diagnostics::Diagnostic; use ruff_diagnostics::Diagnostic;
use ruff_python_trivia::{NewlineWithTrailingNewline, UniversalNewlineIterator}; use ruff_python_trivia::{NewlineWithTrailingNewline, UniversalNewlineIterator};
use ruff_text_size::{TextRange, TextSize}; use ruff_text_size::{TextRange, TextSize};
use rustpython_parser::lexer::lex;
use rustpython_parser::Mode;
use crate::autofix::source_map::{SourceMap, SourceMarker}; use crate::autofix::source_map::{SourceMap, SourceMarker};
use crate::jupyter::index::JupyterIndex; use crate::jupyter::index::JupyterIndex;
@ -158,10 +160,7 @@ impl Notebook {
) )
})?; })?;
// Check if tokenizing was successful and the file is non-empty // Check if tokenizing was successful and the file is non-empty
if (ruff_rustpython::tokenize(&contents)) if lex(&contents, Mode::Module).any(|result| result.is_err()) {
.last()
.map_or(true, Result::is_err)
{
Diagnostic::new( Diagnostic::new(
SyntaxError { SyntaxError {
message: format!( message: format!(

1
crates/ruff_python_ast/Cargo.toml
View file

@ -20,7 +20,6 @@ anyhow = { workspace = true }
bitflags = { workspace = true } bitflags = { workspace = true }
is-macro = { workspace = true } is-macro = { workspace = true }
itertools = { workspace = true } itertools = { workspace = true }
log = { workspace = true }
memchr = { workspace = true } memchr = { workspace = true }
num-bigint = { workspace = true } num-bigint = { workspace = true }
num-traits = { workspace = true } num-traits = { workspace = true }

2
fuzz/Cargo.toml
View file

@ -24,7 +24,7 @@ ruff_python_ast = { path = "../crates/ruff_python_ast" }
ruff_python_formatter = { path = "../crates/ruff_python_formatter" } ruff_python_formatter = { path = "../crates/ruff_python_formatter" }
similar = { version = "2.2.1" } similar = { version = "2.2.1" }
rustpython-parser = { git = "https://github.com/astral-sh/RustPython-Parser.git", rev = "5ef4ccd6322a8b7c5e8ead0ae3c89426d7be7d72" , default-features = false, features = ["full-lexer", "num-bigint"] } rustpython-parser = { git = "https://github.com/astral-sh/RustPython-Parser.git", rev = "593b46be5e0336fe01917f8ef400bd12d81df8c1" }
# Prevent this from interfering with workspaces # Prevent this from interfering with workspaces
[workspace] [workspace]

0
fuzz/init-fuzzer.sh Normal file → Executable file
View file