Merge pull request #7 from youknowone/lalrpop

Embed generated parser + update lalrpop

.github/workflows/ci.yaml

@@ -37,7 +37,9 @@ jobs:
 
       - uses: Swatinem/rust-cache@v2
 
-      - name: run tests
+      - name: run tests with embedded parser
+        run: cargo test --all --no-default-features
+      - name: run tests with generated parser
         run: cargo test --all --all-features
 
   lint:

parser/Cargo.toml

@@ -9,12 +9,12 @@ license = "MIT"
 edition = "2021"
 
 [features]
-default = ["lalrpop"]  # removing this causes potential build failure
+default = []
 serde = ["dep:serde", "rustpython-compiler-core/serde"]
 
 [build-dependencies]
 anyhow = { workspace = true }
-lalrpop = { version = "0.19.9", optional = true }
+lalrpop = { version = "0.20.0", default-features = false, optional = true }
 phf_codegen = "0.11.1"
 tiny-keccak = { version = "2", features = ["sha3"] }
 
@@ -31,7 +31,7 @@ unicode_names2 = { workspace = true }
 
 unic-emoji-char = "0.9.0"
 unic-ucd-ident  = "0.9.0"
-lalrpop-util = "0.19.8"
+lalrpop-util = { version = "0.20.0", default-features = false }
 phf = "0.11.1"
 rustc-hash = "1.1.0"
 serde = { version = "1.0.133", optional = true, default-features = false, features = ["derive"] }

parser/build.rs

@@ -5,15 +5,39 @@ use std::path::{Path, PathBuf};
 use tiny_keccak::{Hasher, Sha3};
 
 fn main() -> anyhow::Result<()> {
-    const SOURCE: &str = "python.lalrpop";
     let out_dir = PathBuf::from(std::env::var_os("OUT_DIR").unwrap());
-
-    println!("cargo:rerun-if-changed={SOURCE}");
-
-    try_lalrpop(SOURCE, &out_dir.join("python.rs"))?;
     gen_phf(&out_dir);
 
-    Ok(())
+    const SOURCE: &str = "src/python.lalrpop";
+    println!("cargo:rerun-if-changed={SOURCE}");
+
+    let target;
+    let error;
+
+    #[cfg(feature = "lalrpop")]
+    {
+        target = out_dir.join("src/python.rs");
+    }
+    #[cfg(not(feature = "lalrpop"))]
+    {
+        target = PathBuf::from("src/python.rs");
+        error = "python.lalrpop and src/python.rs doesn't match. This is a rustpython-parser bug. Please report it unless you are editing rustpython-parser. Run `lalrpop src/python.lalrpop` to build parser again.";
+    }
+
+    let Some(message) = requires_lalrpop(SOURCE, &target) else {
+        return Ok(());
+    };
+
+    #[cfg(feature = "lalrpop")]
+    {
+        let Err(e) = try_lalrpop() else {
+            return Ok(());
+        };
+        error = e;
+    }
+
+    println!("cargo:warning={message}");
+    panic!("running lalrpop failed. {error:?}");
 }
 
 fn requires_lalrpop(source: &str, target: &Path) -> Option<String> {
@@ -68,28 +92,14 @@ fn requires_lalrpop(source: &str, target: &Path) -> Option<String> {
     None
 }
 
-fn try_lalrpop(source: &str, target: &Path) -> anyhow::Result<()> {
+#[cfg(feature = "lalrpop")]
-    let Some(_message) = requires_lalrpop(source, target) else {
+fn try_lalrpop() -> Result<(), Box<dyn std::error::Error>> {
-        return Ok(());
-    };
-
-    #[cfg(feature = "lalrpop")]
     // We are not using lalrpop::process_root() or Configuration::process_current_dir()
     // because of https://github.com/lalrpop/lalrpop/issues/699.
     lalrpop::Configuration::new()
         .use_cargo_dir_conventions()
         .set_in_dir(Path::new("."))
         .process()
-        .unwrap_or_else(|e| {
-            println!("cargo:warning={_message}");
-            panic!("running lalrpop failed. {e:?}");
-        });
-
-    #[cfg(not(feature = "lalrpop"))]
-    {
-        println!("cargo:warning=try: cargo build --manifest-path=compiler/parser/Cargo.toml --features=lalrpop");
-    }
-    Ok(())
 }
 
 fn sha_equal(expected_sha3_str: &str, actual_sha3: &[u8; 32]) -> bool {

parser/src/lib.rs

@@ -116,14 +116,12 @@ pub use rustpython_ast as ast;
 
 mod function;
 // Skip flattening lexer to distinguish from full parser
+mod context;
 pub mod lexer;
 mod mode;
 mod parser;
-mod string;
-#[rustfmt::skip]
-mod python;
-mod context;
 mod soft_keywords;
+mod string;
 mod token;
 
 pub use mode::Mode;
@@ -133,3 +131,15 @@ pub use parser::{
 };
 pub use string::FStringErrorType;
 pub use token::{StringKind, Tok};
+
+#[rustfmt::skip]
+mod python {
+    #![allow(clippy::all)]
+    #![allow(unused)]
+
+    #[cfg(feature = "lalrpop")]
+    include!(concat!(env!("OUT_DIR"), "/src/python.rs"));
+
+    #[cfg(not(feature = "lalrpop"))]
+    include!("python.rs");
+}

parser/src/parser.rs

@@ -250,7 +250,7 @@ fn parse_error_from_lalrpop(
                 source_path,
             }
         }
-        LalrpopError::UnrecognizedEOF { location, expected } => {
+        LalrpopError::UnrecognizedEof { location, expected } => {
             // This could be an initial indentation error that we should ignore
             let indent_error = expected == ["Indent"];
             if indent_error {