From d99c96d6b6b7a7f96f01a46390d61f3b4f00504f Mon Sep 17 00:00:00 2001 From: Josh Thomas Date: Tue, 9 Sep 2025 19:08:42 -0500 Subject: [PATCH] Replace PyO3 with IPC approach for Python/project information (#214) --- .github/workflows/build.yml | 8 +- .lazy.lua | 2 +- Cargo.lock | 97 +------ Cargo.toml | 3 - README.md | 3 +- crates/djls-dev/Cargo.toml | 16 -- crates/djls-dev/src/bin/djls-tmux.rs | 101 -------- crates/djls-dev/src/build.rs | 55 ---- crates/djls-dev/src/lib.rs | 3 - crates/djls-project/Cargo.toml | 12 +- crates/djls-project/build.rs | 57 ++++- crates/djls-project/src/inspector.rs | 20 ++ crates/djls-project/src/inspector/ipc.rs | 128 +++++++++ crates/djls-project/src/inspector/pool.rs | 206 +++++++++++++++ crates/djls-project/src/inspector/queries.rs | 50 ++++ crates/djls-project/src/lib.rs | 65 ++--- crates/djls-project/src/python.rs | 242 +----------------- crates/djls-project/src/templatetags.rs | 81 +++--- crates/djls-server/Cargo.toml | 8 - crates/djls-server/build.rs | 3 - crates/djls-server/src/session.rs | 4 +- crates/djls/Cargo.toml | 16 -- crates/djls/build.rs | 3 - crates/djls/src/commands/serve.rs | 4 - crates/djls/src/lib.rs | 34 --- crates/djls/src/main.rs | 5 +- docs/index.md | 4 +- noxfile.py | 9 - pyproject.toml | 3 - python/Justfile | 12 + python/build.py | 24 ++ python/pyproject.toml | 12 + python/src/djls_inspector/__init__.py | 0 python/src/djls_inspector/__main__.py | 44 ++++ python/src/djls_inspector/inspector.py | 77 ++++++ python/src/djls_inspector/queries.py | 174 +++++++++++++ python/uv.lock | 8 + .../djls_app/templates/djls_app/base.html | 2 +- uv.lock | 4 +- 39 files changed, 903 insertions(+), 696 deletions(-) delete mode 100644 crates/djls-dev/Cargo.toml delete mode 100644 crates/djls-dev/src/bin/djls-tmux.rs delete mode 100644 crates/djls-dev/src/build.rs delete mode 100644 crates/djls-dev/src/lib.rs create mode 100644 crates/djls-project/src/inspector.rs create mode 100644 crates/djls-project/src/inspector/ipc.rs create mode 100644 crates/djls-project/src/inspector/pool.rs create mode 100644 crates/djls-project/src/inspector/queries.rs delete mode 100644 crates/djls-server/build.rs delete mode 100644 crates/djls/build.rs delete mode 100644 crates/djls/src/lib.rs create mode 100644 python/Justfile create mode 100644 python/build.py create mode 100644 python/pyproject.toml create mode 100644 python/src/djls_inspector/__init__.py create mode 100644 python/src/djls_inspector/__main__.py create mode 100644 python/src/djls_inspector/inspector.py create mode 100644 python/src/djls_inspector/queries.py create mode 100644 python/uv.lock diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 633f2cd..2ad986d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -42,7 +42,7 @@ jobs: uses: PyO3/maturin-action@86b9d133d34bc1b40018696f782949dac11bd380 with: target: ${{ matrix.platform.target }} - args: --release --out dist --find-interpreter --features extension-module + args: --release --out dist --find-interpreter sccache: ${{ !startsWith(github.ref, 'refs/tags/') }} manylinux: auto @@ -78,7 +78,7 @@ jobs: uses: PyO3/maturin-action@86b9d133d34bc1b40018696f782949dac11bd380 with: target: ${{ matrix.platform.target }} - args: --release --out dist --find-interpreter --features extension-module + args: --release --out dist --find-interpreter sccache: ${{ !startsWith(github.ref, 'refs/tags/') }} manylinux: musllinux_1_2 @@ -111,7 +111,7 @@ jobs: uses: PyO3/maturin-action@86b9d133d34bc1b40018696f782949dac11bd380 with: target: ${{ matrix.platform.target }} - args: --release --out dist --find-interpreter --features extension-module + args: --release --out dist --find-interpreter sccache: ${{ !startsWith(github.ref, 'refs/tags/') }} - name: Upload wheels @@ -142,7 +142,7 @@ jobs: uses: PyO3/maturin-action@86b9d133d34bc1b40018696f782949dac11bd380 with: target: ${{ matrix.platform.target }} - args: --release --out dist --find-interpreter --features extension-module + args: --release --out dist --find-interpreter sccache: ${{ !startsWith(github.ref, 'refs/tags/') }} - name: Upload wheels diff --git a/.lazy.lua b/.lazy.lua index 8a68aed..a48b4e7 100644 --- a/.lazy.lua +++ b/.lazy.lua @@ -1,5 +1,5 @@ vim.lsp.config["djls"] = { - cmd = { "uvx", "lsp-devtools", "agent", "--", "djls", "serve" }, + cmd = { "uvx", "lsp-devtools", "agent", "--", "./target/debug/djls", "serve" }, filetypes = { "htmldjango" }, root_markers = { "manage.py", "pyproject.toml" }, } diff --git a/Cargo.lock b/Cargo.lock index f1d8b44..19af69e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -426,10 +426,8 @@ version = "5.2.0" dependencies = [ "anyhow", "clap", - "djls-dev", "djls-project", "djls-server", - "pyo3", "serde_json", ] @@ -446,22 +444,15 @@ dependencies = [ "toml", ] -[[package]] -name = "djls-dev" -version = "0.0.0" -dependencies = [ - "anyhow", - "pyo3-build-config", -] - [[package]] name = "djls-project" version = "0.0.0" dependencies = [ - "djls-dev", + "anyhow", "djls-workspace", - "pyo3", "salsa", + "serde", + "serde_json", "tempfile", "which", ] @@ -474,12 +465,10 @@ dependencies = [ "camino", "dashmap", "djls-conf", - "djls-dev", "djls-project", "djls-templates", "djls-workspace", "percent-encoding", - "pyo3", "salsa", "serde", "serde_json", @@ -896,12 +885,6 @@ dependencies = [ "hashbrown 0.15.5", ] -[[package]] -name = "indoc" -version = "2.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4c7245a08504955605670dbf141fceab975f15ca21570696aebe9d2e71576bd" - [[package]] name = "inotify" version = "0.11.0" @@ -1312,68 +1295,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "pyo3" -version = "0.25.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8970a78afe0628a3e3430376fc5fd76b6b45c4d43360ffd6cdd40bdde72b682a" -dependencies = [ - "indoc", - "libc", - "memoffset", - "once_cell", - "portable-atomic", - "pyo3-build-config", - "pyo3-ffi", - "pyo3-macros", - "unindent", -] - -[[package]] -name = "pyo3-build-config" -version = "0.25.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "458eb0c55e7ece017adeba38f2248ff3ac615e53660d7c71a238d7d2a01c7598" -dependencies = [ - "once_cell", - "target-lexicon", -] - -[[package]] -name = "pyo3-ffi" -version = "0.25.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7114fe5457c61b276ab77c5055f206295b812608083644a5c5b2640c3102565c" -dependencies = [ - "libc", - "pyo3-build-config", -] - -[[package]] -name = "pyo3-macros" -version = "0.25.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8725c0a622b374d6cb051d11a0983786448f7785336139c3c94f5aa6bef7e50" -dependencies = [ - "proc-macro2", - "pyo3-macros-backend", - "quote", - "syn", -] - -[[package]] -name = "pyo3-macros-backend" -version = "0.25.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4109984c22491085343c05b0dbc54ddc405c3cf7b4374fc533f5c3313a572ccc" -dependencies = [ - "heck", - "proc-macro2", - "pyo3-build-config", - "quote", - "syn", -] - [[package]] name = "quote" version = "1.0.40" @@ -1728,12 +1649,6 @@ dependencies = [ "syn", ] -[[package]] -name = "target-lexicon" -version = "0.13.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e502f78cdbb8ba4718f566c418c52bc729126ffd16baee5baa718cf25dd5a69a" - [[package]] name = "tempfile" version = "3.21.0" @@ -2092,12 +2007,6 @@ version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" -[[package]] -name = "unindent" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3" - [[package]] name = "url" version = "2.5.7" diff --git a/Cargo.toml b/Cargo.toml index 6d18e3b..c3c5e38 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,15 +5,12 @@ resolver = "2" [workspace.dependencies] djls = { path = "crates/djls" } djls-conf = { path = "crates/djls-conf" } -djls-dev = { path = "crates/djls-dev" } djls-project = { path = "crates/djls-project" } djls-server = { path = "crates/djls-server" } djls-templates = { path = "crates/djls-templates" } djls-workspace = { path = "crates/djls-workspace" } # core deps, pin exact versions -pyo3 = "0.25.0" -pyo3-build-config = { version = "0.25.0", features = ["resolve-config"] } salsa = "0.23.0" tower-lsp-server = { version = "0.22.0", features = ["proposed"] } diff --git a/README.md b/README.md index 51db31b..39fa68a 100644 --- a/README.md +++ b/README.md @@ -156,11 +156,10 @@ All feature requests should ideally start out as a discussion topic, to gather f ### Development -The project is written in Rust using PyO3 for Python integration. Here is a high-level overview of the project and the various crates: +The project is written in Rust with IPC for Python communication. Here is a high-level overview of the project and the various crates: - Main CLI interface ([`crates/djls/`](./crates/djls/)) - Configuration management ([`crates/djls-conf/`](./crates/djls-conf/)) -- Development tools ([`crates/djls-dev/`](./crates/djls-dev/)) - Django and Python project introspection ([`crates/djls-project/`](./crates/djls-project/)) - LSP server implementation ([`crates/djls-server/`](./crates/djls-server/)) - Template parsing ([`crates/djls-templates/`](./crates/djls-templates/)) diff --git a/crates/djls-dev/Cargo.toml b/crates/djls-dev/Cargo.toml deleted file mode 100644 index ab8e190..0000000 --- a/crates/djls-dev/Cargo.toml +++ /dev/null @@ -1,16 +0,0 @@ -[package] -name = "djls-dev" -version = "0.0.0" -edition = "2021" -publish = false - -[[bin]] -name = "djls-tmux" -path = "src/bin/djls-tmux.rs" - -[dependencies] -anyhow = { workspace = true } -pyo3-build-config = { workspace = true } - -[lints] -workspace = true diff --git a/crates/djls-dev/src/bin/djls-tmux.rs b/crates/djls-dev/src/bin/djls-tmux.rs deleted file mode 100644 index 0c8d5f4..0000000 --- a/crates/djls-dev/src/bin/djls-tmux.rs +++ /dev/null @@ -1,101 +0,0 @@ -use std::io::Write; -use std::process::Command; -use std::process::Stdio; - -use anyhow::Context; -use anyhow::Result; - -fn main() -> Result<()> { - // Kill any existing session - let _ = Command::new("tmux") - .args(["kill-session", "-t", "djls-debug"]) - .output(); - - let _ = Command::new("pkill").args(["-f", "lsp-devtools"]).output(); - - // Start tmux in control mode - let mut tmux = Command::new("tmux") - .args(["-C", "-f", "/dev/null"]) - .stdin(Stdio::piped()) - .stdout(Stdio::piped()) - .spawn() - .context("Failed to start tmux control mode")?; - - let stdin = tmux.stdin.as_mut().context("Failed to get tmux stdin")?; - - // Create session with editor, setting DJANGO_SETTINGS_MODULE and PYTHONPATH - writeln!( - stdin, - "new-session -d -s djls-debug 'DJANGO_SETTINGS_MODULE=djls_test.settings PYTHONPATH=tests/project:$PYTHONPATH nvim tests/project/djls_app/templates/djls_app/base.html'" - )?; - - // Add devtools pane (20% width on the right) - writeln!( - stdin, - "split-window -t djls-debug -h -p 20 'just dev devtools record'" - )?; - - // Split the right pane horizontally for server logs (50/50 split) - // Updated to handle dated log files properly - writeln!( - stdin, - r#"split-window -t djls-debug:0.1 -v -p 50 'bash -c "log=\$(ls -t /tmp/djls.log.* 2>/dev/null | head -1); if [ -z \"\$log\" ]; then echo \"Waiting for server logs...\"; while [ -z \"\$log\" ]; do sleep 1; log=\$(ls -t /tmp/djls.log.* 2>/dev/null | head -1); done; fi; echo \"Tailing \$log\"; tail -F \"\$log\""'"# - )?; - - // Set pane titles - writeln!(stdin, "select-pane -t djls-debug:0.0 -T 'Editor'")?; - writeln!(stdin, "select-pane -t djls-debug:0.1 -T 'LSP Messages'")?; - writeln!(stdin, "select-pane -t djls-debug:0.2 -T 'Server Logs'")?; - - // Enable pane borders with titles at the top - writeln!(stdin, "set -t djls-debug pane-border-status top")?; - - // Enable mouse support for scrolling and pane interaction - writeln!(stdin, "set -t djls-debug mouse on")?; - - // Add custom keybind to kill session (capital K) - writeln!(stdin, "bind-key K kill-session")?; - - // Configure status bar with keybind hints - writeln!(stdin, "set -t djls-debug status on")?; - writeln!(stdin, "set -t djls-debug status-position bottom")?; - writeln!( - stdin, - "set -t djls-debug status-style 'bg=colour235,fg=colour250'" - )?; - - // Left side: session name - writeln!(stdin, "set -t djls-debug status-left '[#S] '")?; - writeln!(stdin, "set -t djls-debug status-left-length 20")?; - - // Right side: keybind hints - updated to include mouse info - writeln!(stdin, "set -t djls-debug status-right ' Mouse: scroll/click | C-b d: detach | C-b K: kill | C-b x: kill pane | C-b z: zoom | C-b ?: help '")?; - writeln!(stdin, "set -t djls-debug status-right-length 120")?; - - // Center: window name - writeln!(stdin, "set -t djls-debug status-justify centre")?; - - // Focus editor pane - writeln!(stdin, "select-pane -t djls-debug:0.0")?; - - // Exit control mode - writeln!(stdin, "detach")?; - stdin.flush()?; - - // Close stdin to signal we're done sending commands - drop(tmux.stdin.take()); - - // Wait for control mode to finish - tmux.wait()?; - - // Attach to session - Command::new("tmux") - .args(["attach-session", "-t", "djls-debug"]) - .status() - .context("Failed to attach to session")?; - - // Cleanup on exit - let _ = Command::new("pkill").args(["-f", "lsp-devtools"]).output(); - - Ok(()) -} diff --git a/crates/djls-dev/src/build.rs b/crates/djls-dev/src/build.rs deleted file mode 100644 index 3565b97..0000000 --- a/crates/djls-dev/src/build.rs +++ /dev/null @@ -1,55 +0,0 @@ -/// Sets up the necessary Cargo directives for linking against the Python library. -/// -/// This function should be called from the `build.rs` script of any crate -/// within the workspace whose compiled artifact (e.g., test executable, binary) -/// needs to link against the Python C API at compile time and find the -/// corresponding shared library at runtime. This is typically required for -/// crates using `pyo3` features like `Python::with_gil` or defining `#[pyfunction]`s -/// directly in their test or binary code. -/// -/// It uses `pyo3-build-config` to detect the active Python installation and -/// prints the required `cargo:rustc-link-search` and `cargo:rustc-link-lib` -/// directives to Cargo, enabling the linker to find and link against the -/// appropriate Python library (e.g., libpythonX.Y.so). -/// -/// It also adds an RPATH linker argument on Unix-like systems (`-Wl,-rpath,...`) -/// to help the resulting executable find the Python shared library at runtime -/// without needing manual `LD_LIBRARY_PATH` adjustments in typical scenarios. -pub fn setup_python_linking() { - // Instruct Cargo to rerun the calling build script if the Python config changes. - // Using PYO3_CONFIG_FILE is a reliable way to detect changes in the active Python env. - println!("cargo:rerun-if-changed=build.rs"); - - // Set up #[cfg] flags first (useful for conditional compilation) - pyo3_build_config::use_pyo3_cfgs(); - - // Get the Python interpreter configuration directly - let config = pyo3_build_config::get(); - - let is_extension_module = std::env::var("CARGO_FEATURE_EXTENSION_MODULE").is_ok(); - - // Only link libpython explicitly if we are NOT building an extension module. - if !is_extension_module { - if let Some(lib_name) = &config.lib_name { - println!("cargo:rustc-link-lib=dylib={lib_name}"); - } else { - // Warn only if linking is actually needed but we can't find the lib name - println!("cargo:warning=Python library name not found in config (needed for non-extension module -builds)."); - } - } - - // Add the library search path and RPATH if available. - // These are needed for test executables and potential future standalone binaries, - // and generally harmless for extension modules. - if let Some(lib_dir) = &config.lib_dir { - println!("cargo:rustc-link-search=native={lib_dir}"); - #[cfg(not(windows))] - println!("cargo:rustc-link-arg=-Wl,-rpath,{lib_dir}"); - } else { - // Warn only if linking is actually needed but we can't find the lib dir - if !is_extension_module { - println!("cargo:warning=Python library directory not found in config."); - } - } -} diff --git a/crates/djls-dev/src/lib.rs b/crates/djls-dev/src/lib.rs deleted file mode 100644 index 84752da..0000000 --- a/crates/djls-dev/src/lib.rs +++ /dev/null @@ -1,3 +0,0 @@ -mod build; - -pub use build::setup_python_linking; diff --git a/crates/djls-project/Cargo.toml b/crates/djls-project/Cargo.toml index c5f9485..41311e4 100644 --- a/crates/djls-project/Cargo.toml +++ b/crates/djls-project/Cargo.toml @@ -3,22 +3,20 @@ name = "djls-project" version = "0.0.0" edition = "2021" -[features] -extension-module = [] -default = [] - [dependencies] djls-workspace = { workspace = true } -pyo3 = { workspace = true } +anyhow = { workspace = true } salsa = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +tempfile = { workspace = true } which = { workspace = true} [build-dependencies] -djls-dev = { workspace = true } +which = { workspace = true } [dev-dependencies] -tempfile = { workspace = true } [lints] workspace = true diff --git a/crates/djls-project/build.rs b/crates/djls-project/build.rs index bcfdfd6..4e9633d 100644 --- a/crates/djls-project/build.rs +++ b/crates/djls-project/build.rs @@ -1,3 +1,56 @@ -fn main() { - djls_dev::setup_python_linking(); +use std::env; +use std::path::PathBuf; +use std::process::Command; + +fn get_python_executable_path() -> PathBuf { + let python = which::which("python3") + .or_else(|_| which::which("python")) + .expect("Python not found. Please install Python to build this project."); + println!( + "cargo:warning=Building Python inspector with: {}", + python.display() + ); + python +} + +fn main() { + println!("cargo:rerun-if-changed=../../python/src/djls_inspector"); + println!("cargo:rerun-if-changed=../../python/build.py"); + + let workspace_dir = env::var("CARGO_WORKSPACE_DIR").expect("CARGO_WORKSPACE_DIR not set"); + let workspace_path = PathBuf::from(workspace_dir); + let python_dir = workspace_path.join("python"); + let dist_dir = python_dir.join("dist"); + let pyz_path = dist_dir.join("djls_inspector.pyz"); + + std::fs::create_dir_all(&dist_dir).expect("Failed to create python/dist directory"); + + let python = get_python_executable_path(); + + let output = Command::new(&python) + .arg("build.py") + .current_dir(&python_dir) + .output() + .expect("Failed to run Python build script"); + + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + let stdout = String::from_utf8_lossy(&output.stdout); + panic!("Failed to build Python inspector:\nSTDOUT:\n{stdout}\nSTDERR:\n{stderr}"); + } + + assert!( + pyz_path.exists(), + "Python inspector zipapp was not created at expected location: {}", + pyz_path.display() + ); + + let metadata = + std::fs::metadata(&pyz_path).expect("Failed to get metadata for inspector zipapp"); + + println!( + "cargo:warning=Successfully built Python inspector: {} ({} bytes)", + pyz_path.display(), + metadata.len() + ); } diff --git a/crates/djls-project/src/inspector.rs b/crates/djls-project/src/inspector.rs new file mode 100644 index 0000000..805f742 --- /dev/null +++ b/crates/djls-project/src/inspector.rs @@ -0,0 +1,20 @@ +pub mod ipc; +pub mod pool; +pub mod queries; + +pub use queries::Query; +use serde::Deserialize; +use serde::Serialize; + +#[derive(Serialize)] +pub struct DjlsRequest { + #[serde(flatten)] + pub query: Query, +} + +#[derive(Debug, Deserialize)] +pub struct DjlsResponse { + pub ok: bool, + pub data: Option, + pub error: Option, +} diff --git a/crates/djls-project/src/inspector/ipc.rs b/crates/djls-project/src/inspector/ipc.rs new file mode 100644 index 0000000..903af9c --- /dev/null +++ b/crates/djls-project/src/inspector/ipc.rs @@ -0,0 +1,128 @@ +use std::io::BufRead; +use std::io::BufReader; +use std::io::Write; +use std::path::Path; +use std::process::Child; +use std::process::Command; +use std::process::Stdio; + +use anyhow::Context; +use anyhow::Result; +use serde_json; +use tempfile::NamedTempFile; + +use super::DjlsRequest; +use super::DjlsResponse; +use crate::python::PythonEnvironment; + +const INSPECTOR_PYZ: &[u8] = include_bytes!(concat!( + env!("CARGO_WORKSPACE_DIR"), + "/python/dist/djls_inspector.pyz" +)); + +pub struct InspectorProcess { + child: Child, + stdin: std::process::ChildStdin, + stdout: BufReader, + _zipapp_file: NamedTempFile, +} + +impl InspectorProcess { + pub fn new(python_env: &PythonEnvironment, project_path: &Path) -> Result { + let mut zipapp_file = tempfile::Builder::new() + .prefix("djls_inspector_") + .suffix(".pyz") + .tempfile() + .context("Failed to create temp file for inspector")?; + + zipapp_file + .write_all(INSPECTOR_PYZ) + .context("Failed to write inspector zipapp to temp file")?; + zipapp_file + .flush() + .context("Failed to flush inspector zipapp")?; + + let zipapp_path = zipapp_file.path(); + + let mut cmd = Command::new(&python_env.python_path); + cmd.arg(zipapp_path) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::inherit()) + .current_dir(project_path); + + if let Ok(pythonpath) = std::env::var("PYTHONPATH") { + let mut paths = vec![project_path.to_string_lossy().to_string()]; + paths.push(pythonpath); + cmd.env("PYTHONPATH", paths.join(":")); + } else { + cmd.env("PYTHONPATH", project_path); + } + + if let Ok(settings) = std::env::var("DJANGO_SETTINGS_MODULE") { + cmd.env("DJANGO_SETTINGS_MODULE", settings); + } else { + // Try to detect settings module + if project_path.join("manage.py").exists() { + // Look for common settings modules + for candidate in &["settings", "config.settings", "project.settings"] { + let parts: Vec<&str> = candidate.split('.').collect(); + let mut path = project_path.to_path_buf(); + for part in &parts[..parts.len() - 1] { + path = path.join(part); + } + if let Some(last) = parts.last() { + path = path.join(format!("{last}.py")); + } + + if path.exists() { + cmd.env("DJANGO_SETTINGS_MODULE", candidate); + break; + } + } + } + } + + let mut child = cmd.spawn().context("Failed to spawn inspector process")?; + + let stdin = child.stdin.take().context("Failed to get stdin handle")?; + let stdout = BufReader::new(child.stdout.take().context("Failed to get stdout handle")?); + + Ok(Self { + child, + stdin, + stdout, + _zipapp_file: zipapp_file, + }) + } + + /// Send a request and receive a response + pub fn query(&mut self, request: &DjlsRequest) -> Result { + let request_json = serde_json::to_string(request)?; + + writeln!(self.stdin, "{request_json}")?; + self.stdin.flush()?; + + let mut response_line = String::new(); + self.stdout + .read_line(&mut response_line) + .context("Failed to read response from inspector")?; + + let response: DjlsResponse = + serde_json::from_str(&response_line).context("Failed to parse inspector response")?; + + Ok(response) + } + + pub fn is_running(&mut self) -> bool { + matches!(self.child.try_wait(), Ok(None)) + } +} + +impl Drop for InspectorProcess { + fn drop(&mut self) { + // Try to terminate the child process gracefully + let _ = self.child.kill(); + let _ = self.child.wait(); + } +} diff --git a/crates/djls-project/src/inspector/pool.rs b/crates/djls-project/src/inspector/pool.rs new file mode 100644 index 0000000..1db929a --- /dev/null +++ b/crates/djls-project/src/inspector/pool.rs @@ -0,0 +1,206 @@ +use std::path::Path; +use std::sync::Arc; +use std::sync::Mutex; +use std::time::Duration; +use std::time::Instant; + +use anyhow::Result; + +use super::ipc::InspectorProcess; +use super::DjlsRequest; +use super::DjlsResponse; +use crate::python::PythonEnvironment; + +/// Global singleton pool for convenience +static GLOBAL_POOL: std::sync::OnceLock = std::sync::OnceLock::new(); + +pub fn global_pool() -> &'static InspectorPool { + GLOBAL_POOL.get_or_init(InspectorPool::new) +} +const DEFAULT_IDLE_TIMEOUT: Duration = Duration::from_secs(60); + +/// Manages a pool of inspector processes with automatic cleanup +#[derive(Clone)] +pub struct InspectorPool { + inner: Arc>, +} + +impl std::fmt::Debug for InspectorPool { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("InspectorPool") + .field("has_active_process", &self.has_active_process()) + .finish() + } +} + +struct InspectorPoolInner { + process: Option, + idle_timeout: Duration, +} + +struct InspectorProcessHandle { + process: InspectorProcess, + last_used: Instant, + python_env: PythonEnvironment, + project_path: std::path::PathBuf, +} + +impl InspectorPool { + #[must_use] + pub fn new() -> Self { + Self::with_timeout(DEFAULT_IDLE_TIMEOUT) + } + + #[must_use] + pub fn with_timeout(idle_timeout: Duration) -> Self { + Self { + inner: Arc::new(Mutex::new(InspectorPoolInner { + process: None, + idle_timeout, + })), + } + } + + /// Execute a query, reusing existing process if available and not idle + /// + /// # Panics + /// + /// Panics if the inspector pool mutex is poisoned (another thread panicked while holding the lock) + pub fn query( + &self, + python_env: &PythonEnvironment, + project_path: &Path, + request: &DjlsRequest, + ) -> Result { + let mut inner = self.inner.lock().expect("Inspector pool mutex poisoned"); + let idle_timeout = inner.idle_timeout; + + // Check if we need to drop the existing process + let need_new_process = if let Some(handle) = &mut inner.process { + let idle_too_long = handle.last_used.elapsed() > idle_timeout; + let not_running = !handle.process.is_running(); + let different_env = + handle.python_env != *python_env || handle.project_path != project_path; + + idle_too_long || not_running || different_env + } else { + true + }; + + if need_new_process { + inner.process = None; + } + + // Get or create process + if inner.process.is_none() { + let process = InspectorProcess::new(python_env, project_path)?; + inner.process = Some(InspectorProcessHandle { + process, + last_used: Instant::now(), + python_env: python_env.clone(), + project_path: project_path.to_path_buf(), + }); + } + + // Now we can safely get a mutable reference + let handle = inner + .process + .as_mut() + .expect("Process should exist after creation"); + + // Execute query + let response = handle.process.query(request)?; + handle.last_used = Instant::now(); + + Ok(response) + } + + /// Manually close the inspector process + /// + /// # Panics + /// + /// Panics if the inspector pool mutex is poisoned + pub fn close(&self) { + let mut inner = self.inner.lock().expect("Inspector pool mutex poisoned"); + inner.process = None; + } + + /// Check if there's an active process + /// + /// # Panics + /// + /// Panics if the inspector pool mutex is poisoned + #[must_use] + pub fn has_active_process(&self) -> bool { + let mut inner = self.inner.lock().expect("Inspector pool mutex poisoned"); + if let Some(handle) = &mut inner.process { + handle.process.is_running() && handle.last_used.elapsed() <= inner.idle_timeout + } else { + false + } + } + + /// Get the configured idle timeout + /// + /// # Panics + /// + /// Panics if the inspector pool mutex is poisoned + #[must_use] + pub fn idle_timeout(&self) -> Duration { + let inner = self.inner.lock().expect("Inspector pool mutex poisoned"); + inner.idle_timeout + } + + /// Start a background cleanup task that periodically checks for idle processes + /// + /// # Panics + /// + /// The spawned thread will panic if the inspector pool mutex is poisoned + pub fn start_cleanup_task(self: Arc) { + std::thread::spawn(move || { + loop { + std::thread::sleep(Duration::from_secs(30)); // Check every 30 seconds + + let mut inner = self.inner.lock().expect("Inspector pool mutex poisoned"); + if let Some(handle) = &inner.process { + if handle.last_used.elapsed() > inner.idle_timeout { + // Process is idle, drop it + inner.process = None; + } + } + } + }); + } +} + +impl Default for InspectorPool { + fn default() -> Self { + Self::new() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_pool_creation() { + let pool = InspectorPool::new(); + assert_eq!(pool.idle_timeout(), DEFAULT_IDLE_TIMEOUT); + assert!(!pool.has_active_process()); + } + + #[test] + fn test_pool_with_custom_timeout() { + let timeout = Duration::from_secs(120); + let pool = InspectorPool::with_timeout(timeout); + assert_eq!(pool.idle_timeout(), timeout); + } + + #[test] + fn test_pool_close() { + let pool = InspectorPool::new(); + pool.close(); + assert!(!pool.has_active_process()); + } +} diff --git a/crates/djls-project/src/inspector/queries.rs b/crates/djls-project/src/inspector/queries.rs new file mode 100644 index 0000000..a3460f2 --- /dev/null +++ b/crates/djls-project/src/inspector/queries.rs @@ -0,0 +1,50 @@ +use std::path::PathBuf; + +use serde::Deserialize; +use serde::Serialize; + +#[derive(Serialize, Deserialize)] +#[serde(tag = "query", content = "args")] +#[serde(rename_all = "snake_case")] +pub enum Query { + PythonEnv, + Templatetags, + DjangoInit, +} + +#[derive(Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum VersionReleaseLevel { + Alpha, + Beta, + Candidate, + Final, +} + +#[derive(Serialize, Deserialize)] +pub struct PythonEnvironmentQueryData { + pub sys_base_prefix: PathBuf, + pub sys_executable: PathBuf, + pub sys_path: Vec, + pub sys_platform: String, + pub sys_prefix: PathBuf, + pub sys_version_info: (u32, u32, u32, VersionReleaseLevel, u32), +} + +#[derive(Serialize, Deserialize)] +pub struct TemplateTagQueryData { + pub templatetags: Vec, +} + +#[derive(Serialize, Deserialize)] +pub struct TemplateTag { + pub name: String, + pub module: String, + pub doc: Option, +} + +#[derive(Serialize, Deserialize)] +pub struct DjangoInitQueryData { + pub success: bool, + pub message: Option, +} diff --git a/crates/djls-project/src/lib.rs b/crates/djls-project/src/lib.rs index 685cd10..0761bd7 100644 --- a/crates/djls-project/src/lib.rs +++ b/crates/djls-project/src/lib.rs @@ -1,17 +1,23 @@ mod db; +pub mod inspector; mod meta; -mod python; +pub mod python; mod system; mod templatetags; use std::fmt; use std::path::Path; use std::path::PathBuf; +use std::sync::Arc; +use anyhow::Context; +use anyhow::Result; pub use db::find_python_environment; pub use db::Db; +use inspector::pool::InspectorPool; +use inspector::DjlsRequest; +use inspector::Query; pub use meta::ProjectMetadata; -use pyo3::prelude::*; pub use python::PythonEnvironment; pub use templatetags::TemplateTags; @@ -20,6 +26,7 @@ pub struct DjangoProject { path: PathBuf, env: Option, template_tags: Option, + inspector_pool: Arc, } impl DjangoProject { @@ -29,45 +36,39 @@ impl DjangoProject { path, env: None, template_tags: None, + inspector_pool: Arc::new(InspectorPool::new()), } } - pub fn initialize(&mut self, db: &dyn Db) -> PyResult<()> { + pub fn initialize(&mut self, db: &dyn Db) -> Result<()> { // Use the database to find the Python environment self.env = find_python_environment(db); - if self.env.is_none() { - return Err(PyErr::new::( - "Could not find Python environment", - )); + let env = self + .env + .as_ref() + .context("Could not find Python environment")?; + + // Initialize Django + let request = DjlsRequest { + query: Query::DjangoInit, + }; + let response = self.inspector_pool.query(env, &self.path, &request)?; + + if !response.ok { + anyhow::bail!("Failed to initialize Django: {:?}", response.error); } - Python::with_gil(|py| { - let sys = py.import("sys")?; - let py_path = sys.getattr("path")?; + // Get template tags + let request = DjlsRequest { + query: Query::Templatetags, + }; + let response = self.inspector_pool.query(env, &self.path, &request)?; - if let Some(path_str) = self.path.to_str() { - py_path.call_method1("insert", (0, path_str))?; - } + if let Some(data) = response.data { + self.template_tags = Some(TemplateTags::from_json(&data)?); + } - let env = self.env.as_ref().ok_or_else(|| { - PyErr::new::( - "Internal error: Python environment missing after initialization", - ) - })?; - env.activate(py)?; - - match py.import("django") { - Ok(django) => { - django.call_method0("setup")?; - self.template_tags = Some(TemplateTags::from_python(py)?); - Ok(()) - } - Err(e) => { - eprintln!("Failed to import Django: {e}"); - Err(e) - } - } - }) + Ok(()) } #[must_use] diff --git a/crates/djls-project/src/python.rs b/crates/djls-project/src/python.rs index 87d86f2..2e2810b 100644 --- a/crates/djls-project/src/python.rs +++ b/crates/djls-project/src/python.rs @@ -2,15 +2,13 @@ use std::fmt; use std::path::Path; use std::path::PathBuf; -use pyo3::prelude::*; - use crate::system; #[derive(Clone, Debug, PartialEq)] pub struct PythonEnvironment { - python_path: PathBuf, - sys_path: Vec, - sys_prefix: PathBuf, + pub python_path: PathBuf, + pub sys_path: Vec, + pub sys_prefix: PathBuf, } impl PythonEnvironment { @@ -74,19 +72,6 @@ impl PythonEnvironment { }) } - pub fn activate(&self, py: Python) -> PyResult<()> { - let sys = py.import("sys")?; - let py_path = sys.getattr("path")?; - - for path in &self.sys_path { - if let Some(path_str) = path.to_str() { - py_path.call_method1("append", (path_str,))?; - } - } - - Ok(()) - } - fn from_system_python() -> Option { let Ok(python_path) = system::find_executable("python") else { return None; @@ -181,20 +166,6 @@ mod tests { prefix } - fn get_sys_path(py: Python) -> PyResult> { - let sys = py.import("sys")?; - let py_path = sys.getattr("path")?; - py_path.extract::>() - } - - fn create_test_env(sys_paths: Vec) -> PythonEnvironment { - PythonEnvironment { - python_path: PathBuf::from("dummy/bin/python"), - sys_prefix: PathBuf::from("dummy"), - sys_path: sys_paths, - } - } - mod env_discovery { use which::Error as WhichError; @@ -493,213 +464,6 @@ mod tests { } } - mod env_activation { - use super::*; - - #[test] - #[ignore = "Requires Python runtime - run with --ignored flag"] - fn test_activate_appends_paths() -> PyResult<()> { - let temp_dir = tempdir().unwrap(); - let path1 = temp_dir.path().join("scripts"); - let path2 = temp_dir.path().join("libs"); - fs::create_dir_all(&path1).unwrap(); - fs::create_dir_all(&path2).unwrap(); - - let test_env = create_test_env(vec![path1.clone(), path2.clone()]); - - pyo3::prepare_freethreaded_python(); - - Python::with_gil(|py| { - let initial_sys_path = get_sys_path(py)?; - let initial_len = initial_sys_path.len(); - - test_env.activate(py)?; - - let final_sys_path = get_sys_path(py)?; - assert_eq!( - final_sys_path.len(), - initial_len + 2, - "Should have added 2 paths" - ); - assert_eq!( - final_sys_path.get(initial_len).unwrap(), - path1.to_str().expect("Path 1 should be valid UTF-8") - ); - assert_eq!( - final_sys_path.get(initial_len + 1).unwrap(), - path2.to_str().expect("Path 2 should be valid UTF-8") - ); - - Ok(()) - }) - } - - #[test] - #[ignore = "Requires Python runtime - run with --ignored flag"] - fn test_activate_empty_sys_path() -> PyResult<()> { - let test_env = create_test_env(vec![]); - - pyo3::prepare_freethreaded_python(); - - Python::with_gil(|py| { - let initial_sys_path = get_sys_path(py)?; - - test_env.activate(py)?; - - let final_sys_path = get_sys_path(py)?; - assert_eq!( - final_sys_path, initial_sys_path, - "sys.path should remain unchanged for empty env.sys_path" - ); - - Ok(()) - }) - } - - #[test] - #[ignore = "Requires Python runtime - run with --ignored flag"] - fn test_activate_with_non_existent_paths() -> PyResult<()> { - let temp_dir = tempdir().unwrap(); - let path1 = temp_dir.path().join("non_existent_dir"); - let path2 = temp_dir.path().join("another_missing/path"); - - let test_env = create_test_env(vec![path1.clone(), path2.clone()]); - - pyo3::prepare_freethreaded_python(); - - Python::with_gil(|py| { - let initial_sys_path = get_sys_path(py)?; - let initial_len = initial_sys_path.len(); - - test_env.activate(py)?; - - let final_sys_path = get_sys_path(py)?; - assert_eq!( - final_sys_path.len(), - initial_len + 2, - "Should still add 2 paths even if they don't exist" - ); - assert_eq!( - final_sys_path.get(initial_len).unwrap(), - path1.to_str().unwrap() - ); - assert_eq!( - final_sys_path.get(initial_len + 1).unwrap(), - path2.to_str().unwrap() - ); - - Ok(()) - }) - } - - #[test] - #[cfg(unix)] - #[ignore = "Requires Python runtime - run with --ignored flag"] - fn test_activate_skips_non_utf8_paths_unix() -> PyResult<()> { - use std::ffi::OsStr; - use std::os::unix::ffi::OsStrExt; - - let temp_dir = tempdir().unwrap(); - let valid_path = temp_dir.path().join("valid_dir"); - fs::create_dir(&valid_path).unwrap(); - - let invalid_bytes = b"invalid_\xff_utf8"; - let os_str = OsStr::from_bytes(invalid_bytes); - let non_utf8_path = PathBuf::from(os_str); - assert!( - non_utf8_path.to_str().is_none(), - "Path should not be convertible to UTF-8 str" - ); - - let test_env = create_test_env(vec![valid_path.clone(), non_utf8_path.clone()]); - - pyo3::prepare_freethreaded_python(); - - Python::with_gil(|py| { - let initial_sys_path = get_sys_path(py)?; - let initial_len = initial_sys_path.len(); - - test_env.activate(py)?; - - let final_sys_path = get_sys_path(py)?; - assert_eq!( - final_sys_path.len(), - initial_len + 1, - "Should only add valid UTF-8 paths" - ); - assert_eq!( - final_sys_path.get(initial_len).unwrap(), - valid_path.to_str().unwrap() - ); - - let invalid_path_lossy = non_utf8_path.to_string_lossy(); - assert!( - !final_sys_path - .iter() - .any(|p| p.contains(&*invalid_path_lossy)), - "Non-UTF8 path should not be present in sys.path" - ); - - Ok(()) - }) - } - - #[test] - #[cfg(windows)] - #[ignore = "Requires Python runtime - run with --ignored flag"] - fn test_activate_skips_non_utf8_paths_windows() -> PyResult<()> { - use std::ffi::OsString; - use std::os::windows::ffi::OsStringExt; - - let temp_dir = tempdir().unwrap(); - let valid_path = temp_dir.path().join("valid_dir"); - - let invalid_wide: Vec = vec![ - 'i' as u16, 'n' as u16, 'v' as u16, 'a' as u16, 'l' as u16, 'i' as u16, 'd' as u16, - '_' as u16, 0xD800, '_' as u16, 'w' as u16, 'i' as u16, 'd' as u16, 'e' as u16, - ]; - let os_string = OsString::from_wide(&invalid_wide); - let non_utf8_path = PathBuf::from(os_string); - - assert!( - non_utf8_path.to_str().is_none(), - "Path with lone surrogate should not be convertible to UTF-8 str" - ); - - let test_env = create_test_env(vec![valid_path.clone(), non_utf8_path.clone()]); - - pyo3::prepare_freethreaded_python(); - - Python::with_gil(|py| { - let initial_sys_path = get_sys_path(py)?; - let initial_len = initial_sys_path.len(); - - test_env.activate(py)?; - - let final_sys_path = get_sys_path(py)?; - assert_eq!( - final_sys_path.len(), - initial_len + 1, - "Should only add paths convertible to valid UTF-8" - ); - assert_eq!( - final_sys_path.get(initial_len).unwrap(), - valid_path.to_str().unwrap() - ); - - let invalid_path_lossy = non_utf8_path.to_string_lossy(); - assert!( - !final_sys_path - .iter() - .any(|p| p.contains(&*invalid_path_lossy)), - "Non-UTF8 path (from invalid wide chars) should not be present in sys.path" - ); - - Ok(()) - }) - } - } - mod salsa_integration { use std::sync::Arc; diff --git a/crates/djls-project/src/templatetags.rs b/crates/djls-project/src/templatetags.rs index 7f6f004..237a454 100644 --- a/crates/djls-project/src/templatetags.rs +++ b/crates/djls-project/src/templatetags.rs @@ -1,8 +1,8 @@ use std::ops::Deref; -use pyo3::prelude::*; -use pyo3::types::PyDict; -use pyo3::types::PyList; +use anyhow::Context; +use anyhow::Result; +use serde_json::Value; #[derive(Debug, Default, Clone)] pub struct TemplateTags(Vec); @@ -16,57 +16,46 @@ impl Deref for TemplateTags { } impl TemplateTags { - fn new() -> Self { - Self(Vec::new()) - } + pub fn from_json(data: &Value) -> Result { + let mut tags = Vec::new(); - fn process_library( - module_name: &str, - library: &Bound<'_, PyAny>, - tags: &mut Vec, - ) -> PyResult<()> { - let tags_dict = library.getattr("tags")?; - let dict = tags_dict.downcast::()?; + // Parse the JSON response from the inspector + let templatetags = data + .get("templatetags") + .context("Missing 'templatetags' field in response")? + .as_array() + .context("'templatetags' field is not an array")?; - for (key, value) in dict.iter() { - let tag_name = key.extract::()?; - let doc = value.getattr("__doc__")?.extract().ok(); + for tag_data in templatetags { + let name = tag_data + .get("name") + .and_then(|v| v.as_str()) + .context("Missing or invalid 'name' field")? + .to_string(); - let library_name = if module_name.is_empty() { - "builtins".to_string() - } else { - module_name.split('.').next_back().unwrap_or("").to_string() - }; + let module = tag_data + .get("module") + .and_then(|v| v.as_str()) + .context("Missing or invalid 'module' field")?; - tags.push(TemplateTag::new(tag_name, library_name, doc)); - } - Ok(()) - } + // Extract library name from module (e.g., "django.templatetags.static" -> "static") + let library = module + .split('.') + .filter(|part| part.contains("templatetags")) + .nth(1) + .or_else(|| module.split('.').next_back()) + .unwrap_or("builtins") + .to_string(); - pub fn from_python(py: Python) -> PyResult { - let mut template_tags = TemplateTags::new(); + let doc = tag_data + .get("doc") + .and_then(|v| v.as_str()) + .map(String::from); - let engine = py - .import("django.template.engine")? - .getattr("Engine")? - .call_method0("get_default")?; - - // Built-in template tags - let builtins_attr = engine.getattr("template_builtins")?; - let builtins = builtins_attr.downcast::()?; - for builtin in builtins { - Self::process_library("", &builtin, &mut template_tags.0)?; + tags.push(TemplateTag::new(name, library, doc)); } - // Custom template libraries - let libraries_attr = engine.getattr("template_libraries")?; - let libraries = libraries_attr.downcast::()?; - for (module_name, library) in libraries.iter() { - let module_name = module_name.extract::()?; - Self::process_library(&module_name, &library, &mut template_tags.0)?; - } - - Ok(template_tags) + Ok(TemplateTags(tags)) } } diff --git a/crates/djls-server/Cargo.toml b/crates/djls-server/Cargo.toml index 7829bf0..543e4a6 100644 --- a/crates/djls-server/Cargo.toml +++ b/crates/djls-server/Cargo.toml @@ -3,10 +3,6 @@ name = "djls-server" version = "0.0.0" edition = "2021" -[features] -extension-module = [] -default = [] - [dependencies] djls-conf = { workspace = true } djls-project = { workspace = true } @@ -17,7 +13,6 @@ anyhow = { workspace = true } camino = { workspace = true } dashmap = { workspace = true } percent-encoding = { workspace = true } -pyo3 = { workspace = true } salsa = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } @@ -28,9 +23,6 @@ tracing-appender = { workspace = true } tracing-subscriber = { workspace = true } url = { workspace = true } -[build-dependencies] -djls-dev = { workspace = true } - [dev-dependencies] tempfile = { workspace = true } diff --git a/crates/djls-server/build.rs b/crates/djls-server/build.rs deleted file mode 100644 index bcfdfd6..0000000 --- a/crates/djls-server/build.rs +++ /dev/null @@ -1,3 +0,0 @@ -fn main() { - djls_dev::setup_python_linking(); -} diff --git a/crates/djls-server/src/session.rs b/crates/djls-server/src/session.rs index c2031e3..55a1ba0 100644 --- a/crates/djls-server/src/session.rs +++ b/crates/djls-server/src/session.rs @@ -6,6 +6,7 @@ use std::path::PathBuf; use std::sync::Arc; +use anyhow::Result; use dashmap::DashMap; use djls_conf::Settings; use djls_project::DjangoProject; @@ -15,7 +16,6 @@ use djls_workspace::paths; use djls_workspace::PositionEncoding; use djls_workspace::TextDocument; use djls_workspace::Workspace; -use pyo3::PyResult; use tower_lsp_server::lsp_types; use url::Url; @@ -156,7 +156,7 @@ impl Session { } /// Initialize the project with the database. - pub fn initialize_project(&mut self) -> PyResult<()> { + pub fn initialize_project(&mut self) -> Result<()> { if let Some(project) = self.project.as_mut() { project.initialize(&self.db) } else { diff --git a/crates/djls/Cargo.toml b/crates/djls/Cargo.toml index 4ca96ea..c663ed4 100644 --- a/crates/djls/Cargo.toml +++ b/crates/djls/Cargo.toml @@ -3,29 +3,13 @@ name = "djls" version = "5.2.0" edition = "2021" -[lib] -name = "djls" -crate-type = ["cdylib"] - -[features] -extension-module = [ - "djls-server/extension-module", - "djls-project/extension-module", - "pyo3/extension-module" -] -default = [] - [dependencies] djls-project = { workspace = true } djls-server = { workspace = true } anyhow = { workspace = true } clap = { workspace = true } -pyo3 = { workspace = true } serde_json = { workspace = true } -[build-dependencies] -djls-dev = { workspace = true } - [lints] workspace = true diff --git a/crates/djls/build.rs b/crates/djls/build.rs deleted file mode 100644 index bcfdfd6..0000000 --- a/crates/djls/build.rs +++ /dev/null @@ -1,3 +0,0 @@ -fn main() { - djls_dev::setup_python_linking(); -} diff --git a/crates/djls/src/commands/serve.rs b/crates/djls/src/commands/serve.rs index 3388fdd..2c255df 100644 --- a/crates/djls/src/commands/serve.rs +++ b/crates/djls/src/commands/serve.rs @@ -22,10 +22,6 @@ impl Command for Serve { fn execute(&self, _args: &Args) -> Result { djls_server::run()?; - // Exit here instead of returning control to the `Cli`, for ... reasons? - // If we don't exit here, ~~~ something ~~~ goes on with PyO3 (I assume) - // or the Python entrypoint wrapper to indefinitely hang the CLI and keep - // the process running Exit::success() .with_message("Server completed successfully") .process_exit() diff --git a/crates/djls/src/lib.rs b/crates/djls/src/lib.rs deleted file mode 100644 index 2be9f2b..0000000 --- a/crates/djls/src/lib.rs +++ /dev/null @@ -1,34 +0,0 @@ -/// `PyO3` entrypoint for the Django Language Server CLI. -/// -/// This module provides a Python interface using `PyO3` to solve Python runtime -/// interpreter linking issues. The `PyO3` approach avoids complexities with -/// static/dynamic linking when building binaries that interact with Python. -mod args; -mod cli; -mod commands; -mod exit; - -use std::env; - -use pyo3::prelude::*; - -#[pyfunction] -/// Entry point called by Python when the CLI is invoked. -/// This function handles argument parsing from Python and routes to the Rust CLI logic. -fn entrypoint(_py: Python) -> PyResult<()> { - // Skip python interpreter and script path, add command name - let args: Vec = std::iter::once("djls".to_string()) - .chain(env::args().skip(2)) - .collect(); - - // Run the CLI with the adjusted args - cli::run(args).map_err(|e| pyo3::exceptions::PyRuntimeError::new_err(e.to_string()))?; - - Ok(()) -} - -#[pymodule] -fn djls(_py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { - m.add_function(wrap_pyfunction!(entrypoint, m)?)?; - Ok(()) -} diff --git a/crates/djls/src/main.rs b/crates/djls/src/main.rs index 8872343..dba126d 100644 --- a/crates/djls/src/main.rs +++ b/crates/djls/src/main.rs @@ -1,7 +1,4 @@ -/// Binary interface for local development only. -/// -/// This binary exists for development and testing with `cargo run`. -/// The production CLI is distributed through the `PyO3` interface in lib.rs. +/// Binary interface for the Django Language Server CLI. mod args; mod cli; mod commands; diff --git a/docs/index.md b/docs/index.md index 80d4017..b232bf6 100644 --- a/docs/index.md +++ b/docs/index.md @@ -48,7 +48,7 @@ The foundation is solid though: - [x] Working server architecture - [x] Language Server Protocol implementation in Rust - - [x] Direct Django project interaction through PyO3 + - [x] Django interaction via IPC - [x] Single binary distribution with Python packaging - [x] Custom template parser to support LSP features - [x] Basic HTML parsing, including style and script tags @@ -178,7 +178,7 @@ All feature requests should ideally start out as a discussion topic, to gather f ### Development -The project is written in Rust using PyO3 for Python integration. Here is a high-level overview of the project and the various crates: +The project is written in Rust with IPC for Python communication. Here is a high-level overview of the project and the various crates: - Main CLI interface ([`crates/djls/`](https://github.com/joshuadavidthomas/django-language-server/blob/main/crates/djls/)) - Django and Python project introspection ([`crates/djls-project/`](https://github.com/joshuadavidthomas/django-language-server/blob/main/crates/djls-project/)) diff --git a/noxfile.py b/noxfile.py index 194b569..c61ce5b 100644 --- a/noxfile.py +++ b/noxfile.py @@ -94,12 +94,6 @@ def tests(session, django): command = ["cargo", "test"] - # TODO: Remove this exclusion once PyO3 is replaced with subprocess oracle pattern - # Temporarily exclude djls-project tests on Windows due to PyO3 DLL loading issues - # (STATUS_DLL_NOT_FOUND when the test executable tries to load Python) - if platform.system() == "Windows": - command.extend(["--workspace", "--exclude", "djls-project"]) - if session.posargs: args = [] for arg in session.posargs: @@ -148,9 +142,6 @@ def gha_matrix(session): include_list = [] for os_name in os_list: for combo in versions_list: - # Skip Python 3.9 on macOS due to PyO3/framework linking issues - if os_name.startswith("macos") and combo["python-version"] == "3.9": - continue include_list.append({**combo, "os": os_name}) matrix = {"include": include_list} diff --git a/pyproject.toml b/pyproject.toml index 291beb4..4d0a487 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -69,9 +69,6 @@ classifiers = [ "Topic :: Text Editors :: Integrated Development Environments (IDE)" ] -[project.scripts] -djls = "djls:entrypoint" - [project.urls] Documentation = "https://django-language-server.readthedocs.io/" Issues = "https://github.com/joshuadavidthomas/django-language-server/issues" diff --git a/python/Justfile b/python/Justfile new file mode 100644 index 0000000..0c473e9 --- /dev/null +++ b/python/Justfile @@ -0,0 +1,12 @@ +set unstable := true + +[private] +default: + @just --list + +[private] +fmt: + @just --fmt + +build: + uv run build.py diff --git a/python/build.py b/python/build.py new file mode 100644 index 0000000..c3d1815 --- /dev/null +++ b/python/build.py @@ -0,0 +1,24 @@ +from __future__ import annotations + +import zipapp +from pathlib import Path + + +def main(): + source_dir = Path(__file__).parent / "src" / "djls_inspector" + output_file = Path(__file__).parent / "dist" / "djls_inspector.pyz" + output_file.parent.mkdir(exist_ok=True) + + zipapp.create_archive( + source_dir, + target=output_file, + interpreter=None, # No shebang - will be invoked explicitly + compressed=True, + ) + + print(f"Successfully created {output_file}") + print(f"Size: {output_file.stat().st_size} bytes") + + +if __name__ == "__main__": + main() diff --git a/python/pyproject.toml b/python/pyproject.toml new file mode 100644 index 0000000..2dff63f --- /dev/null +++ b/python/pyproject.toml @@ -0,0 +1,12 @@ +[build-system] +requires = ["uv_build>=0.7.21,<0.8"] +build-backend = "uv_build" + +[project] +authors = [ + { name = "Josh Thomas", email = "josh@joshthomas.dev" } +] +dependencies = [] +name = "djls-inspector" +requires-python = ">=3.9" +version = "0.1.0" diff --git a/python/src/djls_inspector/__init__.py b/python/src/djls_inspector/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/python/src/djls_inspector/__main__.py b/python/src/djls_inspector/__main__.py new file mode 100644 index 0000000..a67359f --- /dev/null +++ b/python/src/djls_inspector/__main__.py @@ -0,0 +1,44 @@ +from __future__ import annotations + +import json +import sys + +# When running from zipapp, we need to import without the package prefix +# since the zipapp root is already the package +try: + # Try direct import (when running as zipapp) + from inspector import DjlsResponse + from inspector import handle_request +except ImportError: + # Fall back to package import (when running with python -m) + from djls_inspector.inspector import DjlsResponse + from djls_inspector.inspector import handle_request + + +def main() -> None: + try: + for line in sys.stdin: + line = line.strip() + if not line: + continue + + try: + request = json.loads(line) + response = handle_request(request) + except json.JSONDecodeError as e: + response = DjlsResponse(ok=False, error=f"Invalid JSON: {e}") + except Exception as e: + response = DjlsResponse(ok=False, error=f"Unexpected error: {e}") + + response_json = json.dumps(response.to_dict()) + print(response_json, flush=True) + + except KeyboardInterrupt: + sys.exit(0) + except Exception as e: + print(f"Fatal error in inspector: {e}", file=sys.stderr) + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/python/src/djls_inspector/inspector.py b/python/src/djls_inspector/inspector.py new file mode 100644 index 0000000..43dd2d7 --- /dev/null +++ b/python/src/djls_inspector/inspector.py @@ -0,0 +1,77 @@ +from __future__ import annotations + +from dataclasses import asdict +from dataclasses import dataclass +from typing import Any + +try: + # Try direct import (when running as zipapp) + from queries import Query + from queries import QueryData + from queries import get_installed_templatetags + from queries import get_python_environment_info + from queries import initialize_django +except ImportError: + # Fall back to relative import (when running with python -m) + from .queries import Query + from .queries import QueryData + from .queries import get_installed_templatetags + from .queries import get_python_environment_info + from .queries import initialize_django + + +@dataclass +class DjlsRequest: + query: Query + args: list[str] | None = None + + +@dataclass +class DjlsResponse: + ok: bool + data: QueryData | None = None + error: str | None = None + + def to_dict(self) -> dict[str, Any]: + d = asdict(self) + # Convert Path objects to strings for JSON serialization + if self.data: + if hasattr(self.data, "__dataclass_fields__"): + data_dict = asdict(self.data) + # Convert Path objects to strings + for key, value in data_dict.items(): + if key in ["sys_base_prefix", "sys_executable", "sys_prefix"]: + if value: + data_dict[key] = str(value) + elif key == "sys_path": + data_dict[key] = [str(p) for p in value] + d["data"] = data_dict + return d + + +def handle_request(request: dict[str, Any]) -> DjlsResponse: + try: + query_str = request.get("query") + if not query_str: + return DjlsResponse(ok=False, error="Missing 'query' field in request") + + try: + query = Query(query_str) + except ValueError: + return DjlsResponse(ok=False, error=f"Unknown query type: {query_str}") + + args = request.get("args") + + if query == Query.PYTHON_ENV: + return DjlsResponse(ok=True, data=get_python_environment_info()) + + elif query == Query.TEMPLATETAGS: + return DjlsResponse(ok=True, data=get_installed_templatetags()) + + elif query == Query.DJANGO_INIT: + return DjlsResponse(ok=True, data=initialize_django()) + + return DjlsResponse(ok=False, error=f"Unhandled query type: {query}") + + except Exception as e: + return DjlsResponse(ok=False, error=str(e)) diff --git a/python/src/djls_inspector/queries.py b/python/src/djls_inspector/queries.py new file mode 100644 index 0000000..3204614 --- /dev/null +++ b/python/src/djls_inspector/queries.py @@ -0,0 +1,174 @@ +from __future__ import annotations + +import sys +from dataclasses import dataclass +from enum import Enum +from pathlib import Path +from typing import Literal + + +class Query(str, Enum): + PYTHON_ENV = "python_env" + TEMPLATETAGS = "templatetags" + DJANGO_INIT = "django_init" + + +@dataclass +class PythonEnvironmentQueryData: + sys_base_prefix: Path + sys_executable: Path + sys_path: list[Path] + sys_platform: str + sys_prefix: Path + sys_version_info: tuple[ + int, int, int, Literal["alpha", "beta", "candidate", "final"], int + ] + + +def get_python_environment_info(): + return PythonEnvironmentQueryData( + sys_base_prefix=Path(sys.base_prefix), + sys_executable=Path(sys.executable), + sys_path=[Path(p) for p in sys.path], + sys_platform=sys.platform, + sys_prefix=Path(sys.prefix), + sys_version_info=( + sys.version_info.major, + sys.version_info.minor, + sys.version_info.micro, + sys.version_info.releaselevel, + sys.version_info.serial, + ), + ) + + +@dataclass +class TemplateTagQueryData: + templatetags: list[TemplateTag] + + +@dataclass +class TemplateTag: + name: str + module: str + doc: str | None + + +def get_installed_templatetags() -> TemplateTagQueryData: + import django + from django.apps import apps + from django.template.engine import Engine + from django.template.library import import_library + + # Ensure Django is set up + if not apps.ready: + django.setup() + + templatetags: list[TemplateTag] = [] + + engine = Engine.get_default() + + for library in engine.template_builtins: + if library.tags: + for tag_name, tag_func in library.tags.items(): + templatetags.append( + TemplateTag( + name=tag_name, module=tag_func.__module__, doc=tag_func.__doc__ + ) + ) + + for lib_module in engine.libraries.values(): + library = import_library(lib_module) + if library and library.tags: + for tag_name, tag_func in library.tags.items(): + templatetags.append( + TemplateTag( + name=tag_name, module=tag_func.__module__, doc=tag_func.__doc__ + ) + ) + + return TemplateTagQueryData(templatetags=templatetags) + + +@dataclass +class DjangoInitQueryData: + success: bool + message: str | None = None + + +def initialize_django() -> DjangoInitQueryData: + import os + import django + from django.apps import apps + + try: + # Check if Django settings are configured + if not os.environ.get("DJANGO_SETTINGS_MODULE"): + # Try to find and set settings module + import sys + from pathlib import Path + + # Look for manage.py to determine project structure + current_path = Path.cwd() + manage_py = None + + # Search up to 3 levels for manage.py + for _ in range(3): + if (current_path / "manage.py").exists(): + manage_py = current_path / "manage.py" + break + if current_path.parent == current_path: + break + current_path = current_path.parent + + if not manage_py: + return DjangoInitQueryData( + success=False, + message="Could not find manage.py or DJANGO_SETTINGS_MODULE not set", + ) + + # Add project directory to sys.path + project_dir = manage_py.parent + if str(project_dir) not in sys.path: + sys.path.insert(0, str(project_dir)) + + # Try to find settings module - look for common patterns + # First check if there's a directory with the same name as the parent + project_name = project_dir.name + settings_candidates = [ + f"{project_name}.settings", # e.g., myproject.settings + "settings", # Just settings.py in root + "config.settings", # Common pattern + "project.settings", # Another common pattern + ] + + # Also check for any directory containing settings.py + for item in project_dir.iterdir(): + if item.is_dir() and (item / "settings.py").exists(): + candidate = f"{item.name}.settings" + if candidate not in settings_candidates: + settings_candidates.insert( + 0, candidate + ) # Prioritize found settings + + for settings_candidate in settings_candidates: + try: + __import__(settings_candidate) + os.environ["DJANGO_SETTINGS_MODULE"] = settings_candidate + break + except ImportError: + continue + + # Set up Django + if not apps.ready: + django.setup() + + return DjangoInitQueryData( + success=True, message="Django initialized successfully" + ) + + except Exception as e: + return DjangoInitQueryData(success=False, message=str(e)) + + +QueryData = PythonEnvironmentQueryData | TemplateTagQueryData | DjangoInitQueryData diff --git a/python/uv.lock b/python/uv.lock new file mode 100644 index 0000000..76e7051 --- /dev/null +++ b/python/uv.lock @@ -0,0 +1,8 @@ +version = 1 +revision = 2 +requires-python = ">=3.9" + +[[package]] +name = "djls-inspector" +version = "0.1.0" +source = { editable = "." } diff --git a/tests/project/djls_app/templates/djls_app/base.html b/tests/project/djls_app/templates/djls_app/base.html index b2b60a8..6856111 100644 --- a/tests/project/djls_app/templates/djls_app/base.html +++ b/tests/project/djls_app/templates/djls_app/base.html @@ -19,7 +19,7 @@ {% endif %} Logo {# This is a comment #} - {% endblock %} + {% endblock content %} diff --git a/uv.lock b/uv.lock index 4b87572..448a00d 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 3 +revision = 2 requires-python = ">=3.9" resolution-markers = [ "python_full_version >= '3.10'", @@ -277,7 +277,7 @@ wheels = [ [[package]] name = "django-language-server" -version = "5.2.0a0" +version = "5.2.0" source = { editable = "." } [package.dev-dependencies]