Replace PyO3 with IPC approach for Python/project information (#214)

.github/workflows/build.yml

@@ -42,7 +42,7 @@ jobs:
         uses: PyO3/maturin-action@86b9d133d34bc1b40018696f782949dac11bd380
         with:
           target: ${{ matrix.platform.target }}
-          args: --release --out dist --find-interpreter --features extension-module
+          args: --release --out dist --find-interpreter
           sccache: ${{ !startsWith(github.ref, 'refs/tags/') }}
           manylinux: auto
 
@@ -78,7 +78,7 @@ jobs:
         uses: PyO3/maturin-action@86b9d133d34bc1b40018696f782949dac11bd380
         with:
           target: ${{ matrix.platform.target }}
-          args: --release --out dist --find-interpreter --features extension-module
+          args: --release --out dist --find-interpreter
           sccache: ${{ !startsWith(github.ref, 'refs/tags/') }}
           manylinux: musllinux_1_2
 
@@ -111,7 +111,7 @@ jobs:
         uses: PyO3/maturin-action@86b9d133d34bc1b40018696f782949dac11bd380
         with:
           target: ${{ matrix.platform.target }}
-          args: --release --out dist --find-interpreter --features extension-module
+          args: --release --out dist --find-interpreter
           sccache: ${{ !startsWith(github.ref, 'refs/tags/') }}
 
       - name: Upload wheels
@@ -142,7 +142,7 @@ jobs:
         uses: PyO3/maturin-action@86b9d133d34bc1b40018696f782949dac11bd380
         with:
           target: ${{ matrix.platform.target }}
-          args: --release --out dist --find-interpreter --features extension-module
+          args: --release --out dist --find-interpreter
           sccache: ${{ !startsWith(github.ref, 'refs/tags/') }}
 
       - name: Upload wheels

.lazy.lua

@@ -1,5 +1,5 @@
 vim.lsp.config["djls"] = {
-  cmd = { "uvx", "lsp-devtools", "agent", "--", "djls", "serve" },
+  cmd = { "uvx", "lsp-devtools", "agent", "--", "./target/debug/djls", "serve" },
   filetypes = { "htmldjango" },
   root_markers = { "manage.py", "pyproject.toml" },
 }

Cargo.lock

@@ -426,10 +426,8 @@ version = "5.2.0"
 dependencies = [
  "anyhow",
  "clap",
- "djls-dev",
  "djls-project",
  "djls-server",
- "pyo3",
  "serde_json",
 ]
 
@@ -446,22 +444,15 @@ dependencies = [
  "toml",
 ]
 
-[[package]]
-name = "djls-dev"
-version = "0.0.0"
-dependencies = [
- "anyhow",
- "pyo3-build-config",
-]
-
 [[package]]
 name = "djls-project"
 version = "0.0.0"
 dependencies = [
- "djls-dev",
+ "anyhow",
  "djls-workspace",
- "pyo3",
  "salsa",
+ "serde",
+ "serde_json",
  "tempfile",
  "which",
 ]
@@ -474,12 +465,10 @@ dependencies = [
  "camino",
  "dashmap",
  "djls-conf",
- "djls-dev",
  "djls-project",
  "djls-templates",
  "djls-workspace",
  "percent-encoding",
- "pyo3",
  "salsa",
  "serde",
  "serde_json",
@@ -896,12 +885,6 @@ dependencies = [
  "hashbrown 0.15.5",
 ]
 
-[[package]]
-name = "indoc"
-version = "2.0.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f4c7245a08504955605670dbf141fceab975f15ca21570696aebe9d2e71576bd"
-
 [[package]]
 name = "inotify"
 version = "0.11.0"
@@ -1312,68 +1295,6 @@ dependencies = [
  "unicode-ident",
 ]
 
-[[package]]
-name = "pyo3"
-version = "0.25.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8970a78afe0628a3e3430376fc5fd76b6b45c4d43360ffd6cdd40bdde72b682a"
-dependencies = [
- "indoc",
- "libc",
- "memoffset",
- "once_cell",
- "portable-atomic",
- "pyo3-build-config",
- "pyo3-ffi",
- "pyo3-macros",
- "unindent",
-]
-
-[[package]]
-name = "pyo3-build-config"
-version = "0.25.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "458eb0c55e7ece017adeba38f2248ff3ac615e53660d7c71a238d7d2a01c7598"
-dependencies = [
- "once_cell",
- "target-lexicon",
-]
-
-[[package]]
-name = "pyo3-ffi"
-version = "0.25.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7114fe5457c61b276ab77c5055f206295b812608083644a5c5b2640c3102565c"
-dependencies = [
- "libc",
- "pyo3-build-config",
-]
-
-[[package]]
-name = "pyo3-macros"
-version = "0.25.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a8725c0a622b374d6cb051d11a0983786448f7785336139c3c94f5aa6bef7e50"
-dependencies = [
- "proc-macro2",
- "pyo3-macros-backend",
- "quote",
- "syn",
-]
-
-[[package]]
-name = "pyo3-macros-backend"
-version = "0.25.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4109984c22491085343c05b0dbc54ddc405c3cf7b4374fc533f5c3313a572ccc"
-dependencies = [
- "heck",
- "proc-macro2",
- "pyo3-build-config",
- "quote",
- "syn",
-]
-
 [[package]]
 name = "quote"
 version = "1.0.40"
@@ -1728,12 +1649,6 @@ dependencies = [
  "syn",
 ]
 
-[[package]]
-name = "target-lexicon"
-version = "0.13.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e502f78cdbb8ba4718f566c418c52bc729126ffd16baee5baa718cf25dd5a69a"
-
 [[package]]
 name = "tempfile"
 version = "3.21.0"
@@ -2092,12 +2007,6 @@ version = "1.12.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493"
 
-[[package]]
-name = "unindent"
-version = "0.2.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3"
-
 [[package]]
 name = "url"
 version = "2.5.7"

Cargo.toml

@@ -5,15 +5,12 @@ resolver = "2"
 [workspace.dependencies]
 djls = { path = "crates/djls" }
 djls-conf = { path = "crates/djls-conf" }
-djls-dev = { path = "crates/djls-dev" }
 djls-project = { path = "crates/djls-project" }
 djls-server = { path = "crates/djls-server" }
 djls-templates = { path = "crates/djls-templates" }
 djls-workspace = { path = "crates/djls-workspace" }
 
 # core deps, pin exact versions
-pyo3 = "0.25.0"
-pyo3-build-config = { version = "0.25.0", features = ["resolve-config"] }
 salsa = "0.23.0"
 tower-lsp-server = { version = "0.22.0", features = ["proposed"] }
 

README.md

@@ -156,11 +156,10 @@ All feature requests should ideally start out as a discussion topic, to gather f
 
 ### Development
 
-The project is written in Rust using PyO3 for Python integration. Here is a high-level overview of the project and the various crates:
+The project is written in Rust with IPC for Python communication. Here is a high-level overview of the project and the various crates:
 
 - Main CLI interface ([`crates/djls/`](./crates/djls/))
 - Configuration management ([`crates/djls-conf/`](./crates/djls-conf/))
-- Development tools ([`crates/djls-dev/`](./crates/djls-dev/))
 - Django and Python project introspection ([`crates/djls-project/`](./crates/djls-project/))
 - LSP server implementation ([`crates/djls-server/`](./crates/djls-server/))
 - Template parsing ([`crates/djls-templates/`](./crates/djls-templates/))

crates/djls-dev/Cargo.toml

@@ -1,16 +0,0 @@
-[package]
-name = "djls-dev"
-version = "0.0.0"
-edition = "2021"
-publish = false
-
-[[bin]]
-name = "djls-tmux"
-path = "src/bin/djls-tmux.rs"
-
-[dependencies]
-anyhow = { workspace = true }
-pyo3-build-config = { workspace = true }
-
-[lints]
-workspace = true

crates/djls-dev/src/bin/djls-tmux.rs

@@ -1,101 +0,0 @@
-use std::io::Write;
-use std::process::Command;
-use std::process::Stdio;
-
-use anyhow::Context;
-use anyhow::Result;
-
-fn main() -> Result<()> {
-    // Kill any existing session
-    let _ = Command::new("tmux")
-        .args(["kill-session", "-t", "djls-debug"])
-        .output();
-
-    let _ = Command::new("pkill").args(["-f", "lsp-devtools"]).output();
-
-    // Start tmux in control mode
-    let mut tmux = Command::new("tmux")
-        .args(["-C", "-f", "/dev/null"])
-        .stdin(Stdio::piped())
-        .stdout(Stdio::piped())
-        .spawn()
-        .context("Failed to start tmux control mode")?;
-
-    let stdin = tmux.stdin.as_mut().context("Failed to get tmux stdin")?;
-
-    // Create session with editor, setting DJANGO_SETTINGS_MODULE and PYTHONPATH
-    writeln!(
-        stdin,
-        "new-session -d -s djls-debug 'DJANGO_SETTINGS_MODULE=djls_test.settings PYTHONPATH=tests/project:$PYTHONPATH nvim tests/project/djls_app/templates/djls_app/base.html'"
-    )?;
-
-    // Add devtools pane (20% width on the right)
-    writeln!(
-        stdin,
-        "split-window -t djls-debug -h -p 20 'just dev devtools record'"
-    )?;
-
-    // Split the right pane horizontally for server logs (50/50 split)
-    // Updated to handle dated log files properly
-    writeln!(
-        stdin,
-        r#"split-window -t djls-debug:0.1 -v -p 50 'bash -c "log=\$(ls -t /tmp/djls.log.* 2>/dev/null | head -1); if [ -z \"\$log\" ]; then echo \"Waiting for server logs...\"; while [ -z \"\$log\" ]; do sleep 1; log=\$(ls -t /tmp/djls.log.* 2>/dev/null | head -1); done; fi; echo \"Tailing \$log\"; tail -F \"\$log\""'"#
-    )?;
-
-    // Set pane titles
-    writeln!(stdin, "select-pane -t djls-debug:0.0 -T 'Editor'")?;
-    writeln!(stdin, "select-pane -t djls-debug:0.1 -T 'LSP Messages'")?;
-    writeln!(stdin, "select-pane -t djls-debug:0.2 -T 'Server Logs'")?;
-
-    // Enable pane borders with titles at the top
-    writeln!(stdin, "set -t djls-debug pane-border-status top")?;
-
-    // Enable mouse support for scrolling and pane interaction
-    writeln!(stdin, "set -t djls-debug mouse on")?;
-
-    // Add custom keybind to kill session (capital K)
-    writeln!(stdin, "bind-key K kill-session")?;
-
-    // Configure status bar with keybind hints
-    writeln!(stdin, "set -t djls-debug status on")?;
-    writeln!(stdin, "set -t djls-debug status-position bottom")?;
-    writeln!(
-        stdin,
-        "set -t djls-debug status-style 'bg=colour235,fg=colour250'"
-    )?;
-
-    // Left side: session name
-    writeln!(stdin, "set -t djls-debug status-left '[#S] '")?;
-    writeln!(stdin, "set -t djls-debug status-left-length 20")?;
-
-    // Right side: keybind hints - updated to include mouse info
-    writeln!(stdin, "set -t djls-debug status-right ' Mouse: scroll/click | C-b d: detach | C-b K: kill | C-b x: kill pane | C-b z: zoom | C-b ?: help '")?;
-    writeln!(stdin, "set -t djls-debug status-right-length 120")?;
-
-    // Center: window name
-    writeln!(stdin, "set -t djls-debug status-justify centre")?;
-
-    // Focus editor pane
-    writeln!(stdin, "select-pane -t djls-debug:0.0")?;
-
-    // Exit control mode
-    writeln!(stdin, "detach")?;
-    stdin.flush()?;
-
-    // Close stdin to signal we're done sending commands
-    drop(tmux.stdin.take());
-
-    // Wait for control mode to finish
-    tmux.wait()?;
-
-    // Attach to session
-    Command::new("tmux")
-        .args(["attach-session", "-t", "djls-debug"])
-        .status()
-        .context("Failed to attach to session")?;
-
-    // Cleanup on exit
-    let _ = Command::new("pkill").args(["-f", "lsp-devtools"]).output();
-
-    Ok(())
-}

crates/djls-dev/src/build.rs

@@ -1,55 +0,0 @@
-/// Sets up the necessary Cargo directives for linking against the Python library.
-///
-/// This function should be called from the `build.rs` script of any crate
-/// within the workspace whose compiled artifact (e.g., test executable, binary)
-/// needs to link against the Python C API at compile time and find the
-/// corresponding shared library at runtime. This is typically required for
-/// crates using `pyo3` features like `Python::with_gil` or defining `#[pyfunction]`s
-/// directly in their test or binary code.
-///
-/// It uses `pyo3-build-config` to detect the active Python installation and
-/// prints the required `cargo:rustc-link-search` and `cargo:rustc-link-lib`
-/// directives to Cargo, enabling the linker to find and link against the
-/// appropriate Python library (e.g., libpythonX.Y.so).
-///
-/// It also adds an RPATH linker argument on Unix-like systems (`-Wl,-rpath,...`)
-/// to help the resulting executable find the Python shared library at runtime
-/// without needing manual `LD_LIBRARY_PATH` adjustments in typical scenarios.
-pub fn setup_python_linking() {
-    // Instruct Cargo to rerun the calling build script if the Python config changes.
-    // Using PYO3_CONFIG_FILE is a reliable way to detect changes in the active Python env.
-    println!("cargo:rerun-if-changed=build.rs");
-
-    // Set up #[cfg] flags first (useful for conditional compilation)
-    pyo3_build_config::use_pyo3_cfgs();
-
-    // Get the Python interpreter configuration directly
-    let config = pyo3_build_config::get();
-
-    let is_extension_module = std::env::var("CARGO_FEATURE_EXTENSION_MODULE").is_ok();
-
-    // Only link libpython explicitly if we are NOT building an extension module.
-    if !is_extension_module {
-        if let Some(lib_name) = &config.lib_name {
-            println!("cargo:rustc-link-lib=dylib={lib_name}");
-        } else {
-            // Warn only if linking is actually needed but we can't find the lib name
-            println!("cargo:warning=Python library name not found in config (needed for non-extension module
-builds).");
-        }
-    }
-
-    // Add the library search path and RPATH if available.
-    // These are needed for test executables and potential future standalone binaries,
-    // and generally harmless for extension modules.
-    if let Some(lib_dir) = &config.lib_dir {
-        println!("cargo:rustc-link-search=native={lib_dir}");
-        #[cfg(not(windows))]
-        println!("cargo:rustc-link-arg=-Wl,-rpath,{lib_dir}");
-    } else {
-        // Warn only if linking is actually needed but we can't find the lib dir
-        if !is_extension_module {
-            println!("cargo:warning=Python library directory not found in config.");
-        }
-    }
-}

crates/djls-dev/src/lib.rs

@@ -1,3 +0,0 @@
-mod build;
-
-pub use build::setup_python_linking;

crates/djls-project/Cargo.toml

@@ -3,22 +3,20 @@ name = "djls-project"
 version = "0.0.0"
 edition = "2021"
 
-[features]
-extension-module = []
-default = []
-
 [dependencies]
 djls-workspace = { workspace = true }
 
-pyo3 = { workspace = true }
+anyhow = { workspace = true }
 salsa = { workspace = true }
+serde = { workspace = true }
+serde_json = { workspace = true }
+tempfile = { workspace = true }
 which = { workspace = true}
 
 [build-dependencies]
-djls-dev = { workspace = true }
+which = { workspace = true }
 
 [dev-dependencies]
-tempfile = { workspace = true }
 
 [lints]
 workspace = true

crates/djls-project/build.rs

@@ -1,3 +1,56 @@
-fn main() {
+use std::env;
-    djls_dev::setup_python_linking();
+use std::path::PathBuf;
+use std::process::Command;
+
+fn get_python_executable_path() -> PathBuf {
+    let python = which::which("python3")
+        .or_else(|_| which::which("python"))
+        .expect("Python not found. Please install Python to build this project.");
+    println!(
+        "cargo:warning=Building Python inspector with: {}",
+        python.display()
+    );
+    python
+}
+
+fn main() {
+    println!("cargo:rerun-if-changed=../../python/src/djls_inspector");
+    println!("cargo:rerun-if-changed=../../python/build.py");
+
+    let workspace_dir = env::var("CARGO_WORKSPACE_DIR").expect("CARGO_WORKSPACE_DIR not set");
+    let workspace_path = PathBuf::from(workspace_dir);
+    let python_dir = workspace_path.join("python");
+    let dist_dir = python_dir.join("dist");
+    let pyz_path = dist_dir.join("djls_inspector.pyz");
+
+    std::fs::create_dir_all(&dist_dir).expect("Failed to create python/dist directory");
+
+    let python = get_python_executable_path();
+
+    let output = Command::new(&python)
+        .arg("build.py")
+        .current_dir(&python_dir)
+        .output()
+        .expect("Failed to run Python build script");
+
+    if !output.status.success() {
+        let stderr = String::from_utf8_lossy(&output.stderr);
+        let stdout = String::from_utf8_lossy(&output.stdout);
+        panic!("Failed to build Python inspector:\nSTDOUT:\n{stdout}\nSTDERR:\n{stderr}");
+    }
+
+    assert!(
+        pyz_path.exists(),
+        "Python inspector zipapp was not created at expected location: {}",
+        pyz_path.display()
+    );
+
+    let metadata =
+        std::fs::metadata(&pyz_path).expect("Failed to get metadata for inspector zipapp");
+
+    println!(
+        "cargo:warning=Successfully built Python inspector: {} ({} bytes)",
+        pyz_path.display(),
+        metadata.len()
+    );
 }

crates/djls-project/src/inspector.rs

@@ -0,0 +1,20 @@
+pub mod ipc;
+pub mod pool;
+pub mod queries;
+
+pub use queries::Query;
+use serde::Deserialize;
+use serde::Serialize;
+
+#[derive(Serialize)]
+pub struct DjlsRequest {
+    #[serde(flatten)]
+    pub query: Query,
+}
+
+#[derive(Debug, Deserialize)]
+pub struct DjlsResponse {
+    pub ok: bool,
+    pub data: Option<serde_json::Value>,
+    pub error: Option<String>,
+}

crates/djls-project/src/inspector/ipc.rs

@@ -0,0 +1,128 @@
+use std::io::BufRead;
+use std::io::BufReader;
+use std::io::Write;
+use std::path::Path;
+use std::process::Child;
+use std::process::Command;
+use std::process::Stdio;
+
+use anyhow::Context;
+use anyhow::Result;
+use serde_json;
+use tempfile::NamedTempFile;
+
+use super::DjlsRequest;
+use super::DjlsResponse;
+use crate::python::PythonEnvironment;
+
+const INSPECTOR_PYZ: &[u8] = include_bytes!(concat!(
+    env!("CARGO_WORKSPACE_DIR"),
+    "/python/dist/djls_inspector.pyz"
+));
+
+pub struct InspectorProcess {
+    child: Child,
+    stdin: std::process::ChildStdin,
+    stdout: BufReader<std::process::ChildStdout>,
+    _zipapp_file: NamedTempFile,
+}
+
+impl InspectorProcess {
+    pub fn new(python_env: &PythonEnvironment, project_path: &Path) -> Result<Self> {
+        let mut zipapp_file = tempfile::Builder::new()
+            .prefix("djls_inspector_")
+            .suffix(".pyz")
+            .tempfile()
+            .context("Failed to create temp file for inspector")?;
+
+        zipapp_file
+            .write_all(INSPECTOR_PYZ)
+            .context("Failed to write inspector zipapp to temp file")?;
+        zipapp_file
+            .flush()
+            .context("Failed to flush inspector zipapp")?;
+
+        let zipapp_path = zipapp_file.path();
+
+        let mut cmd = Command::new(&python_env.python_path);
+        cmd.arg(zipapp_path)
+            .stdin(Stdio::piped())
+            .stdout(Stdio::piped())
+            .stderr(Stdio::inherit())
+            .current_dir(project_path);
+
+        if let Ok(pythonpath) = std::env::var("PYTHONPATH") {
+            let mut paths = vec![project_path.to_string_lossy().to_string()];
+            paths.push(pythonpath);
+            cmd.env("PYTHONPATH", paths.join(":"));
+        } else {
+            cmd.env("PYTHONPATH", project_path);
+        }
+
+        if let Ok(settings) = std::env::var("DJANGO_SETTINGS_MODULE") {
+            cmd.env("DJANGO_SETTINGS_MODULE", settings);
+        } else {
+            // Try to detect settings module
+            if project_path.join("manage.py").exists() {
+                // Look for common settings modules
+                for candidate in &["settings", "config.settings", "project.settings"] {
+                    let parts: Vec<&str> = candidate.split('.').collect();
+                    let mut path = project_path.to_path_buf();
+                    for part in &parts[..parts.len() - 1] {
+                        path = path.join(part);
+                    }
+                    if let Some(last) = parts.last() {
+                        path = path.join(format!("{last}.py"));
+                    }
+
+                    if path.exists() {
+                        cmd.env("DJANGO_SETTINGS_MODULE", candidate);
+                        break;
+                    }
+                }
+            }
+        }
+
+        let mut child = cmd.spawn().context("Failed to spawn inspector process")?;
+
+        let stdin = child.stdin.take().context("Failed to get stdin handle")?;
+        let stdout = BufReader::new(child.stdout.take().context("Failed to get stdout handle")?);
+
+        Ok(Self {
+            child,
+            stdin,
+            stdout,
+            _zipapp_file: zipapp_file,
+        })
+    }
+
+    /// Send a request and receive a response
+    pub fn query(&mut self, request: &DjlsRequest) -> Result<DjlsResponse> {
+        let request_json = serde_json::to_string(request)?;
+
+        writeln!(self.stdin, "{request_json}")?;
+        self.stdin.flush()?;
+
+        let mut response_line = String::new();
+        self.stdout
+            .read_line(&mut response_line)
+            .context("Failed to read response from inspector")?;
+
+        let response: DjlsResponse =
+            serde_json::from_str(&response_line).context("Failed to parse inspector response")?;
+
+        Ok(response)
+    }
+
+    pub fn is_running(&mut self) -> bool {
+        matches!(self.child.try_wait(), Ok(None))
+    }
+}
+
+impl Drop for InspectorProcess {
+    fn drop(&mut self) {
+        // Try to terminate the child process gracefully
+        let _ = self.child.kill();
+        let _ = self.child.wait();
+    }
+}

crates/djls-project/src/inspector/pool.rs

@@ -0,0 +1,206 @@
+use std::path::Path;
+use std::sync::Arc;
+use std::sync::Mutex;
+use std::time::Duration;
+use std::time::Instant;
+
+use anyhow::Result;
+
+use super::ipc::InspectorProcess;
+use super::DjlsRequest;
+use super::DjlsResponse;
+use crate::python::PythonEnvironment;
+
+/// Global singleton pool for convenience
+static GLOBAL_POOL: std::sync::OnceLock<InspectorPool> = std::sync::OnceLock::new();
+
+pub fn global_pool() -> &'static InspectorPool {
+    GLOBAL_POOL.get_or_init(InspectorPool::new)
+}
+const DEFAULT_IDLE_TIMEOUT: Duration = Duration::from_secs(60);
+
+/// Manages a pool of inspector processes with automatic cleanup
+#[derive(Clone)]
+pub struct InspectorPool {
+    inner: Arc<Mutex<InspectorPoolInner>>,
+}
+
+impl std::fmt::Debug for InspectorPool {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("InspectorPool")
+            .field("has_active_process", &self.has_active_process())
+            .finish()
+    }
+}
+
+struct InspectorPoolInner {
+    process: Option<InspectorProcessHandle>,
+    idle_timeout: Duration,
+}
+
+struct InspectorProcessHandle {
+    process: InspectorProcess,
+    last_used: Instant,
+    python_env: PythonEnvironment,
+    project_path: std::path::PathBuf,
+}
+
+impl InspectorPool {
+    #[must_use]
+    pub fn new() -> Self {
+        Self::with_timeout(DEFAULT_IDLE_TIMEOUT)
+    }
+
+    #[must_use]
+    pub fn with_timeout(idle_timeout: Duration) -> Self {
+        Self {
+            inner: Arc::new(Mutex::new(InspectorPoolInner {
+                process: None,
+                idle_timeout,
+            })),
+        }
+    }
+
+    /// Execute a query, reusing existing process if available and not idle
+    ///
+    /// # Panics
+    ///
+    /// Panics if the inspector pool mutex is poisoned (another thread panicked while holding the lock)
+    pub fn query(
+        &self,
+        python_env: &PythonEnvironment,
+        project_path: &Path,
+        request: &DjlsRequest,
+    ) -> Result<DjlsResponse> {
+        let mut inner = self.inner.lock().expect("Inspector pool mutex poisoned");
+        let idle_timeout = inner.idle_timeout;
+
+        // Check if we need to drop the existing process
+        let need_new_process = if let Some(handle) = &mut inner.process {
+            let idle_too_long = handle.last_used.elapsed() > idle_timeout;
+            let not_running = !handle.process.is_running();
+            let different_env =
+                handle.python_env != *python_env || handle.project_path != project_path;
+
+            idle_too_long || not_running || different_env
+        } else {
+            true
+        };
+
+        if need_new_process {
+            inner.process = None;
+        }
+
+        // Get or create process
+        if inner.process.is_none() {
+            let process = InspectorProcess::new(python_env, project_path)?;
+            inner.process = Some(InspectorProcessHandle {
+                process,
+                last_used: Instant::now(),
+                python_env: python_env.clone(),
+                project_path: project_path.to_path_buf(),
+            });
+        }
+
+        // Now we can safely get a mutable reference
+        let handle = inner
+            .process
+            .as_mut()
+            .expect("Process should exist after creation");
+
+        // Execute query
+        let response = handle.process.query(request)?;
+        handle.last_used = Instant::now();
+
+        Ok(response)
+    }
+
+    /// Manually close the inspector process
+    ///
+    /// # Panics
+    ///
+    /// Panics if the inspector pool mutex is poisoned
+    pub fn close(&self) {
+        let mut inner = self.inner.lock().expect("Inspector pool mutex poisoned");
+        inner.process = None;
+    }
+
+    /// Check if there's an active process
+    ///
+    /// # Panics
+    ///
+    /// Panics if the inspector pool mutex is poisoned
+    #[must_use]
+    pub fn has_active_process(&self) -> bool {
+        let mut inner = self.inner.lock().expect("Inspector pool mutex poisoned");
+        if let Some(handle) = &mut inner.process {
+            handle.process.is_running() && handle.last_used.elapsed() <= inner.idle_timeout
+        } else {
+            false
+        }
+    }
+
+    /// Get the configured idle timeout
+    ///
+    /// # Panics
+    ///
+    /// Panics if the inspector pool mutex is poisoned
+    #[must_use]
+    pub fn idle_timeout(&self) -> Duration {
+        let inner = self.inner.lock().expect("Inspector pool mutex poisoned");
+        inner.idle_timeout
+    }
+
+    /// Start a background cleanup task that periodically checks for idle processes
+    ///
+    /// # Panics
+    ///
+    /// The spawned thread will panic if the inspector pool mutex is poisoned
+    pub fn start_cleanup_task(self: Arc<Self>) {
+        std::thread::spawn(move || {
+            loop {
+                std::thread::sleep(Duration::from_secs(30)); // Check every 30 seconds
+
+                let mut inner = self.inner.lock().expect("Inspector pool mutex poisoned");
+                if let Some(handle) = &inner.process {
+                    if handle.last_used.elapsed() > inner.idle_timeout {
+                        // Process is idle, drop it
+                        inner.process = None;
+                    }
+                }
+            }
+        });
+    }
+}
+
+impl Default for InspectorPool {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_pool_creation() {
+        let pool = InspectorPool::new();
+        assert_eq!(pool.idle_timeout(), DEFAULT_IDLE_TIMEOUT);
+        assert!(!pool.has_active_process());
+    }
+
+    #[test]
+    fn test_pool_with_custom_timeout() {
+        let timeout = Duration::from_secs(120);
+        let pool = InspectorPool::with_timeout(timeout);
+        assert_eq!(pool.idle_timeout(), timeout);
+    }
+
+    #[test]
+    fn test_pool_close() {
+        let pool = InspectorPool::new();
+        pool.close();
+        assert!(!pool.has_active_process());
+    }
+}

crates/djls-project/src/inspector/queries.rs

@@ -0,0 +1,50 @@
+use std::path::PathBuf;
+
+use serde::Deserialize;
+use serde::Serialize;
+
+#[derive(Serialize, Deserialize)]
+#[serde(tag = "query", content = "args")]
+#[serde(rename_all = "snake_case")]
+pub enum Query {
+    PythonEnv,
+    Templatetags,
+    DjangoInit,
+}
+
+#[derive(Serialize, Deserialize)]
+#[serde(rename_all = "lowercase")]
+pub enum VersionReleaseLevel {
+    Alpha,
+    Beta,
+    Candidate,
+    Final,
+}
+
+#[derive(Serialize, Deserialize)]
+pub struct PythonEnvironmentQueryData {
+    pub sys_base_prefix: PathBuf,
+    pub sys_executable: PathBuf,
+    pub sys_path: Vec<PathBuf>,
+    pub sys_platform: String,
+    pub sys_prefix: PathBuf,
+    pub sys_version_info: (u32, u32, u32, VersionReleaseLevel, u32),
+}
+
+#[derive(Serialize, Deserialize)]
+pub struct TemplateTagQueryData {
+    pub templatetags: Vec<TemplateTag>,
+}
+
+#[derive(Serialize, Deserialize)]
+pub struct TemplateTag {
+    pub name: String,
+    pub module: String,
+    pub doc: Option<String>,
+}
+
+#[derive(Serialize, Deserialize)]
+pub struct DjangoInitQueryData {
+    pub success: bool,
+    pub message: Option<String>,
+}

crates/djls-project/src/lib.rs

@@ -1,17 +1,23 @@
 mod db;
+pub mod inspector;
 mod meta;
-mod python;
+pub mod python;
 mod system;
 mod templatetags;
 
 use std::fmt;
 use std::path::Path;
 use std::path::PathBuf;
+use std::sync::Arc;
 
+use anyhow::Context;
+use anyhow::Result;
 pub use db::find_python_environment;
 pub use db::Db;
+use inspector::pool::InspectorPool;
+use inspector::DjlsRequest;
+use inspector::Query;
 pub use meta::ProjectMetadata;
-use pyo3::prelude::*;
 pub use python::PythonEnvironment;
 pub use templatetags::TemplateTags;
 
@@ -20,6 +26,7 @@ pub struct DjangoProject {
     path: PathBuf,
     env: Option<PythonEnvironment>,
     template_tags: Option<TemplateTags>,
+    inspector_pool: Arc<InspectorPool>,
 }
 
 impl DjangoProject {
@@ -29,45 +36,39 @@ impl DjangoProject {
             path,
             env: None,
             template_tags: None,
+            inspector_pool: Arc::new(InspectorPool::new()),
         }
     }
 
-    pub fn initialize(&mut self, db: &dyn Db) -> PyResult<()> {
+    pub fn initialize(&mut self, db: &dyn Db) -> Result<()> {
         // Use the database to find the Python environment
         self.env = find_python_environment(db);
-        if self.env.is_none() {
+        let env = self
-            return Err(PyErr::new::<pyo3::exceptions::PyRuntimeError, _>(
+            .env
-                "Could not find Python environment",
+            .as_ref()
-            ));
+            .context("Could not find Python environment")?;
+
+        // Initialize Django
+        let request = DjlsRequest {
+            query: Query::DjangoInit,
+        };
+        let response = self.inspector_pool.query(env, &self.path, &request)?;
+
+        if !response.ok {
+            anyhow::bail!("Failed to initialize Django: {:?}", response.error);
         }
 
-        Python::with_gil(|py| {
+        // Get template tags
-            let sys = py.import("sys")?;
+        let request = DjlsRequest {
-            let py_path = sys.getattr("path")?;
+            query: Query::Templatetags,
+        };
+        let response = self.inspector_pool.query(env, &self.path, &request)?;
 
-            if let Some(path_str) = self.path.to_str() {
+        if let Some(data) = response.data {
-                py_path.call_method1("insert", (0, path_str))?;
+            self.template_tags = Some(TemplateTags::from_json(&data)?);
-            }
+        }
 
-            let env = self.env.as_ref().ok_or_else(|| {
+        Ok(())
-                PyErr::new::<pyo3::exceptions::PyRuntimeError, _>(
-                    "Internal error: Python environment missing after initialization",
-                )
-            })?;
-            env.activate(py)?;
-
-            match py.import("django") {
-                Ok(django) => {
-                    django.call_method0("setup")?;
-                    self.template_tags = Some(TemplateTags::from_python(py)?);
-                    Ok(())
-                }
-                Err(e) => {
-                    eprintln!("Failed to import Django: {e}");
-                    Err(e)
-                }
-            }
-        })
     }
 
     #[must_use]

crates/djls-project/src/python.rs

@@ -2,15 +2,13 @@ use std::fmt;
 use std::path::Path;
 use std::path::PathBuf;
 
-use pyo3::prelude::*;
-
 use crate::system;
 
 #[derive(Clone, Debug, PartialEq)]
 pub struct PythonEnvironment {
-    python_path: PathBuf,
+    pub python_path: PathBuf,
-    sys_path: Vec<PathBuf>,
+    pub sys_path: Vec<PathBuf>,
-    sys_prefix: PathBuf,
+    pub sys_prefix: PathBuf,
 }
 
 impl PythonEnvironment {
@@ -74,19 +72,6 @@ impl PythonEnvironment {
         })
     }
 
-    pub fn activate(&self, py: Python) -> PyResult<()> {
-        let sys = py.import("sys")?;
-        let py_path = sys.getattr("path")?;
-
-        for path in &self.sys_path {
-            if let Some(path_str) = path.to_str() {
-                py_path.call_method1("append", (path_str,))?;
-            }
-        }
-
-        Ok(())
-    }
-
     fn from_system_python() -> Option<Self> {
         let Ok(python_path) = system::find_executable("python") else {
             return None;
@@ -181,20 +166,6 @@ mod tests {
         prefix
     }
 
-    fn get_sys_path(py: Python) -> PyResult<Vec<String>> {
-        let sys = py.import("sys")?;
-        let py_path = sys.getattr("path")?;
-        py_path.extract::<Vec<String>>()
-    }
-
-    fn create_test_env(sys_paths: Vec<PathBuf>) -> PythonEnvironment {
-        PythonEnvironment {
-            python_path: PathBuf::from("dummy/bin/python"),
-            sys_prefix: PathBuf::from("dummy"),
-            sys_path: sys_paths,
-        }
-    }
-
     mod env_discovery {
         use which::Error as WhichError;
 
@@ -493,213 +464,6 @@ mod tests {
         }
     }
 
-    mod env_activation {
-        use super::*;
-
-        #[test]
-        #[ignore = "Requires Python runtime - run with --ignored flag"]
-        fn test_activate_appends_paths() -> PyResult<()> {
-            let temp_dir = tempdir().unwrap();
-            let path1 = temp_dir.path().join("scripts");
-            let path2 = temp_dir.path().join("libs");
-            fs::create_dir_all(&path1).unwrap();
-            fs::create_dir_all(&path2).unwrap();
-
-            let test_env = create_test_env(vec![path1.clone(), path2.clone()]);
-
-            pyo3::prepare_freethreaded_python();
-
-            Python::with_gil(|py| {
-                let initial_sys_path = get_sys_path(py)?;
-                let initial_len = initial_sys_path.len();
-
-                test_env.activate(py)?;
-
-                let final_sys_path = get_sys_path(py)?;
-                assert_eq!(
-                    final_sys_path.len(),
-                    initial_len + 2,
-                    "Should have added 2 paths"
-                );
-                assert_eq!(
-                    final_sys_path.get(initial_len).unwrap(),
-                    path1.to_str().expect("Path 1 should be valid UTF-8")
-                );
-                assert_eq!(
-                    final_sys_path.get(initial_len + 1).unwrap(),
-                    path2.to_str().expect("Path 2 should be valid UTF-8")
-                );
-
-                Ok(())
-            })
-        }
-
-        #[test]
-        #[ignore = "Requires Python runtime - run with --ignored flag"]
-        fn test_activate_empty_sys_path() -> PyResult<()> {
-            let test_env = create_test_env(vec![]);
-
-            pyo3::prepare_freethreaded_python();
-
-            Python::with_gil(|py| {
-                let initial_sys_path = get_sys_path(py)?;
-
-                test_env.activate(py)?;
-
-                let final_sys_path = get_sys_path(py)?;
-                assert_eq!(
-                    final_sys_path, initial_sys_path,
-                    "sys.path should remain unchanged for empty env.sys_path"
-                );
-
-                Ok(())
-            })
-        }
-
-        #[test]
-        #[ignore = "Requires Python runtime - run with --ignored flag"]
-        fn test_activate_with_non_existent_paths() -> PyResult<()> {
-            let temp_dir = tempdir().unwrap();
-            let path1 = temp_dir.path().join("non_existent_dir");
-            let path2 = temp_dir.path().join("another_missing/path");
-
-            let test_env = create_test_env(vec![path1.clone(), path2.clone()]);
-
-            pyo3::prepare_freethreaded_python();
-
-            Python::with_gil(|py| {
-                let initial_sys_path = get_sys_path(py)?;
-                let initial_len = initial_sys_path.len();
-
-                test_env.activate(py)?;
-
-                let final_sys_path = get_sys_path(py)?;
-                assert_eq!(
-                    final_sys_path.len(),
-                    initial_len + 2,
-                    "Should still add 2 paths even if they don't exist"
-                );
-                assert_eq!(
-                    final_sys_path.get(initial_len).unwrap(),
-                    path1.to_str().unwrap()
-                );
-                assert_eq!(
-                    final_sys_path.get(initial_len + 1).unwrap(),
-                    path2.to_str().unwrap()
-                );
-
-                Ok(())
-            })
-        }
-
-        #[test]
-        #[cfg(unix)]
-        #[ignore = "Requires Python runtime - run with --ignored flag"]
-        fn test_activate_skips_non_utf8_paths_unix() -> PyResult<()> {
-            use std::ffi::OsStr;
-            use std::os::unix::ffi::OsStrExt;
-
-            let temp_dir = tempdir().unwrap();
-            let valid_path = temp_dir.path().join("valid_dir");
-            fs::create_dir(&valid_path).unwrap();
-
-            let invalid_bytes = b"invalid_\xff_utf8";
-            let os_str = OsStr::from_bytes(invalid_bytes);
-            let non_utf8_path = PathBuf::from(os_str);
-            assert!(
-                non_utf8_path.to_str().is_none(),
-                "Path should not be convertible to UTF-8 str"
-            );
-
-            let test_env = create_test_env(vec![valid_path.clone(), non_utf8_path.clone()]);
-
-            pyo3::prepare_freethreaded_python();
-
-            Python::with_gil(|py| {
-                let initial_sys_path = get_sys_path(py)?;
-                let initial_len = initial_sys_path.len();
-
-                test_env.activate(py)?;
-
-                let final_sys_path = get_sys_path(py)?;
-                assert_eq!(
-                    final_sys_path.len(),
-                    initial_len + 1,
-                    "Should only add valid UTF-8 paths"
-                );
-                assert_eq!(
-                    final_sys_path.get(initial_len).unwrap(),
-                    valid_path.to_str().unwrap()
-                );
-
-                let invalid_path_lossy = non_utf8_path.to_string_lossy();
-                assert!(
-                    !final_sys_path
-                        .iter()
-                        .any(|p| p.contains(&*invalid_path_lossy)),
-                    "Non-UTF8 path should not be present in sys.path"
-                );
-
-                Ok(())
-            })
-        }
-
-        #[test]
-        #[cfg(windows)]
-        #[ignore = "Requires Python runtime - run with --ignored flag"]
-        fn test_activate_skips_non_utf8_paths_windows() -> PyResult<()> {
-            use std::ffi::OsString;
-            use std::os::windows::ffi::OsStringExt;
-
-            let temp_dir = tempdir().unwrap();
-            let valid_path = temp_dir.path().join("valid_dir");
-
-            let invalid_wide: Vec<u16> = vec![
-                'i' as u16, 'n' as u16, 'v' as u16, 'a' as u16, 'l' as u16, 'i' as u16, 'd' as u16,
-                '_' as u16, 0xD800, '_' as u16, 'w' as u16, 'i' as u16, 'd' as u16, 'e' as u16,
-            ];
-            let os_string = OsString::from_wide(&invalid_wide);
-            let non_utf8_path = PathBuf::from(os_string);
-
-            assert!(
-                non_utf8_path.to_str().is_none(),
-                "Path with lone surrogate should not be convertible to UTF-8 str"
-            );
-
-            let test_env = create_test_env(vec![valid_path.clone(), non_utf8_path.clone()]);
-
-            pyo3::prepare_freethreaded_python();
-
-            Python::with_gil(|py| {
-                let initial_sys_path = get_sys_path(py)?;
-                let initial_len = initial_sys_path.len();
-
-                test_env.activate(py)?;
-
-                let final_sys_path = get_sys_path(py)?;
-                assert_eq!(
-                    final_sys_path.len(),
-                    initial_len + 1,
-                    "Should only add paths convertible to valid UTF-8"
-                );
-                assert_eq!(
-                    final_sys_path.get(initial_len).unwrap(),
-                    valid_path.to_str().unwrap()
-                );
-
-                let invalid_path_lossy = non_utf8_path.to_string_lossy();
-                assert!(
-                    !final_sys_path
-                        .iter()
-                        .any(|p| p.contains(&*invalid_path_lossy)),
-                    "Non-UTF8 path (from invalid wide chars) should not be present in sys.path"
-                );
-
-                Ok(())
-            })
-        }
-    }
-
     mod salsa_integration {
         use std::sync::Arc;
 

crates/djls-project/src/templatetags.rs

@@ -1,8 +1,8 @@
 use std::ops::Deref;
 
-use pyo3::prelude::*;
+use anyhow::Context;
-use pyo3::types::PyDict;
+use anyhow::Result;
-use pyo3::types::PyList;
+use serde_json::Value;
 
 #[derive(Debug, Default, Clone)]
 pub struct TemplateTags(Vec<TemplateTag>);
@@ -16,57 +16,46 @@ impl Deref for TemplateTags {
 }
 
 impl TemplateTags {
-    fn new() -> Self {
+    pub fn from_json(data: &Value) -> Result<TemplateTags> {
-        Self(Vec::new())
+        let mut tags = Vec::new();
-    }
 
-    fn process_library(
+        // Parse the JSON response from the inspector
-        module_name: &str,
+        let templatetags = data
-        library: &Bound<'_, PyAny>,
+            .get("templatetags")
-        tags: &mut Vec<TemplateTag>,
+            .context("Missing 'templatetags' field in response")?
-    ) -> PyResult<()> {
+            .as_array()
-        let tags_dict = library.getattr("tags")?;
+            .context("'templatetags' field is not an array")?;
-        let dict = tags_dict.downcast::<PyDict>()?;
 
-        for (key, value) in dict.iter() {
+        for tag_data in templatetags {
-            let tag_name = key.extract::<String>()?;
+            let name = tag_data
-            let doc = value.getattr("__doc__")?.extract().ok();
+                .get("name")
+                .and_then(|v| v.as_str())
+                .context("Missing or invalid 'name' field")?
+                .to_string();
 
-            let library_name = if module_name.is_empty() {
+            let module = tag_data
-                "builtins".to_string()
+                .get("module")
-            } else {
+                .and_then(|v| v.as_str())
-                module_name.split('.').next_back().unwrap_or("").to_string()
+                .context("Missing or invalid 'module' field")?;
-            };
 
-            tags.push(TemplateTag::new(tag_name, library_name, doc));
+            // Extract library name from module (e.g., "django.templatetags.static" -> "static")
-        }
+            let library = module
-        Ok(())
+                .split('.')
-    }
+                .filter(|part| part.contains("templatetags"))
+                .nth(1)
+                .or_else(|| module.split('.').next_back())
+                .unwrap_or("builtins")
+                .to_string();
 
-    pub fn from_python(py: Python) -> PyResult<TemplateTags> {
+            let doc = tag_data
-        let mut template_tags = TemplateTags::new();
+                .get("doc")
+                .and_then(|v| v.as_str())
+                .map(String::from);
 
-        let engine = py
+            tags.push(TemplateTag::new(name, library, doc));
-            .import("django.template.engine")?
-            .getattr("Engine")?
-            .call_method0("get_default")?;
-
-        // Built-in template tags
-        let builtins_attr = engine.getattr("template_builtins")?;
-        let builtins = builtins_attr.downcast::<PyList>()?;
-        for builtin in builtins {
-            Self::process_library("", &builtin, &mut template_tags.0)?;
         }
 
-        // Custom template libraries
+        Ok(TemplateTags(tags))
-        let libraries_attr = engine.getattr("template_libraries")?;
-        let libraries = libraries_attr.downcast::<PyDict>()?;
-        for (module_name, library) in libraries.iter() {
-            let module_name = module_name.extract::<String>()?;
-            Self::process_library(&module_name, &library, &mut template_tags.0)?;
-        }
-
-        Ok(template_tags)
     }
 }
 

crates/djls-server/Cargo.toml

@@ -3,10 +3,6 @@ name = "djls-server"
 version = "0.0.0"
 edition = "2021"
 
-[features]
-extension-module = []
-default = []
-
 [dependencies]
 djls-conf = { workspace = true }
 djls-project = { workspace = true }
@@ -17,7 +13,6 @@ anyhow = { workspace = true }
 camino = { workspace = true }
 dashmap = { workspace = true }
 percent-encoding = { workspace = true }
-pyo3 = { workspace = true }
 salsa = { workspace = true }
 serde = { workspace = true }
 serde_json = { workspace = true }
@@ -28,9 +23,6 @@ tracing-appender = { workspace = true }
 tracing-subscriber = { workspace = true }
 url = { workspace = true }
 
-[build-dependencies]
-djls-dev = { workspace = true }
-
 [dev-dependencies]
 tempfile = { workspace = true }
 

crates/djls-server/build.rs

@@ -1,3 +0,0 @@
-fn main() {
-    djls_dev::setup_python_linking();
-}

crates/djls-server/src/session.rs

@@ -6,6 +6,7 @@
 use std::path::PathBuf;
 use std::sync::Arc;
 
+use anyhow::Result;
 use dashmap::DashMap;
 use djls_conf::Settings;
 use djls_project::DjangoProject;
@@ -15,7 +16,6 @@ use djls_workspace::paths;
 use djls_workspace::PositionEncoding;
 use djls_workspace::TextDocument;
 use djls_workspace::Workspace;
-use pyo3::PyResult;
 use tower_lsp_server::lsp_types;
 use url::Url;
 
@@ -156,7 +156,7 @@ impl Session {
     }
 
     /// Initialize the project with the database.
-    pub fn initialize_project(&mut self) -> PyResult<()> {
+    pub fn initialize_project(&mut self) -> Result<()> {
         if let Some(project) = self.project.as_mut() {
             project.initialize(&self.db)
         } else {

crates/djls/Cargo.toml

@@ -3,29 +3,13 @@ name = "djls"
 version = "5.2.0"
 edition = "2021"
 
-[lib]
-name = "djls"
-crate-type = ["cdylib"]
-
-[features]
-extension-module = [
-  "djls-server/extension-module",
-  "djls-project/extension-module",
-  "pyo3/extension-module"
-]
-default = []
-
 [dependencies]
 djls-project = { workspace = true }
 djls-server = { workspace = true }
 
 anyhow = { workspace = true }
 clap = { workspace = true }
-pyo3 = { workspace = true }
 serde_json = { workspace = true }
 
-[build-dependencies]
-djls-dev = { workspace = true }
-
 [lints]
 workspace = true

crates/djls/build.rs

@@ -1,3 +0,0 @@
-fn main() {
-    djls_dev::setup_python_linking();
-}

crates/djls/src/commands/serve.rs

@@ -22,10 +22,6 @@ impl Command for Serve {
     fn execute(&self, _args: &Args) -> Result<Exit> {
         djls_server::run()?;
 
-        // Exit here instead of returning control to the `Cli`, for ... reasons?
-        // If we don't exit here, ~~~ something ~~~ goes on with PyO3 (I assume)
-        // or the Python entrypoint wrapper to indefinitely hang the CLI and keep
-        // the process running
         Exit::success()
             .with_message("Server completed successfully")
             .process_exit()

crates/djls/src/lib.rs

@@ -1,34 +0,0 @@
-/// `PyO3` entrypoint for the Django Language Server CLI.
-///
-/// This module provides a Python interface using `PyO3` to solve Python runtime
-/// interpreter linking issues. The `PyO3` approach avoids complexities with
-/// static/dynamic linking when building binaries that interact with Python.
-mod args;
-mod cli;
-mod commands;
-mod exit;
-
-use std::env;
-
-use pyo3::prelude::*;
-
-#[pyfunction]
-/// Entry point called by Python when the CLI is invoked.
-/// This function handles argument parsing from Python and routes to the Rust CLI logic.
-fn entrypoint(_py: Python) -> PyResult<()> {
-    // Skip python interpreter and script path, add command name
-    let args: Vec<String> = std::iter::once("djls".to_string())
-        .chain(env::args().skip(2))
-        .collect();
-
-    // Run the CLI with the adjusted args
-    cli::run(args).map_err(|e| pyo3::exceptions::PyRuntimeError::new_err(e.to_string()))?;
-
-    Ok(())
-}
-
-#[pymodule]
-fn djls(_py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> {
-    m.add_function(wrap_pyfunction!(entrypoint, m)?)?;
-    Ok(())
-}

crates/djls/src/main.rs

@@ -1,7 +1,4 @@
-/// Binary interface for local development only.
+/// Binary interface for the Django Language Server CLI.
-///
-/// This binary exists for development and testing with `cargo run`.
-/// The production CLI is distributed through the `PyO3` interface in lib.rs.
 mod args;
 mod cli;
 mod commands;

docs/index.md

@@ -48,7 +48,7 @@ The foundation is solid though:
 
 - [x] Working server architecture
     - [x] Language Server Protocol implementation in Rust
-    - [x] Direct Django project interaction through PyO3
+    - [x] Django interaction via IPC
     - [x] Single binary distribution with Python packaging
 - [x] Custom template parser to support LSP features
     - [x] Basic HTML parsing, including style and script tags
@@ -178,7 +178,7 @@ All feature requests should ideally start out as a discussion topic, to gather f
 
 ### Development
 
-The project is written in Rust using PyO3 for Python integration. Here is a high-level overview of the project and the various crates:
+The project is written in Rust with IPC for Python communication. Here is a high-level overview of the project and the various crates:
 
 - Main CLI interface ([`crates/djls/`](https://github.com/joshuadavidthomas/django-language-server/blob/main/crates/djls/))
 - Django and Python project introspection ([`crates/djls-project/`](https://github.com/joshuadavidthomas/django-language-server/blob/main/crates/djls-project/))

noxfile.py

@@ -94,12 +94,6 @@ def tests(session, django):
 
     command = ["cargo", "test"]
 
-    # TODO: Remove this exclusion once PyO3 is replaced with subprocess oracle pattern
-    # Temporarily exclude djls-project tests on Windows due to PyO3 DLL loading issues
-    # (STATUS_DLL_NOT_FOUND when the test executable tries to load Python)
-    if platform.system() == "Windows":
-        command.extend(["--workspace", "--exclude", "djls-project"])
-
     if session.posargs:
         args = []
         for arg in session.posargs:
@@ -148,9 +142,6 @@ def gha_matrix(session):
     include_list = []
     for os_name in os_list:
         for combo in versions_list:
-            # Skip Python 3.9 on macOS due to PyO3/framework linking issues
-            if os_name.startswith("macos") and combo["python-version"] == "3.9":
-                continue
             include_list.append({**combo, "os": os_name})
 
     matrix = {"include": include_list}

pyproject.toml

@@ -69,9 +69,6 @@ classifiers = [
   "Topic :: Text Editors :: Integrated Development Environments (IDE)"
 ]
 
-[project.scripts]
-djls = "djls:entrypoint"
-
 [project.urls]
 Documentation = "https://django-language-server.readthedocs.io/"
 Issues = "https://github.com/joshuadavidthomas/django-language-server/issues"

python/Justfile

@@ -0,0 +1,12 @@
+set unstable := true
+
+[private]
+default:
+    @just --list
+
+[private]
+fmt:
+    @just --fmt
+
+build:
+    uv run build.py

python/build.py

@@ -0,0 +1,24 @@
+from __future__ import annotations
+
+import zipapp
+from pathlib import Path
+
+
+def main():
+    source_dir = Path(__file__).parent / "src" / "djls_inspector"
+    output_file = Path(__file__).parent / "dist" / "djls_inspector.pyz"
+    output_file.parent.mkdir(exist_ok=True)
+
+    zipapp.create_archive(
+        source_dir,
+        target=output_file,
+        interpreter=None,  # No shebang - will be invoked explicitly
+        compressed=True,
+    )
+
+    print(f"Successfully created {output_file}")
+    print(f"Size: {output_file.stat().st_size} bytes")
+
+
+if __name__ == "__main__":
+    main()

python/pyproject.toml

@@ -0,0 +1,12 @@
+[build-system]
+requires = ["uv_build>=0.7.21,<0.8"]
+build-backend = "uv_build"
+
+[project]
+authors = [
+  { name = "Josh Thomas", email = "josh@joshthomas.dev" }
+]
+dependencies = []
+name = "djls-inspector"
+requires-python = ">=3.9"
+version = "0.1.0"

python/src/djls_inspector/__main__.py

@@ -0,0 +1,44 @@
+from __future__ import annotations
+
+import json
+import sys
+
+# When running from zipapp, we need to import without the package prefix
+# since the zipapp root is already the package
+try:
+    # Try direct import (when running as zipapp)
+    from inspector import DjlsResponse
+    from inspector import handle_request
+except ImportError:
+    # Fall back to package import (when running with python -m)
+    from djls_inspector.inspector import DjlsResponse
+    from djls_inspector.inspector import handle_request
+
+
+def main() -> None:
+    try:
+        for line in sys.stdin:
+            line = line.strip()
+            if not line:
+                continue
+
+            try:
+                request = json.loads(line)
+                response = handle_request(request)
+            except json.JSONDecodeError as e:
+                response = DjlsResponse(ok=False, error=f"Invalid JSON: {e}")
+            except Exception as e:
+                response = DjlsResponse(ok=False, error=f"Unexpected error: {e}")
+
+            response_json = json.dumps(response.to_dict())
+            print(response_json, flush=True)
+
+    except KeyboardInterrupt:
+        sys.exit(0)
+    except Exception as e:
+        print(f"Fatal error in inspector: {e}", file=sys.stderr)
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

python/src/djls_inspector/inspector.py

@@ -0,0 +1,77 @@
+from __future__ import annotations
+
+from dataclasses import asdict
+from dataclasses import dataclass
+from typing import Any
+
+try:
+    # Try direct import (when running as zipapp)
+    from queries import Query
+    from queries import QueryData
+    from queries import get_installed_templatetags
+    from queries import get_python_environment_info
+    from queries import initialize_django
+except ImportError:
+    # Fall back to relative import (when running with python -m)
+    from .queries import Query
+    from .queries import QueryData
+    from .queries import get_installed_templatetags
+    from .queries import get_python_environment_info
+    from .queries import initialize_django
+
+
+@dataclass
+class DjlsRequest:
+    query: Query
+    args: list[str] | None = None
+
+
+@dataclass
+class DjlsResponse:
+    ok: bool
+    data: QueryData | None = None
+    error: str | None = None
+
+    def to_dict(self) -> dict[str, Any]:
+        d = asdict(self)
+        # Convert Path objects to strings for JSON serialization
+        if self.data:
+            if hasattr(self.data, "__dataclass_fields__"):
+                data_dict = asdict(self.data)
+                # Convert Path objects to strings
+                for key, value in data_dict.items():
+                    if key in ["sys_base_prefix", "sys_executable", "sys_prefix"]:
+                        if value:
+                            data_dict[key] = str(value)
+                    elif key == "sys_path":
+                        data_dict[key] = [str(p) for p in value]
+                d["data"] = data_dict
+        return d
+
+
+def handle_request(request: dict[str, Any]) -> DjlsResponse:
+    try:
+        query_str = request.get("query")
+        if not query_str:
+            return DjlsResponse(ok=False, error="Missing 'query' field in request")
+
+        try:
+            query = Query(query_str)
+        except ValueError:
+            return DjlsResponse(ok=False, error=f"Unknown query type: {query_str}")
+
+        args = request.get("args")
+
+        if query == Query.PYTHON_ENV:
+            return DjlsResponse(ok=True, data=get_python_environment_info())
+
+        elif query == Query.TEMPLATETAGS:
+            return DjlsResponse(ok=True, data=get_installed_templatetags())
+
+        elif query == Query.DJANGO_INIT:
+            return DjlsResponse(ok=True, data=initialize_django())
+
+        return DjlsResponse(ok=False, error=f"Unhandled query type: {query}")
+
+    except Exception as e:
+        return DjlsResponse(ok=False, error=str(e))

python/src/djls_inspector/queries.py

@@ -0,0 +1,174 @@
+from __future__ import annotations
+
+import sys
+from dataclasses import dataclass
+from enum import Enum
+from pathlib import Path
+from typing import Literal
+
+
+class Query(str, Enum):
+    PYTHON_ENV = "python_env"
+    TEMPLATETAGS = "templatetags"
+    DJANGO_INIT = "django_init"
+
+
+@dataclass
+class PythonEnvironmentQueryData:
+    sys_base_prefix: Path
+    sys_executable: Path
+    sys_path: list[Path]
+    sys_platform: str
+    sys_prefix: Path
+    sys_version_info: tuple[
+        int, int, int, Literal["alpha", "beta", "candidate", "final"], int
+    ]
+
+
+def get_python_environment_info():
+    return PythonEnvironmentQueryData(
+        sys_base_prefix=Path(sys.base_prefix),
+        sys_executable=Path(sys.executable),
+        sys_path=[Path(p) for p in sys.path],
+        sys_platform=sys.platform,
+        sys_prefix=Path(sys.prefix),
+        sys_version_info=(
+            sys.version_info.major,
+            sys.version_info.minor,
+            sys.version_info.micro,
+            sys.version_info.releaselevel,
+            sys.version_info.serial,
+        ),
+    )
+
+
+@dataclass
+class TemplateTagQueryData:
+    templatetags: list[TemplateTag]
+
+
+@dataclass
+class TemplateTag:
+    name: str
+    module: str
+    doc: str | None
+
+
+def get_installed_templatetags() -> TemplateTagQueryData:
+    import django
+    from django.apps import apps
+    from django.template.engine import Engine
+    from django.template.library import import_library
+
+    # Ensure Django is set up
+    if not apps.ready:
+        django.setup()
+
+    templatetags: list[TemplateTag] = []
+
+    engine = Engine.get_default()
+
+    for library in engine.template_builtins:
+        if library.tags:
+            for tag_name, tag_func in library.tags.items():
+                templatetags.append(
+                    TemplateTag(
+                        name=tag_name, module=tag_func.__module__, doc=tag_func.__doc__
+                    )
+                )
+
+    for lib_module in engine.libraries.values():
+        library = import_library(lib_module)
+        if library and library.tags:
+            for tag_name, tag_func in library.tags.items():
+                templatetags.append(
+                    TemplateTag(
+                        name=tag_name, module=tag_func.__module__, doc=tag_func.__doc__
+                    )
+                )
+
+    return TemplateTagQueryData(templatetags=templatetags)
+
+
+@dataclass
+class DjangoInitQueryData:
+    success: bool
+    message: str | None = None
+
+
+def initialize_django() -> DjangoInitQueryData:
+    import os
+    import django
+    from django.apps import apps
+
+    try:
+        # Check if Django settings are configured
+        if not os.environ.get("DJANGO_SETTINGS_MODULE"):
+            # Try to find and set settings module
+            import sys
+            from pathlib import Path
+
+            # Look for manage.py to determine project structure
+            current_path = Path.cwd()
+            manage_py = None
+
+            # Search up to 3 levels for manage.py
+            for _ in range(3):
+                if (current_path / "manage.py").exists():
+                    manage_py = current_path / "manage.py"
+                    break
+                if current_path.parent == current_path:
+                    break
+                current_path = current_path.parent
+
+            if not manage_py:
+                return DjangoInitQueryData(
+                    success=False,
+                    message="Could not find manage.py or DJANGO_SETTINGS_MODULE not set",
+                )
+
+            # Add project directory to sys.path
+            project_dir = manage_py.parent
+            if str(project_dir) not in sys.path:
+                sys.path.insert(0, str(project_dir))
+
+            # Try to find settings module - look for common patterns
+            # First check if there's a directory with the same name as the parent
+            project_name = project_dir.name
+            settings_candidates = [
+                f"{project_name}.settings",  # e.g., myproject.settings
+                "settings",  # Just settings.py in root
+                "config.settings",  # Common pattern
+                "project.settings",  # Another common pattern
+            ]
+
+            # Also check for any directory containing settings.py
+            for item in project_dir.iterdir():
+                if item.is_dir() and (item / "settings.py").exists():
+                    candidate = f"{item.name}.settings"
+                    if candidate not in settings_candidates:
+                        settings_candidates.insert(
+                            0, candidate
+                        )  # Prioritize found settings
+
+            for settings_candidate in settings_candidates:
+                try:
+                    __import__(settings_candidate)
+                    os.environ["DJANGO_SETTINGS_MODULE"] = settings_candidate
+                    break
+                except ImportError:
+                    continue
+
+        # Set up Django
+        if not apps.ready:
+            django.setup()
+
+        return DjangoInitQueryData(
+            success=True, message="Django initialized successfully"
+        )
+
+    except Exception as e:
+        return DjangoInitQueryData(success=False, message=str(e))
+
+
+QueryData = PythonEnvironmentQueryData | TemplateTagQueryData | DjangoInitQueryData

python/uv.lock

@@ -0,0 +1,8 @@
+version = 1
+revision = 2
+requires-python = ">=3.9"
+
+[[package]]
+name = "djls-inspector"
+version = "0.1.0"
+source = { editable = "." }

tests/project/djls_app/templates/djls_app/base.html

@@ -19,7 +19,7 @@
       {% endif %}
       <img src="{% static 'images/logo.png' %}" alt="Logo">
       {# This is a comment #}
-    {% endblock %}
+    {% endblock content %}
 
   </body>
 </html>

uv.lock

@@ -1,5 +1,5 @@
 version = 1
-revision = 3
+revision = 2
 requires-python = ">=3.9"
 resolution-markers = [
     "python_full_version >= '3.10'",
@@ -277,7 +277,7 @@ wheels = [
 
 [[package]]
 name = "django-language-server"
-version = "5.2.0a0"
+version = "5.2.0"
 source = { editable = "." }
 
 [package.dev-dependencies]