switch from runner to ipc and long-running sidecar process (#21)

crates/djls-python/Cargo.toml

@@ -4,6 +4,8 @@ version = "0.0.0"
 edition = "2021"
 
 [dependencies]
+djls-ipc = { workspace = true }
+
 serde = { workspace = true }
 serde_json = { workspace = true }
 thiserror = { workspace = true }

crates/djls-python/src/lib.rs

@@ -1,11 +1,7 @@
 mod packaging;
 mod python;
-mod runner;
-mod scripts;
 
 pub use crate::packaging::ImportCheck;
+pub use crate::packaging::PackagingError;
 pub use crate::python::Python;
 pub use crate::python::PythonError;
-pub use crate::runner::Runner;
-pub use crate::runner::RunnerError;
-pub use crate::runner::ScriptRunner;

crates/djls-python/src/packaging.rs

@@ -1,6 +1,4 @@
-use crate::python::Python;
-use crate::runner::{RunnerError, ScriptRunner};
-use crate::scripts;
+use djls_ipc::{parse_json_response, JsonResponse, PythonProcess, TransportError};
 use serde::Deserialize;
 use std::collections::HashMap;
 use std::fmt;
@@ -59,8 +57,16 @@ pub struct ImportCheck {
     can_import: bool,
 }
 
-impl ScriptRunner for ImportCheck {
-    const SCRIPT: &'static str = scripts::HAS_IMPORT;
+impl TryFrom<JsonResponse> for ImportCheck {
+    type Error = TransportError;
+
+    fn try_from(response: JsonResponse) -> Result<Self, Self::Error> {
+        response
+            .data()
+            .clone()
+            .ok_or_else(|| TransportError::Process("No data in response".to_string()))
+            .and_then(|data| serde_json::from_value(data).map_err(TransportError::Json))
+    }
 }
 
 impl ImportCheck {
@@ -68,9 +74,14 @@ impl ImportCheck {
         self.can_import
     }
 
-    pub fn check(py: &Python, module: &str) -> Result<bool, RunnerError> {
-        let result = ImportCheck::run_with_py_args(py, module)?;
-        Ok(result.can_import)
+    pub fn check(
+        python: &mut PythonProcess,
+        modules: Option<Vec<String>>,
+    ) -> Result<bool, PackagingError> {
+        let response = python.send("has_import", modules)?;
+        let response = parse_json_response(response)?;
+        let check = Self::try_from(response)?;
+        Ok(check.can_import)
     }
 }
 
@@ -82,15 +93,9 @@ pub enum PackagingError {
     #[error("JSON parsing error: {0}")]
     Json(#[from] serde_json::Error),
 
-    #[error(transparent)]
-    Runner(#[from] Box<RunnerError>),
+    #[error("Transport error: {0}")]
+    Transport(#[from] TransportError),
 
     #[error("UTF-8 conversion error: {0}")]
     Utf8(#[from] std::string::FromUtf8Error),
 }
-
-impl From<RunnerError> for PackagingError {
-    fn from(err: RunnerError) -> Self {
-        PackagingError::Runner(Box::new(err))
-    }
-}

crates/djls-python/src/python.rs

@@ -1,10 +1,8 @@
 use crate::packaging::{Packages, PackagingError};
-use crate::runner::{Runner, RunnerError, ScriptRunner};
-use crate::scripts;
+use djls_ipc::{parse_json_response, JsonResponse, PythonProcess, TransportError};
 use serde::Deserialize;
 use std::fmt;
-use std::path::{Path, PathBuf};
-use which::which;
+use std::path::PathBuf;
 
 #[derive(Clone, Debug, Deserialize)]
 pub struct VersionInfo {
@@ -60,29 +58,23 @@ pub struct Python {
     packages: Packages,
 }
 
-#[derive(Debug, Deserialize)]
-pub struct PythonSetup(Python);
+impl TryFrom<JsonResponse> for Python {
+    type Error = TransportError;
 
-impl ScriptRunner for PythonSetup {
-    const SCRIPT: &'static str = scripts::PYTHON_SETUP;
-}
-
-impl From<PythonSetup> for Python {
-    fn from(setup: PythonSetup) -> Self {
-        setup.0
+    fn try_from(response: JsonResponse) -> Result<Self, Self::Error> {
+        response
+            .data()
+            .clone()
+            .ok_or_else(|| TransportError::Process("No data in response".to_string()))
+            .and_then(|data| serde_json::from_value(data).map_err(TransportError::Json))
     }
 }
 
 impl Python {
-    pub fn initialize() -> Result<Self, PythonError> {
-        let executable = which("python")?;
-        Ok(PythonSetup::run_with_path(&executable)?.into())
-    }
-}
-
-impl Runner for Python {
-    fn get_executable(&self) -> &Path {
-        &self.sys_executable
+    pub fn setup(python: &mut PythonProcess) -> Result<Self, PythonError> {
+        let response = python.send("python_setup", None)?;
+        let response = parse_json_response(response)?;
+        Ok(Self::try_from(response)?)
     }
 }
 
@@ -123,15 +115,9 @@ pub enum PythonError {
     #[error("Failed to locate Python executable: {0}")]
     PythonNotFound(#[from] which::Error),
 
-    #[error(transparent)]
-    Runner(#[from] Box<RunnerError>),
+    #[error("Transport error: {0}")]
+    Transport(#[from] TransportError),
 
     #[error("UTF-8 conversion error: {0}")]
     Utf8(#[from] std::string::FromUtf8Error),
 }
-
-impl From<RunnerError> for PythonError {
-    fn from(err: RunnerError) -> Self {
-        PythonError::Runner(Box::new(err))
-    }
-}

crates/djls-python/src/runner.rs

@@ -1,179 +0,0 @@
-use crate::python::{Python, PythonError};
-use serde::ser::Error;
-use serde::Deserialize;
-use std::path::{Path, PathBuf};
-use std::process::Command;
-
-pub trait Runner {
-    fn get_executable(&self) -> &Path;
-
-    fn run_module(&self, command: &str) -> std::io::Result<String> {
-        let output = Command::new(self.get_executable())
-            .arg("-m")
-            .arg(command)
-            .output()?;
-
-        if !output.status.success() {
-            let error = String::from_utf8_lossy(&output.stderr);
-            return Err(std::io::Error::new(
-                std::io::ErrorKind::Other,
-                format!("Command failed: {}", error),
-            ));
-        }
-
-        Ok(String::from_utf8_lossy(&output.stdout).into_owned())
-    }
-
-    fn run_module_with_args(&self, command: &str, args: &[&str]) -> std::io::Result<String> {
-        let output = Command::new(self.get_executable())
-            .arg("-m")
-            .arg(command)
-            .args(args)
-            .output()?;
-
-        if !output.status.success() {
-            let error = String::from_utf8_lossy(&output.stderr);
-            return Err(std::io::Error::new(
-                std::io::ErrorKind::Other,
-                format!("Command failed: {}", error),
-            ));
-        }
-
-        Ok(String::from_utf8_lossy(&output.stdout).into_owned())
-    }
-
-    fn run_python_code(&self, code: &str) -> std::io::Result<String> {
-        let output = Command::new(self.get_executable())
-            .args(["-c", code])
-            .output()?;
-
-        if !output.status.success() {
-            let error = String::from_utf8_lossy(&output.stderr);
-            return Err(std::io::Error::new(
-                std::io::ErrorKind::Other,
-                format!("Python execution failed: {}", error),
-            ));
-        }
-
-        Ok(String::from_utf8_lossy(&output.stdout).into_owned())
-    }
-
-    fn run_python_code_with_args(&self, code: &str, args: &str) -> std::io::Result<String> {
-        let output = Command::new(self.get_executable())
-            .args(["-c", code, args])
-            .output()?;
-
-        if !output.status.success() {
-            let error = String::from_utf8_lossy(&output.stderr);
-            return Err(std::io::Error::new(
-                std::io::ErrorKind::Other,
-                format!("Python execution failed: {}", error),
-            ));
-        }
-
-        Ok(String::from_utf8_lossy(&output.stdout).into_owned())
-    }
-
-    fn run_script<T>(&self, script: &str) -> Result<T, serde_json::Error>
-    where
-        T: for<'de> Deserialize<'de>,
-    {
-        let result = self
-            .run_python_code(script)
-            .map_err(|e| serde_json::Error::custom(e.to_string()))?;
-        serde_json::from_str(&result)
-    }
-
-    fn run_script_with_args<T>(&self, script: &str, args: &str) -> Result<T, serde_json::Error>
-    where
-        T: for<'de> Deserialize<'de>,
-    {
-        let result = self
-            .run_python_code_with_args(script, args)
-            .map_err(|e| serde_json::Error::custom(e.to_string()))?;
-        serde_json::from_str(&result)
-    }
-}
-
-pub struct SimpleRunner {
-    executable: PathBuf,
-}
-
-impl SimpleRunner {
-    pub fn new(executable: PathBuf) -> Self {
-        Self { executable }
-    }
-}
-
-impl Runner for SimpleRunner {
-    fn get_executable(&self) -> &Path {
-        &self.executable
-    }
-}
-
-pub trait ScriptRunner: Sized {
-    const SCRIPT: &'static str;
-
-    fn run_with_exe<R: Runner>(runner: &R) -> Result<Self, RunnerError>
-    where
-        Self: for<'de> Deserialize<'de>,
-    {
-        let result = runner.run_script(Self::SCRIPT).map_err(RunnerError::from)?;
-        Ok(result)
-    }
-
-    fn run_with_exe_args<R: Runner>(runner: &R, args: &str) -> Result<Self, RunnerError>
-    where
-        Self: for<'de> Deserialize<'de>,
-    {
-        let result = runner
-            .run_script_with_args(Self::SCRIPT, args)
-            .map_err(RunnerError::from)?;
-        Ok(result)
-    }
-
-    fn run_with_path(executable: &Path) -> Result<Self, RunnerError>
-    where
-        Self: for<'de> Deserialize<'de>,
-    {
-        let runner = &SimpleRunner::new(executable.to_path_buf());
-        Self::run_with_exe(runner)
-    }
-
-    fn run_with_path_args(executable: &Path, args: &str) -> Result<Self, RunnerError>
-    where
-        Self: for<'de> Deserialize<'de>,
-    {
-        let runner = &SimpleRunner::new(executable.to_path_buf());
-        Self::run_with_exe_args(runner, args)
-    }
-
-    fn run_with_py(python: &Python) -> Result<Self, RunnerError>
-    where
-        Self: for<'de> Deserialize<'de>,
-    {
-        Self::run_with_exe(python)
-    }
-
-    fn run_with_py_args(python: &Python, args: &str) -> Result<Self, RunnerError>
-    where
-        Self: for<'de> Deserialize<'de>,
-    {
-        Self::run_with_exe_args(python, args)
-    }
-}
-
-#[derive(Debug, thiserror::Error)]
-pub enum RunnerError {
-    #[error("IO error: {0}")]
-    Io(#[from] std::io::Error),
-
-    #[error("JSON parsing error: {0}")]
-    Json(#[from] serde_json::Error),
-
-    #[error(transparent)]
-    Python(#[from] PythonError),
-
-    #[error("UTF-8 conversion error: {0}")]
-    Utf8(#[from] std::string::FromUtf8Error),
-}

crates/djls-python/src/scripts.rs

@@ -1,14 +0,0 @@
-#[macro_export]
-macro_rules! include_script {
-    ($name:expr) => {
-        include_str!(concat!(
-            env!("CARGO_WORKSPACE_DIR"),
-            "python/djls/scripts/",
-            $name,
-            ".py"
-        ))
-    };
-}
-
-pub const HAS_IMPORT: &str = include_script!("has_import");
-pub const PYTHON_SETUP: &str = include_script!["python_setup"];