fix: path resolution bugs

crates/erg_common/config.rs

@@ -4,22 +4,21 @@
 use std::env;
 use std::fmt;
 use std::fs::File;
-use std::io::Write;
+use std::io::{stdin, BufRead, BufReader, Read, Write};
-use std::io::{stdin, BufRead, BufReader, Read};
 use std::path::{Path, PathBuf};
 use std::process;
 use std::str::FromStr;
 
+use crate::consts::ERG_MODE;
 use crate::env::{erg_py_external_lib_path, erg_pystd_path, erg_std_path, python_site_packages};
 use crate::help_messages::{command_message, mode_message, OPTIONS};
 use crate::levenshtein::get_similar_name;
-use crate::normalize_path;
 use crate::pathutil::add_postfix_foreach;
 use crate::python_util::{detect_magic_number, get_python_version, get_sys_path, PythonVersion};
 use crate::random::random;
 use crate::serialize::{get_magic_num_from_bytes, get_ver_from_magic_num};
 use crate::stdin::GLOBAL_STDIN;
-use crate::{power_assert, read_file};
+use crate::{normalize_path, power_assert, read_file};
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
 pub enum ErgMode {
@@ -440,6 +439,10 @@ impl Input {
         }
     }
 
+    pub fn sys_path(&self) -> Vec<PathBuf> {
+        get_sys_path(self.unescaped_path().parent())
+    }
+
     /// resolution order:
     /// 1. `{path/to}.er`
     /// 2. `{path/to}/__init__.er`
@@ -519,12 +522,12 @@ impl Input {
     }
 
     pub fn resolve_py(&self, path: &Path) -> Result<PathBuf, std::io::Error> {
-        self.resolve_local_py(path).or_else(|_| {
+        if ERG_MODE || path.starts_with("./") {
-            // For now, only see `site-packages`
+            if let Ok(path) = self.resolve_local_py(path) {
-            let site_packages = get_sys_path()
+                return Ok(path);
-                .into_iter()
+            }
-                .filter(|p| p.as_os_str().to_string_lossy().contains("site-packages"));
+        }
-            for sys_path in site_packages {
+        for sys_path in self.sys_path() {
             let mut dir = sys_path;
             dir.push(path);
             dir.set_extension("py");
@@ -542,7 +545,6 @@ impl Input {
             std::io::ErrorKind::NotFound,
             format!("cannot find module `{}`", path.display()),
         ))
-        })
     }
 
     pub fn resolve_path(&self, path: &Path) -> Option<PathBuf> {
@@ -604,6 +606,8 @@ impl Input {
         }
     }
 
+    /// 1. `site-packages/{path/to}.d.er`
+    /// 2. `site-packages/{path.d/to.d}/__init__.d.er`
     fn resolve_std_decl_path(root: PathBuf, path: &Path) -> Option<PathBuf> {
         let mut path = add_postfix_foreach(path, ".d");
         path.set_extension("d.er"); // set_extension overrides the previous one
@@ -624,14 +628,19 @@ impl Input {
         }
     }
 
-    /// 1. `site-packages/__pycache__/{path}.d.er`
+    /// 1. `site-packages/__pycache__/{path/to}.d.er`
-    /// 2. `site-packages/{path}/__pycache__/__init__.d.er`
+    /// 2. `site-packages/{path/to}/__pycache__/__init__.d.er`
+    ///
+    /// e.g. `toml/encoder`
+    ///     -> `site-packages/toml/__pycache__/encoder.d.er`, `site-packages/toml/encoder/__pycache__/__init__.d.er`
     fn resolve_site_pkgs_decl_path(site_packages: PathBuf, path: &Path) -> Option<PathBuf> {
-        let mut path_buf = path.to_path_buf();
+        let dir = path.parent().unwrap_or_else(|| Path::new(""));
-        path_buf.set_extension("d.er"); // set_extension overrides the previous one
+        let mut file_path = PathBuf::from(path.file_stem().unwrap_or_default());
+        file_path.set_extension("d.er"); // set_extension overrides the previous one
         if let Ok(path) = site_packages
+            .join(dir)
             .join("__pycache__")
-            .join(&path_buf)
+            .join(&file_path)
             .canonicalize()
         {
             Some(normalize_path(path))

crates/erg_common/env.rs

@@ -54,7 +54,7 @@ fn _erg_external_lib_path() -> PathBuf {
         })
 }
 fn _python_site_packages() -> impl Iterator<Item = PathBuf> {
-    get_sys_path()
+    get_sys_path(None)
         .into_iter()
         .filter(|p| p.ends_with("site-packages"))
         .map(|p| {
@@ -98,6 +98,12 @@ pub fn python_site_packages() -> Vec<PathBuf> {
     PYTHON_SITE_PACKAGES.with(|s| s.clone())
 }
 
+pub fn is_std_decl_path(path: &Path) -> bool {
+    path.starts_with(erg_pystd_path())
+        || path.starts_with(erg_std_decl_path())
+        || path.starts_with(erg_py_external_lib_path())
+}
+
 pub fn is_pystd_main_module(path: &Path) -> bool {
     let mut path = PathBuf::from(path);
     if path.ends_with("__init__.d.er") {

crates/erg_common/pathutil.rs

@@ -161,3 +161,7 @@ pub fn squash(path: PathBuf) -> PathBuf {
     }
     result
 }
+
+pub fn remove_verbatim(path: &Path) -> String {
+    path.to_string_lossy().replace("\\\\?\\", "")
+}

crates/erg_common/python_util.rs

@@ -1,10 +1,12 @@
 //! utilities for calling CPython.
 //!
 //! CPythonを呼び出すためのユーティリティー
-use std::path::PathBuf;
+use std::fs;
+use std::path::{Path, PathBuf};
 use std::process::Command;
 
 use crate::fn_name_full;
+use crate::pathutil::remove_verbatim;
 use crate::serialize::get_magic_num_from_bytes;
 
 #[cfg(unix)]
@@ -688,19 +690,24 @@ pub fn env_python_version() -> PythonVersion {
     get_python_version(&which_python())
 }
 
-pub fn get_sys_path() -> Vec<PathBuf> {
+pub fn get_sys_path(working_dir: Option<&Path>) -> Vec<PathBuf> {
+    let working_dir = fs::canonicalize(working_dir.unwrap_or(Path::new(""))).unwrap_or_default();
+    let working_dir = remove_verbatim(&working_dir);
     let py_command = which_python();
-    let code = "import sys; print('\\n'.join(sys.path))";
+    let code = "import os, sys; print('\\n'.join(map(lambda p: os.path.abspath(p), sys.path)))";
     let out = if cfg!(windows) {
         Command::new("cmd")
             .arg("/C")
+            .arg("cd")
+            .arg(working_dir)
+            .arg("&&")
             .arg(py_command)
             .arg("-c")
             .arg(code)
             .output()
             .expect("cannot get the sys.path")
     } else {
-        let exec_command = format!("{py_command} -c \"{code}\"");
+        let exec_command = format!("cd {working_dir} && {py_command} -c \"{code}\"");
         Command::new("sh")
             .arg("-c")
             .arg(exec_command)

crates/erg_compiler/context/initialize/classes.rs

@@ -2566,6 +2566,9 @@ impl Context {
                 Const,
                 Some(QUANTIFIED_FUNC),
             );
+        } else {
+            self.register_builtin_const(MUT_INT, vis.clone(), ValueObj::builtin_class(Int));
+            self.register_builtin_const(MUT_STR, vis, ValueObj::builtin_class(Str));
         }
     }
 }

crates/erg_compiler/context/register.rs

@@ -6,8 +6,8 @@ use std::process::{Command, Stdio};
 use std::time::SystemTime;
 
 use erg_common::config::ErgMode;
-use erg_common::consts::PYTHON_MODE;
+use erg_common::consts::{ERG_MODE, PYTHON_MODE};
-use erg_common::env::is_pystd_main_module;
+use erg_common::env::{is_pystd_main_module, is_std_decl_path};
 use erg_common::erg_util::BUILTIN_ERG_MODS;
 use erg_common::levenshtein::get_similar_name;
 use erg_common::pathutil::{DirKind, FileKind};
@@ -15,8 +15,7 @@ use erg_common::python_util::BUILTIN_PYTHON_MODS;
 use erg_common::set::Set;
 use erg_common::traits::{Locational, Stream};
 use erg_common::triple::Triple;
-use erg_common::{get_hash, log, set};
+use erg_common::{get_hash, log, set, unique_in_place, Str};
-use erg_common::{unique_in_place, Str};
 
 use ast::{
     ConstIdentifier, Decorator, DefId, Identifier, OperationKind, PolyTypeSpec, PreDeclTypeSpec,
@@ -145,6 +144,16 @@ impl std::str::FromStr for PylyzerStatus {
     }
 }
 
+enum Availability {
+    Available,
+    InProgress,
+    NotFound,
+    Unreadable,
+    OutOfDate,
+}
+
+use Availability::*;
+
 const UBAR: &Str = &Str::ever("_");
 
 impl Context {
@@ -1816,7 +1825,9 @@ impl Context {
                 return Err(self.import_err(line!(), __name__, loc));
             }
         };
+        if ERG_MODE {
             self.check_mod_vis(path.as_path(), __name__, loc)?;
+        }
         if let Some(referrer) = self.cfg.input.path() {
             let graph = &self.shared.as_ref().unwrap().graph;
             graph.inc_ref(referrer, path.clone());
@@ -1942,24 +1953,41 @@ impl Context {
         Str::from(name)
     }
 
-    fn can_reuse(path: &Path) -> Option<PylyzerStatus> {
+    fn availability(path: &Path) -> Availability {
-        let file = std::fs::File::open(path).ok()?;
+        let Ok(file) = std::fs::File::open(path) else {
+            return Availability::NotFound;
+        };
+        if is_std_decl_path(path) {
+            return Availability::Available;
+        }
         let mut line = "".to_string();
-        std::io::BufReader::new(file).read_line(&mut line).ok()?;
+        let Ok(_) = std::io::BufReader::new(file).read_line(&mut line) else {
-        let status = line.parse::<PylyzerStatus>().ok()?;
+            return Availability::Unreadable;
-        let meta = std::fs::metadata(&status.file).ok()?;
+        };
+        if line.is_empty() {
+            return Availability::InProgress;
+        }
+        let Ok(status) = line.parse::<PylyzerStatus>() else {
+            return Availability::Available;
+        };
+        let Some(meta) = std::fs::metadata(&status.file).ok() else {
+            return Availability::NotFound;
+        };
         let dummy_hash = meta.len();
         if status.hash != dummy_hash {
-            None
+            Availability::OutOfDate
         } else {
-            Some(status)
+            Availability::Available
         }
     }
 
     fn get_decl_path(&self, __name__: &Str, loc: &impl Locational) -> CompileResult<PathBuf> {
         match self.cfg.input.resolve_decl_path(Path::new(&__name__[..])) {
             Some(path) => {
-                if Self::can_reuse(&path).is_none() {
+                if self.cfg.input.decl_file_is(&path) {
+                    return Ok(path);
+                }
+                if matches!(Self::availability(&path), OutOfDate | NotFound | Unreadable) {
                     let _ = self.try_gen_py_decl_file(__name__);
                 }
                 if is_pystd_main_module(path.as_path())
@@ -1998,6 +2026,9 @@ impl Context {
 
     fn try_gen_py_decl_file(&self, __name__: &Str) -> Result<PathBuf, ()> {
         if let Ok(path) = self.cfg.input.resolve_py(Path::new(&__name__[..])) {
+            if self.cfg.input.path() == Some(path.as_path()) {
+                return Ok(path);
+            }
             let (out, err) = if self.cfg.mode == ErgMode::LanguageServer || self.cfg.quiet_repl {
                 (Stdio::null(), Stdio::null())
             } else {