Merge pull request #444 from erg-lang/fix-443

Fix #443
This commit is contained in:
Shunsuke Shibayama 2023-07-26 20:45:23 +09:00 committed by GitHub
commit 17b44b1fa8
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
17 changed files with 177 additions and 145 deletions

View file

@ -46,7 +46,7 @@ impl<Checker: BuildRunnable, Parser: Parsable> Server<Checker, Parser> {
"checking {uri} passed, found warns: {}",
artifact.warns.len()
))?;
let uri_and_diags = self.make_uri_and_diags(uri.clone(), artifact.warns.clone());
let uri_and_diags = self.make_uri_and_diags(artifact.warns.clone());
// clear previous diagnostics
self.send_diagnostics(uri.clone().raw(), vec![])?;
for (uri, diags) in uri_and_diags.into_iter() {
@ -64,7 +64,7 @@ impl<Checker: BuildRunnable, Parser: Parsable> Server<Checker, Parser> {
.into_iter()
.chain(artifact.warns.clone().into_iter())
.collect();
let uri_and_diags = self.make_uri_and_diags(uri.clone(), diags);
let uri_and_diags = self.make_uri_and_diags(diags);
if uri_and_diags.is_empty() {
self.send_diagnostics(uri.clone().raw(), vec![])?;
}
@ -130,21 +130,18 @@ impl<Checker: BuildRunnable, Parser: Parsable> Server<Checker, Parser> {
Ok(())
}
fn make_uri_and_diags(
&mut self,
uri: NormalizedUrl,
errors: CompileErrors,
) -> Vec<(Url, Vec<Diagnostic>)> {
fn make_uri_and_diags(&mut self, errors: CompileErrors) -> Vec<(Url, Vec<Diagnostic>)> {
let mut uri_and_diags: Vec<(Url, Vec<Diagnostic>)> = vec![];
for err in errors.into_iter() {
let loc = err.core.get_loc_with_fallback();
let res_uri = if let Some(path) = err.input.path() {
Url::from_file_path(path.canonicalize().unwrap_or(path.to_path_buf()))
} else {
Ok(uri.clone().raw())
};
let res_uri = Url::from_file_path(
err.input
.path()
.canonicalize()
.unwrap_or(err.input.path().to_path_buf()),
);
let Ok(err_uri) = res_uri else {
crate::_log!("failed to get uri: {}", err.input.path().unwrap().display());
crate::_log!("failed to get uri: {}", err.input.path().display());
continue;
};
let mut message = remove_style(&err.core.main_message);

View file

@ -933,7 +933,7 @@ pub trait ErrorDisplay {
let (color, mark) = core.specified_theme();
let (gutter_color, chars) = core.theme.characters();
let mut msg = String::new();
msg += &core.fmt_header(color, self.caused_by(), self.input().kind.enclosed_name());
msg += &core.fmt_header(color, self.caused_by(), self.input().kind.as_str());
msg += "\n\n";
for sub_msg in &core.sub_messages {
msg += &sub_msg.format_code_and_pointer(self, color, gutter_color, mark, chars);
@ -957,7 +957,7 @@ pub trait ErrorDisplay {
write!(
f,
"{}\n\n",
core.fmt_header(color, self.caused_by(), self.input().kind.enclosed_name())
core.fmt_header(color, self.caused_by(), self.input().kind.as_str())
)?;
for sub_msg in &core.sub_messages {
write!(

View file

@ -73,14 +73,17 @@ impl InputKind {
matches!(self, Self::REPL | Self::DummyREPL(_))
}
pub fn path(&self) -> Option<&Path> {
pub fn path(&self) -> &Path {
match self {
Self::File(path) => Some(path),
_ => None,
Self::File(filename) => filename.as_path(),
Self::REPL | Self::Pipe(_) => Path::new("<stdin>"),
Self::DummyREPL(_stdin) => Path::new("<stdin>"),
Self::Str(_) => Path::new("<string>"),
Self::Dummy => Path::new("<dummy>"),
}
}
pub fn enclosed_name(&self) -> &str {
pub fn as_str(&self) -> &str {
match self {
Self::File(filename) => filename.to_str().unwrap_or("_"),
Self::REPL | Self::DummyREPL(_) | Self::Pipe(_) => "<stdin>",
@ -176,10 +179,6 @@ impl Input {
self.id
}
pub fn path(&self) -> Option<&Path> {
self.kind.path()
}
pub fn dir(&self) -> PathBuf {
self.kind.dir()
}
@ -189,7 +188,7 @@ impl Input {
}
pub fn enclosed_name(&self) -> &str {
self.kind.enclosed_name()
self.kind.as_str()
}
pub fn lineno(&self) -> usize {
@ -214,78 +213,40 @@ impl Input {
pub fn file_stem(&self) -> String {
match &self.kind {
InputKind::File(filename) => format!(
"{}_{}",
filename
.file_stem()
.and_then(|f| f.to_str())
.unwrap_or("_")
.trim_end_matches(".d"),
self.id
),
InputKind::REPL | InputKind::Pipe(_) => format!("stdin_{}", self.id),
InputKind::DummyREPL(stdin) => format!("stdin_{}_{}", stdin.name, self.id),
InputKind::Str(_) => format!("string_{}", self.id),
InputKind::Dummy => "dummy".to_string(),
InputKind::File(filename) => filename
.file_stem()
.and_then(|f| f.to_str())
.unwrap_or("_")
.trim_end_matches(".d")
.to_string(),
InputKind::REPL | InputKind::Pipe(_) => "<stdin>".to_string(),
InputKind::DummyREPL(stdin) => format!("<stdin_{}>", stdin.name),
InputKind::Str(_) => "<string>".to_string(),
InputKind::Dummy => "<dummy>".to_string(),
}
}
pub fn full_path(&self) -> PathBuf {
match &self.kind {
InputKind::File(filename) => {
PathBuf::from(format!("{}_{}", filename.display(), self.id))
}
InputKind::File(filename) => filename.clone(),
_ => PathBuf::from(self.file_stem()),
}
}
pub fn filename(&self) -> String {
match &self.kind {
InputKind::File(filename) => format!(
"{}_{}",
filename.file_name().and_then(|f| f.to_str()).unwrap_or("_"),
self.id
),
_ => self.file_stem(),
}
}
pub fn unescaped_file_stem(&self) -> &str {
match &self.kind {
InputKind::File(filename) => filename
.file_stem()
.and_then(|f| f.to_str())
.unwrap_or("_")
.trim_end_matches(".d"),
InputKind::REPL | InputKind::Pipe(_) => "stdin",
InputKind::DummyREPL(_stdin) => "stdin",
InputKind::Str(_) => "string",
InputKind::Dummy => "dummy",
}
}
pub fn unescaped_filename(&self) -> &str {
match &self.kind {
InputKind::File(filename) => filename
.file_name()
.and_then(|f| f.to_str())
.unwrap_or("_")
.trim_end_matches(".d"),
InputKind::REPL | InputKind::Pipe(_) => "stdin",
InputKind::DummyREPL(_stdin) => "stdin",
InputKind::Str(_) => "string",
InputKind::Dummy => "dummy",
.trim_end_matches(".d")
.to_string(),
_ => self.file_stem(),
}
}
pub fn unescaped_path(&self) -> &Path {
match &self.kind {
InputKind::File(filename) => filename.as_path(),
InputKind::REPL | InputKind::Pipe(_) => Path::new("stdin"),
InputKind::DummyREPL(_stdin) => Path::new("stdin"),
InputKind::Str(_) => Path::new("string"),
InputKind::Dummy => Path::new("dummy"),
}
pub fn path(&self) -> &Path {
self.kind.path()
}
pub fn module_name(&self) -> String {
@ -432,7 +393,7 @@ impl Input {
}
pub fn sys_path(&self) -> Result<Vec<PathBuf>, std::io::Error> {
get_sys_path(self.unescaped_path().parent())
get_sys_path(self.path().parent())
}
/// resolution order:
@ -687,7 +648,7 @@ impl Input {
}
pub fn decl_file_is(&self, decl_path: &Path) -> bool {
let mut py_path = self.unescaped_path().to_path_buf();
let mut py_path = self.path().to_path_buf();
py_path.set_extension("d.er");
if decl_path == py_path {
return true;

View file

@ -1,11 +1,14 @@
//! utilities for calling CPython.
//!
//! CPythonを呼び出すためのユーティリティー
use std::fs;
use std::env;
use std::fs::{self, File};
use std::io::Write;
use std::path::{Path, PathBuf};
use std::process::{Command, Stdio};
use std::process::{Command, ExitStatus, Stdio};
use crate::fn_name_full;
use crate::io::Output;
use crate::pathutil::remove_verbatim;
use crate::serialize::get_magic_num_from_bytes;
@ -849,3 +852,42 @@ pub fn spawn_py(py_command: Option<&str>, code: &str) {
.expect("cannot execute python");
}
}
pub fn exec_py_code(code: &str, output: Output) -> std::io::Result<ExitStatus> {
let mut out = if cfg!(windows) {
let fallback = |err: std::io::Error| {
// if the filename or extension is too long
// create a temporary file and execute it
if err.raw_os_error() == Some(206) {
let tmp_dir = env::temp_dir();
let tmp_file = tmp_dir.join("tmp.py");
File::create(&tmp_file)
.unwrap()
.write_all(code.as_bytes())
.unwrap();
Command::new(which_python())
.arg(tmp_file)
.stdout(output.clone())
.spawn()
} else {
Err(err)
}
};
Command::new(which_python())
.arg("-c")
.arg(code)
.stdout(output.clone())
.spawn()
.or_else(fallback)
.expect("cannot execute python")
} else {
let exec_command = format!("{} -c \"{code}\"", which_python());
Command::new("sh")
.arg("-c")
.arg(exec_command)
.stdout(output)
.spawn()
.expect("cannot execute python")
};
out.wait()
}

View file

@ -93,7 +93,7 @@ impl Runnable for HIRBuilder {
impl Buildable for HIRBuilder {
fn inherit(cfg: ErgConfig, shared: SharedCompilerResource) -> Self {
let mod_name = Str::rc(cfg.input.unescaped_file_stem());
let mod_name = Str::from(cfg.input.file_stem());
Self::new_with_cache(cfg, mod_name, shared)
}
fn build(&mut self, src: String, mode: &str) -> Result<CompleteArtifact, IncompleteArtifact> {

View file

@ -55,7 +55,7 @@ impl Context {
/// Get the context of the module. If it was in analysis, wait until analysis is complete and join the thread.
/// If you only want to know if the module is registered, use `mod_registered`.
pub(crate) fn get_mod_with_path(&self, path: &Path) -> Option<&Context> {
if self.module_path() == Some(path) {
if self.module_path() == path {
return self.get_module();
}
if self.shared.is_some()

View file

@ -976,12 +976,12 @@ impl Context {
)
}
pub(crate) fn module_path(&self) -> Option<&Path> {
pub(crate) fn module_path(&self) -> &Path {
self.cfg.input.path()
}
pub(crate) fn absolutize(&self, loc: Location) -> AbsLocation {
AbsLocation::new(self.module_path().map(PathBuf::from), loc)
AbsLocation::new(Some(PathBuf::from(self.module_path())), loc)
}
#[inline]
@ -1049,7 +1049,7 @@ impl Context {
pub(crate) fn _get_module_from_stack(&self, path: &Path) -> Option<&Context> {
self.get_outer().and_then(|outer| {
if outer.kind == ContextKind::Module && outer.module_path() == Some(path) {
if outer.kind == ContextKind::Module && outer.module_path() == path {
Some(outer)
} else {
outer._get_module_from_stack(path)

View file

@ -1854,10 +1854,9 @@ impl Context {
if ERG_MODE {
self.check_mod_vis(path.as_path(), __name__, loc)?;
}
if let Some(referrer) = self.cfg.input.path() {
if self.shared().graph.inc_ref(referrer, path.clone()).is_err() {
self.build_cyclic_mod(&path);
}
let referrer = self.cfg.input.path();
if self.shared().graph.inc_ref(referrer, path.clone()).is_err() {
self.build_cyclic_mod(&path);
}
self.build_erg_mod(path, __name__, loc)
}
@ -2123,7 +2122,7 @@ impl Context {
fn try_gen_py_decl_file(&self, __name__: &Str) -> Result<PathBuf, ()> {
if let Ok(path) = self.cfg.input.resolve_py(Path::new(&__name__[..])) {
if self.cfg.input.path() == Some(path.as_path()) {
if self.cfg.input.path() == path.as_path() {
return Ok(path);
}
let (out, err) = if self.cfg.mode == ErgMode::LanguageServer || self.cfg.quiet_repl {
@ -2163,13 +2162,12 @@ impl Context {
let py_mod_cache = self.py_mod_cache();
let path = self.get_decl_path(__name__, loc)?;
// module itself
if self.cfg.input.path() == Some(path.as_path()) {
if self.cfg.input.path() == path.as_path() {
return Ok(path);
}
if let Some(referrer) = self.cfg.input.path() {
if self.shared().graph.inc_ref(referrer, path.clone()).is_err() {
self.build_cyclic_mod(&path);
}
let referrer = self.cfg.input.path();
if self.shared().graph.inc_ref(referrer, path.clone()).is_err() {
self.build_cyclic_mod(&path);
}
if py_mod_cache.get(&path).is_some() {
return Ok(path);

View file

@ -383,7 +383,7 @@ impl<'a> HIRLinker<'a> {
// ↓
// # module.er
// self = __import__(__name__)
if matches!((path.canonicalize(), self.cfg.input.unescaped_path().canonicalize()), (Ok(l), Ok(r)) if l == r)
if matches!((path.canonicalize(), self.cfg.input.path().canonicalize()), (Ok(l), Ok(r)) if l == r)
{
*expr = Self::self_module();
return;

View file

@ -173,7 +173,7 @@ impl ASTLowerer {
}
let self_path = self.module.context.module_path();
for (referee, value) in self.module.context.index().members().iter() {
if referee.module.as_deref() != self_path {
if referee.module.as_deref() != Some(self_path) {
continue;
}
let name_is_auto = &value.name[..] == "_"

View file

@ -2490,10 +2490,9 @@ impl ASTLowerer {
pub fn lower(&mut self, ast: AST, mode: &str) -> Result<CompleteArtifact, IncompleteArtifact> {
log!(info "the AST lowering process has started.");
log!(info "the type-checking process has started.");
if let Some(path) = self.cfg.input.path() {
let graph = &self.module.context.shared().graph;
graph.add_node_if_none(path);
}
let path = self.cfg.input.path();
let graph = &self.module.context.shared().graph;
graph.add_node_if_none(path);
let ast = ASTLinker::new(self.cfg.clone())
.link(ast, mode)
.map_err(|errs| {

View file

@ -39,9 +39,7 @@ impl SharedCompilerResource {
trait_impls: SharedTraitImpls::new(),
promises: SharedPromises::new(
graph,
cfg.input
.path()
.map_or(PathBuf::default(), |p| p.canonicalize().unwrap_or_default()),
cfg.input.path().canonicalize().unwrap_or_default(),
),
errors: SharedCompileErrors::new(),
warns: SharedCompileWarnings::new(),

View file

@ -3,8 +3,10 @@ use std::fmt::Write as _;
use std::fs::File;
use std::io::{BufReader, Read, Write as _};
use std::path::Path;
use std::process::ExitStatus;
use erg_common::impl_display_from_debug;
use erg_common::io::Output;
#[allow(unused_imports)]
use erg_common::log;
use erg_common::opcode::CommonOpcode;
@ -12,7 +14,7 @@ use erg_common::opcode308::Opcode308;
use erg_common::opcode309::Opcode309;
use erg_common::opcode310::Opcode310;
use erg_common::opcode311::{BinOpCode, Opcode311};
use erg_common::python_util::{env_magic_number, PythonVersion};
use erg_common::python_util::{env_magic_number, exec_py_code, PythonVersion};
use erg_common::serialize::*;
use erg_common::Str;
@ -439,6 +441,22 @@ impl CodeObj {
Ok(())
}
pub fn executable_code(self, py_magic_num: Option<u32>) -> String {
let mut bytes = Vec::with_capacity(16);
let py_magic_num = py_magic_num.unwrap_or_else(env_magic_number);
let python_ver = get_ver_from_magic_num(py_magic_num);
bytes.append(&mut self.into_bytes(python_ver));
let mut bytecode = "".to_string();
for b in bytes {
write!(bytecode, "\\x{b:0>2x}").unwrap();
}
format!("import marshal; exec(marshal.loads(b'{bytecode}'))")
}
pub fn exec(self, py_magic_num: Option<u32>, output: Output) -> std::io::Result<ExitStatus> {
exec_py_code(&self.executable_code(py_magic_num), output)
}
fn tables_info(&self) -> String {
let mut tables = "".to_string();
if !self.consts.is_empty() {

View file

@ -111,10 +111,7 @@ impl Deserializer {
}
pub fn run(cfg: ErgConfig) -> ExitStatus {
let Some(filename) = cfg.input.path() else {
eprintln!("{:?} is not a filename", cfg.input);
return ExitStatus::ERR1;
};
let filename = cfg.input.path();
match CodeObj::from_pyc(filename) {
Ok(codeobj) => {
println!("{}", codeobj.code_info(None));

View file

@ -64,7 +64,7 @@ impl ASTBuilder {
CompleteArtifact<AST, ParserRunnerErrors>,
IncompleteArtifact<AST, ParserRunnerErrors>,
> {
let name = Str::rc(self.runner.cfg().input.unescaped_filename());
let name = Str::from(self.runner.cfg().input.filename());
let mut desugarer = Desugarer::new();
let artifact = self.runner.parse(src).map_err(|iart| {
iart.map_mod(|module| {
@ -87,7 +87,7 @@ impl ASTBuilder {
CompleteArtifact<AST, ParserRunnerErrors>,
IncompleteArtifact<AST, ParserRunnerErrors>,
> {
let name = Str::rc(self.runner.cfg().input.unescaped_filename());
let name = Str::from(self.runner.cfg().input.filename());
let artifact = self
.runner
.parse(src)

View file

@ -7,7 +7,7 @@ use std::time::Duration;
use erg_common::config::ErgConfig;
use erg_common::error::MultiErrorDisplay;
use erg_common::python_util::{exec_pyc, spawn_py};
use erg_common::python_util::spawn_py;
use erg_common::traits::{ExitStatus, Runnable, Stream};
use erg_compiler::hir::Expr;
@ -33,6 +33,8 @@ enum Inst {
Initialize = 0x04,
/// Informs that the connection is to be / should be terminated.
Exit = 0x05,
/// Send from client to server. Let the server to execute the code.
Execute = 0x06,
/// Informs that it is not a supported instruction.
Unknown = 0x00,
}
@ -45,6 +47,7 @@ impl From<u8> for Inst {
0x03 => Inst::Exception,
0x04 => Inst::Initialize,
0x05 => Inst::Exit,
0x06 => Inst::Execute,
_ => Inst::Unknown,
}
}
@ -280,34 +283,26 @@ impl Runnable for DummyVM {
}
fn exec(&mut self) -> Result<ExitStatus, Self::Errs> {
// Parallel execution is not possible without dumping with a unique file name.
let filename = self.cfg().dump_pyc_filename();
let src = self.cfg_mut().input.read();
let warns = self
.compiler
.compile_and_dump_as_pyc(&filename, src, "exec")
.map_err(|eart| {
eart.warns.write_all_to(&mut self.cfg_mut().output);
eart.errors
})?;
warns.write_all_to(&mut self.cfg_mut().output);
let code = exec_pyc(
&filename,
self.cfg().py_command,
&self.cfg().runtime_args,
self.cfg().output.clone(),
);
remove_file(&filename).unwrap();
Ok(ExitStatus::new(code.unwrap_or(1), warns.len(), 0))
let art = self.compiler.compile(src, "exec").map_err(|eart| {
eart.warns.write_all_to(&mut self.cfg_mut().output);
eart.errors
})?;
art.warns.write_all_to(&mut self.cfg_mut().output);
let stat = art
.object
.exec(self.cfg().py_magic_num, self.cfg().output.clone())
.expect("failed to execute");
let stat = ExitStatus::new(stat.code().unwrap_or(0), art.warns.len(), 0);
Ok(stat)
}
fn eval(&mut self, src: String) -> Result<String, EvalErrors> {
let path = self.cfg().dump_pyc_filename();
let arti = self
.compiler
.eval_compile_and_dump_as_pyc(path, src, "eval")
.eval_compile(src, "eval")
.map_err(|eart| eart.errors)?;
let (last, warns) = (arti.object, arti.warns);
let ((code, last), warns) = (arti.object, arti.warns);
let mut res = warns.to_string();
macro_rules! err_handle {
@ -323,12 +318,13 @@ impl Runnable for DummyVM {
}
// Tell the REPL server to execute the code
if let Err(err) = self
.stream
.as_mut()
.unwrap()
.send_msg(&Message::new(Inst::Load, None))
{
if let Err(err) = self.stream.as_mut().unwrap().send_msg(&Message::new(
Inst::Execute,
Some(
code.executable_code(self.compiler.cfg.py_magic_num)
.into_bytes(),
),
)) {
err_handle!("Sending error: {err}");
};
@ -349,7 +345,7 @@ impl Runnable for DummyVM {
Inst::Print => String::from_utf8(msg.data.unwrap_or_default()),
Inst::Exit => err_handle!("Receiving inst {:?} from server", msg.inst),
// `load` can only be sent from the client to the server
Inst::Load | Inst::Unknown => {
Inst::Load | Inst::Execute | Inst::Unknown => {
err_handle!("Receiving unexpected inst {:?} from server", msg.inst)
}
};

View file

@ -18,6 +18,8 @@ class INST:
INITIALIZE = 0x04
# Informs that the connection is to be / should be terminated.
EXIT = 0x05
# Send from client to server. Let the REPL server to execute the code.
EXECUTE = 0x06
class MessageStream:
def __init__(self, socket):
@ -31,8 +33,7 @@ class MessageStream:
data_len = int.from_bytes(self._read_buf[1:3], 'big')
self._read_buf.extend(self.socket.recv(data_len))
return (inst, self._read_buf[3:].decode())
return (inst, self._read_buf[3:].decode('utf-8'))
def send_msg(self, inst, data=''):
data_bytes = data.encode()
@ -57,7 +58,7 @@ client_stream = MessageStream(client_socket)
while True:
try:
inst, _ = client_stream.recv_msg()
inst, data = client_stream.recv_msg()
except ConnectionResetError: # when the server was crashed
break
if inst == INST.EXIT: # when the server was closed successfully
@ -93,6 +94,31 @@ while True:
res = out + exc + res
buf.append(res)
client_stream.send_msg(resp_inst, ''.join(buf))
elif inst == INST.EXECUTE:
sys.stdout = io.StringIO()
res = ''
exc = ''
resp_inst = INST.PRINT
buf = []
try:
res = str(exec(data, ctx))
except SystemExit:
client_stream.send_msg(INST.EXCEPTION, 'SystemExit')
continue
except Exception as e:
try:
excs = traceback.format_exception(e)
except:
excs = traceback.format_exception_only(e.__class__, e)
exc = ''.join(excs).rstrip()
traceback.clear_frames(e.__traceback__)
resp_inst = INST.INITIALIZE
out = sys.stdout.getvalue()[:-1]
if out and exc or res:
out += '\n'
res = out + exc + res
buf.append(res)
client_stream.send_msg(resp_inst, ''.join(buf))
else:
client_stream.send_msg(INST.UNKNOWN)