From bc6d401aecce5dec992afeff3ed9a7d9fd04eed0 Mon Sep 17 00:00:00 2001 From: Shunsuke Shibayama Date: Fri, 18 Nov 2022 11:02:30 +0900 Subject: [PATCH 1/7] Add transpiler --- compiler/erg_compiler/lib.rs | 1 + compiler/erg_compiler/main.rs | 4 + compiler/erg_compiler/transpile.rs | 233 +++++++++++++++++++++++++++++ src/main.rs | 4 + 4 files changed, 242 insertions(+) create mode 100644 compiler/erg_compiler/transpile.rs diff --git a/compiler/erg_compiler/lib.rs b/compiler/erg_compiler/lib.rs index d3013842..f8629da8 100644 --- a/compiler/erg_compiler/lib.rs +++ b/compiler/erg_compiler/lib.rs @@ -21,5 +21,6 @@ pub mod mod_cache; pub mod optimize; pub mod ownercheck; pub mod reorder; +pub mod transpile; pub mod ty; pub mod varinfo; diff --git a/compiler/erg_compiler/main.rs b/compiler/erg_compiler/main.rs index 66cd9105..cfb8190f 100644 --- a/compiler/erg_compiler/main.rs +++ b/compiler/erg_compiler/main.rs @@ -10,6 +10,7 @@ use erg_common::traits::Runnable; use erg_compiler::build_hir::HIRBuilder; use erg_compiler::lower::ASTLowerer; +use erg_compiler::transpile::Transpiler; use erg_compiler::ty::deserialize::Deserializer; use erg_compiler::Compiler; @@ -31,6 +32,9 @@ fn run() { "check" => { HIRBuilder::run(cfg); } + "transpile" => { + Transpiler::run(cfg); + } "compile" | "exec" => { Compiler::run(cfg); } diff --git a/compiler/erg_compiler/transpile.rs b/compiler/erg_compiler/transpile.rs new file mode 100644 index 00000000..80144636 --- /dev/null +++ b/compiler/erg_compiler/transpile.rs @@ -0,0 +1,233 @@ +use std::fs::File; +use std::io::Write; + +use erg_common::config::ErgConfig; +use erg_common::log; +use erg_common::traits::{Runnable, Stream}; +use erg_common::Str; + +use erg_parser::ast::ParamPattern; + +use crate::build_hir::HIRBuilder; +use crate::desugar_hir::HIRDesugarer; +use crate::error::{CompileError, CompileErrors}; +use crate::hir::{Accessor, Block, Call, Expr, Identifier, Params, Signature, HIR}; +use crate::link::Linker; +use crate::mod_cache::SharedModuleCache; + +#[derive(Debug, Clone)] +pub struct PyScript { + pub filename: Str, + pub code: String, +} + +/// Generates a `PyScript` from an String or other File inputs. +#[derive(Debug, Default)] +pub struct Transpiler { + pub cfg: ErgConfig, + builder: HIRBuilder, + mod_cache: SharedModuleCache, + script_generator: ScriptGenerator, +} + +impl Runnable for Transpiler { + type Err = CompileError; + type Errs = CompileErrors; + const NAME: &'static str = "Erg transpiler"; + + fn new(cfg: ErgConfig) -> Self { + let mod_cache = SharedModuleCache::new(); + let py_mod_cache = SharedModuleCache::new(); + Self { + builder: HIRBuilder::new_with_cache( + cfg.copy(), + "", + mod_cache.clone(), + py_mod_cache, + ), + script_generator: ScriptGenerator::new(), + mod_cache, + cfg, + } + } + + #[inline] + fn cfg(&self) -> &ErgConfig { + &self.cfg + } + + #[inline] + fn finish(&mut self) {} + + fn clear(&mut self) { + // self.builder.clear(); + } + + fn exec(&mut self) -> Result { + let path = self.input().filename().replace(".er", ".py"); + let script = self.transpile(self.input().read(), "exec")?; + let mut f = File::create(&path).unwrap(); + f.write_all(script.code.as_bytes()).unwrap(); + Ok(0) + } + + fn eval(&mut self, src: String) -> Result { + let script = self.transpile(src, "eval")?; + Ok(script.code) + } +} + +impl Transpiler { + pub fn transpile(&mut self, src: String, mode: &str) -> Result { + log!(info "the transpiling process has started."); + let hir = self.build_link_desugar(src, mode)?; + let script = self.script_generator.transpile(hir); + log!(info "code:\n{}", script.code); + log!(info "the transpiling process has completed"); + Ok(script) + } + + fn build_link_desugar(&mut self, src: String, mode: &str) -> Result { + let artifact = self + .builder + .build(src, mode) + .map_err(|artifact| artifact.errors)?; + let linker = Linker::new(&self.cfg, &self.mod_cache); + let hir = linker.link(artifact.hir); + Ok(HIRDesugarer::desugar(hir)) + } +} + +#[derive(Debug, Default)] +pub struct ScriptGenerator { + level: usize, +} + +impl ScriptGenerator { + pub const fn new() -> Self { + Self { level: 0 } + } + + pub fn transpile(&mut self, hir: HIR) -> PyScript { + let mut code = String::new(); + for chunk in hir.module.into_iter() { + code += &self.transpile_expr(chunk); + code.push('\n'); + } + PyScript { + filename: hir.name, + code, + } + } + + pub fn transpile_expr(&mut self, expr: Expr) -> String { + match expr { + Expr::Lit(lit) => lit.token.content.to_string(), + Expr::Call(call) => self.transpile_call(call), + Expr::Accessor(acc) => match acc { + Accessor::Ident(ident) => Self::transpile_ident(ident), + Accessor::Attr(attr) => { + format!( + "{}.{}", + self.transpile_expr(*attr.obj), + Self::transpile_ident(attr.ident) + ) + } + }, + Expr::Def(mut def) => match def.sig { + Signature::Var(var) => { + let mut code = format!("{} = ", Self::transpile_ident(var.ident)); + if def.body.block.len() > 1 { + todo!("transpile instant blocks") + } + let expr = def.body.block.remove(0); + code += &self.transpile_expr(expr); + code + } + Signature::Subr(subr) => { + let mut code = format!( + "def {}({}):\n", + Self::transpile_ident(subr.ident), + self.transpile_params(subr.params) + ); + code += &self.transpile_block(def.body.block); + code + } + }, + other => todo!("transpile {other}"), + } + } + + fn transpile_call(&mut self, mut call: Call) -> String { + match call.obj.local_name() { + Some("assert") => { + let mut code = format!("assert {}", self.transpile_expr(call.args.remove(0))); + if let Some(msg) = call.args.try_remove(0) { + code += &format!(", {}", self.transpile_expr(msg)); + } + code + } + _ => self.transpile_simple_call(call), + } + } + + fn transpile_simple_call(&mut self, mut call: Call) -> String { + let mut code = self.transpile_expr(*call.obj); + code.push('('); + while let Some(arg) = call.args.try_remove_pos(0) { + code += &self.transpile_expr(arg.expr); + code.push(','); + } + while let Some(arg) = call.args.try_remove_kw(0) { + code += &format!("{}={},", arg.keyword, self.transpile_expr(arg.expr)); + } + code.push(')'); + code + } + + fn transpile_ident(ident: Identifier) -> String { + if let Some(py_name) = ident.vi.py_name { + py_name.to_string() + } else if ident.dot.is_some() { + ident.name.into_token().content.to_string() + } else { + let name = ident.name.into_token().content; + let name = name.replace('!', "__erg_proc__"); + let name = name.replace('$', "__erg_shared__"); + format!("__{name}") + } + } + + fn transpile_params(&mut self, params: Params) -> String { + let mut code = String::new(); + for non_default in params.non_defaults { + let ParamPattern::VarName(param) = non_default.pat else { todo!() }; + code += &format!("{},", param.into_token().content); + } + for default in params.defaults { + let ParamPattern::VarName(param) = default.sig.pat else { todo!() }; + code += &format!( + "{}={},", + param.into_token().content, + self.transpile_expr(default.default_val) + ); + } + code + } + + fn transpile_block(&mut self, block: Block) -> String { + self.level += 1; + let mut code = String::new(); + let last = block.len() - 1; + for (i, chunk) in block.into_iter().enumerate() { + code += &" ".repeat(self.level); + if i == last { + code += "return "; + } + code += &self.transpile_expr(chunk); + code.push('\n'); + } + self.level -= 1; + code + } +} diff --git a/src/main.rs b/src/main.rs index 1d7a226e..363f2201 100644 --- a/src/main.rs +++ b/src/main.rs @@ -14,6 +14,7 @@ use erg_parser::ParserRunner; use erg_compiler::build_hir::HIRBuilder; use erg_compiler::lower::ASTLowerer; +use erg_compiler::transpile::Transpiler; use erg_compiler::ty::deserialize::Deserializer; use erg_compiler::Compiler; @@ -37,6 +38,9 @@ fn run() { "compile" => { Compiler::run(cfg); } + "transpile" => { + Transpiler::run(cfg); + } "exec" => { DummyVM::run(cfg); } From 77ff1b44e8694d7351c59d3b6c0c3b3f4e5dbc31 Mon Sep 17 00:00:00 2001 From: Shunsuke Shibayama Date: Fri, 18 Nov 2022 11:07:43 +0900 Subject: [PATCH 2/7] Implement binary/unary transpiling --- compiler/erg_compiler/transpile.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/compiler/erg_compiler/transpile.rs b/compiler/erg_compiler/transpile.rs index 80144636..3e53bdff 100644 --- a/compiler/erg_compiler/transpile.rs +++ b/compiler/erg_compiler/transpile.rs @@ -124,6 +124,17 @@ impl ScriptGenerator { match expr { Expr::Lit(lit) => lit.token.content.to_string(), Expr::Call(call) => self.transpile_call(call), + Expr::BinOp(bin) => { + let mut code = self.transpile_expr(*bin.lhs); + code += &bin.op.content; + code += &self.transpile_expr(*bin.rhs); + code + } + Expr::UnaryOp(unary) => { + let mut code = unary.op.content.to_string(); + code += &self.transpile_expr(*unary.expr); + code + } Expr::Accessor(acc) => match acc { Accessor::Ident(ident) => Self::transpile_ident(ident), Accessor::Attr(attr) => { From f5c69ea037d74547c8d5cbf23372385ef8c0b3f8 Mon Sep 17 00:00:00 2001 From: Shunsuke Shibayama Date: Fri, 18 Nov 2022 11:11:12 +0900 Subject: [PATCH 3/7] Implement method transpiling --- compiler/erg_compiler/transpile.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/compiler/erg_compiler/transpile.rs b/compiler/erg_compiler/transpile.rs index 3e53bdff..cc79f974 100644 --- a/compiler/erg_compiler/transpile.rs +++ b/compiler/erg_compiler/transpile.rs @@ -139,7 +139,7 @@ impl ScriptGenerator { Accessor::Ident(ident) => Self::transpile_ident(ident), Accessor::Attr(attr) => { format!( - "{}.{}", + "({}).{}", self.transpile_expr(*attr.obj), Self::transpile_ident(attr.ident) ) @@ -183,7 +183,10 @@ impl ScriptGenerator { } fn transpile_simple_call(&mut self, mut call: Call) -> String { - let mut code = self.transpile_expr(*call.obj); + let mut code = format!("({})", self.transpile_expr(*call.obj)); + if let Some(attr) = call.attr_name { + code += &format!(".{}", Self::transpile_ident(attr)); + } code.push('('); while let Some(arg) = call.args.try_remove_pos(0) { code += &self.transpile_expr(arg.expr); From ca3612e9b35ca4528bdef66a7c3b1f957f24e738 Mon Sep 17 00:00:00 2001 From: Shunsuke Shibayama Date: Fri, 18 Nov 2022 11:14:11 +0900 Subject: [PATCH 4/7] Update transpile.rs --- compiler/erg_compiler/transpile.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compiler/erg_compiler/transpile.rs b/compiler/erg_compiler/transpile.rs index cc79f974..765e28e9 100644 --- a/compiler/erg_compiler/transpile.rs +++ b/compiler/erg_compiler/transpile.rs @@ -216,12 +216,12 @@ impl ScriptGenerator { let mut code = String::new(); for non_default in params.non_defaults { let ParamPattern::VarName(param) = non_default.pat else { todo!() }; - code += &format!("{},", param.into_token().content); + code += &format!("__{},", param.into_token().content); } for default in params.defaults { let ParamPattern::VarName(param) = default.sig.pat else { todo!() }; code += &format!( - "{}={},", + "__{}={},", param.into_token().content, self.transpile_expr(default.default_val) ); From 7d38836debccb37c792c8f30f652500304525ea8 Mon Sep 17 00:00:00 2001 From: Shunsuke Shibayama Date: Fri, 18 Nov 2022 11:45:40 +0900 Subject: [PATCH 5/7] Implement container transpiling --- compiler/erg_compiler/transpile.rs | 51 +++++++++++++++++++++++++++++- 1 file changed, 50 insertions(+), 1 deletion(-) diff --git a/compiler/erg_compiler/transpile.rs b/compiler/erg_compiler/transpile.rs index 765e28e9..43a04d2e 100644 --- a/compiler/erg_compiler/transpile.rs +++ b/compiler/erg_compiler/transpile.rs @@ -11,7 +11,9 @@ use erg_parser::ast::ParamPattern; use crate::build_hir::HIRBuilder; use crate::desugar_hir::HIRDesugarer; use crate::error::{CompileError, CompileErrors}; -use crate::hir::{Accessor, Block, Call, Expr, Identifier, Params, Signature, HIR}; +use crate::hir::{ + Accessor, Array, Block, Call, Dict, Expr, Identifier, Params, Set, Signature, Tuple, HIR, +}; use crate::link::Linker; use crate::mod_cache::SharedModuleCache; @@ -135,6 +137,53 @@ impl ScriptGenerator { code += &self.transpile_expr(*unary.expr); code } + Expr::Array(array) => match array { + Array::Normal(arr) => { + let mut code = "[".to_string(); + for elem in arr.elems.pos_args { + code += &format!("{},", self.transpile_expr(elem.expr)); + } + code += "]"; + code + } + other => todo!("transpiling {other}"), + }, + Expr::Set(set) => match set { + Set::Normal(st) => { + let mut code = "{".to_string(); + for elem in st.elems.pos_args { + code += &format!("{},", self.transpile_expr(elem.expr)); + } + code += "}"; + code + } + other => todo!("transpiling {other}"), + }, + Expr::Tuple(tuple) => match tuple { + Tuple::Normal(tup) => { + let mut code = "(".to_string(); + for elem in tup.elems.pos_args { + code += &format!("{},", self.transpile_expr(elem.expr)); + } + code += ")"; + code + } + }, + Expr::Dict(dict) => match dict { + Dict::Normal(dic) => { + let mut code = "{".to_string(); + for kv in dic.kvs { + code += &format!( + "({}): ({}),", + self.transpile_expr(kv.key), + self.transpile_expr(kv.value) + ); + } + code += "}"; + code + } + other => todo!("transpiling {other}"), + }, Expr::Accessor(acc) => match acc { Accessor::Ident(ident) => Self::transpile_ident(ident), Accessor::Attr(attr) => { From 6e5191380b143c8a07ca27425f7432d68b467ff8 Mon Sep 17 00:00:00 2001 From: Shunsuke Shibayama Date: Fri, 18 Nov 2022 14:43:25 +0900 Subject: [PATCH 6/7] Implement Record & ClassDef --- compiler/erg_compiler/transpile.rs | 111 +++++++++++++++++++++-------- 1 file changed, 82 insertions(+), 29 deletions(-) diff --git a/compiler/erg_compiler/transpile.rs b/compiler/erg_compiler/transpile.rs index 43a04d2e..45f90838 100644 --- a/compiler/erg_compiler/transpile.rs +++ b/compiler/erg_compiler/transpile.rs @@ -12,10 +12,12 @@ use crate::build_hir::HIRBuilder; use crate::desugar_hir::HIRDesugarer; use crate::error::{CompileError, CompileErrors}; use crate::hir::{ - Accessor, Array, Block, Call, Dict, Expr, Identifier, Params, Set, Signature, Tuple, HIR, + Accessor, Array, Block, Call, ClassDef, Def, Dict, Expr, Identifier, Params, Set, Signature, + Tuple, HIR, }; use crate::link::Linker; use crate::mod_cache::SharedModuleCache; +use crate::ty::Type; #[derive(Debug, Clone)] pub struct PyScript { @@ -111,7 +113,7 @@ impl ScriptGenerator { } pub fn transpile(&mut self, hir: HIR) -> PyScript { - let mut code = String::new(); + let mut code = self.load_prelude(); for chunk in hir.module.into_iter() { code += &self.transpile_expr(chunk); code.push('\n'); @@ -122,7 +124,11 @@ impl ScriptGenerator { } } - pub fn transpile_expr(&mut self, expr: Expr) -> String { + fn load_prelude(&mut self) -> String { + "from collections import namedtuple as NamedTuple__\n".to_string() + } + + fn transpile_expr(&mut self, expr: Expr) -> String { match expr { Expr::Lit(lit) => lit.token.content.to_string(), Expr::Call(call) => self.transpile_call(call), @@ -159,6 +165,20 @@ impl ScriptGenerator { } other => todo!("transpiling {other}"), }, + Expr::Record(rec) => { + let mut attrs = "[".to_string(); + let mut values = "(".to_string(); + for mut attr in rec.attrs.into_iter() { + attrs += &format!("'{}',", Self::transpile_ident(attr.sig.into_ident())); + if attr.body.block.len() > 1 { + todo!("transpile instant blocks") + } + values += &format!("{},", self.transpile_expr(attr.body.block.remove(0))); + } + attrs += "]"; + values += ")"; + format!("NamedTuple__('Record', {attrs}){values}") + } Expr::Tuple(tuple) => match tuple { Tuple::Normal(tup) => { let mut code = "(".to_string(); @@ -194,26 +214,8 @@ impl ScriptGenerator { ) } }, - Expr::Def(mut def) => match def.sig { - Signature::Var(var) => { - let mut code = format!("{} = ", Self::transpile_ident(var.ident)); - if def.body.block.len() > 1 { - todo!("transpile instant blocks") - } - let expr = def.body.block.remove(0); - code += &self.transpile_expr(expr); - code - } - Signature::Subr(subr) => { - let mut code = format!( - "def {}({}):\n", - Self::transpile_ident(subr.ident), - self.transpile_params(subr.params) - ); - code += &self.transpile_block(def.body.block); - code - } - }, + Expr::Def(def) => self.transpile_def(def), + Expr::ClassDef(classdef) => self.transpile_classdef(classdef), other => todo!("transpile {other}"), } } @@ -256,8 +258,8 @@ impl ScriptGenerator { } else { let name = ident.name.into_token().content; let name = name.replace('!', "__erg_proc__"); - let name = name.replace('$', "__erg_shared__"); - format!("__{name}") + let name = name.replace('$', "erg_shared__"); + format!("{name}__") } } @@ -265,12 +267,12 @@ impl ScriptGenerator { let mut code = String::new(); for non_default in params.non_defaults { let ParamPattern::VarName(param) = non_default.pat else { todo!() }; - code += &format!("__{},", param.into_token().content); + code += &format!("{}__,", param.into_token().content); } for default in params.defaults { let ParamPattern::VarName(param) = default.sig.pat else { todo!() }; code += &format!( - "__{}={},", + "{}__ = {},", param.into_token().content, self.transpile_expr(default.default_val) ); @@ -278,13 +280,13 @@ impl ScriptGenerator { code } - fn transpile_block(&mut self, block: Block) -> String { + fn transpile_block(&mut self, block: Block, is_statement: bool) -> String { self.level += 1; let mut code = String::new(); let last = block.len() - 1; for (i, chunk) in block.into_iter().enumerate() { code += &" ".repeat(self.level); - if i == last { + if i == last && !is_statement { code += "return "; } code += &self.transpile_expr(chunk); @@ -293,4 +295,55 @@ impl ScriptGenerator { self.level -= 1; code } + + fn transpile_def(&mut self, mut def: Def) -> String { + match def.sig { + Signature::Var(var) => { + let mut code = format!("{} = ", Self::transpile_ident(var.ident)); + if def.body.block.len() > 1 { + todo!("transpile instant blocks") + } + let expr = def.body.block.remove(0); + code += &self.transpile_expr(expr); + code + } + Signature::Subr(subr) => { + let mut code = format!( + "def {}({}):\n", + Self::transpile_ident(subr.ident), + self.transpile_params(subr.params) + ); + code += &self.transpile_block(def.body.block, false); + code + } + } + } + + fn transpile_classdef(&mut self, classdef: ClassDef) -> String { + let class_name = Self::transpile_ident(classdef.sig.into_ident()); + let mut code = format!("class {class_name}():\n"); + let mut init_method = format!( + "{}def __init__(self, param__):\n", + " ".repeat(self.level + 1) + ); + match classdef.__new__.non_default_params().unwrap()[0].typ() { + Type::Record(rec) => { + for field in rec.keys() { + init_method += &format!( + "{}self.{} = param__.{}\n", + " ".repeat(self.level + 2), + field.symbol, + field.symbol + ); + } + } + other => todo!("{other}"), + } + code += &init_method; + if classdef.need_to_gen_new { + code += &format!("def new(x): return {class_name}.__call__(x)\n"); + } + code += &self.transpile_block(classdef.methods, true); + code + } } From d7ea38eccb30d5e22cd7e72f6b195aa8c8bcdb3e Mon Sep 17 00:00:00 2001 From: Shunsuke Shibayama Date: Fri, 18 Nov 2022 14:54:23 +0900 Subject: [PATCH 7/7] Add AttrDef & Compound --- compiler/erg_compiler/transpile.rs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/compiler/erg_compiler/transpile.rs b/compiler/erg_compiler/transpile.rs index 45f90838..09333582 100644 --- a/compiler/erg_compiler/transpile.rs +++ b/compiler/erg_compiler/transpile.rs @@ -216,6 +216,24 @@ impl ScriptGenerator { }, Expr::Def(def) => self.transpile_def(def), Expr::ClassDef(classdef) => self.transpile_classdef(classdef), + Expr::AttrDef(mut adef) => { + let mut code = format!("{} = ", self.transpile_expr(Expr::Accessor(adef.attr))); + if adef.block.len() > 1 { + todo!("transpile instant blocks") + } + let expr = adef.block.remove(0); + code += &self.transpile_expr(expr); + code + } + // TODO: + Expr::Compound(comp) => { + let mut code = "".to_string(); + for expr in comp.into_iter() { + code += &self.transpile_expr(expr); + code += &format!("\n{}", " ".repeat(self.level)); + } + code + } other => todo!("transpile {other}"), } }