Merge branch 'main' into dict

This commit is contained in:
Shunsuke Shibayama 2022-10-10 13:57:08 +09:00
commit c784ba261e
53 changed files with 1336 additions and 818 deletions

View file

@ -1,22 +1,22 @@
[package]
name = "erg_parser"
version = "0.5.7"
description = "The Erg parser"
authors = ["erg-lang team <moderation.erglang@gmail.com>"]
license = "MIT OR Apache-2.0"
edition = "2021"
repository = "https://github.com/erg-lang/erg/tree/main/src/erg_compiler/erg_parser"
documentation = "https://docs.rs/erg_parser"
homepage = "https://erg-lang.github.io/"
documentation = "http://docs.rs/erg_parser"
version.workspace = true
authors.workspace = true
license.workspace = true
edition.workspace = true
repository.workspace = true
homepage.workspace = true
[features]
debug = [ "erg_common/debug" ]
japanese = [ "erg_common/japanese" ]
simplified_chinese = [ "erg_common/simplified_chinese" ]
traditional_chinese = [ "erg_common/traditional_chinese" ]
debug = ["erg_common/debug"]
japanese = ["erg_common/japanese"]
simplified_chinese = ["erg_common/simplified_chinese"]
traditional_chinese = ["erg_common/traditional_chinese"]
[dependencies]
erg_common = { version = "0.5.7", path = "../erg_common" }
erg_common = { version = "0.5.9-nightly.0", path = "../erg_common" }
[lib]
path = "lib.rs"

View file

@ -11,7 +11,7 @@ use erg_common::{
fmt_option, fmt_vec, impl_display_for_enum, impl_display_for_single_struct,
impl_display_from_nested, impl_displayable_stream_for_wrapper, impl_locational,
impl_locational_for_enum, impl_nested_display_for_chunk_enum, impl_nested_display_for_enum,
impl_stream, impl_stream_for_wrapper,
impl_stream, impl_stream_for_wrapper, option_enum_unwrap,
};
use erg_common::{fmt_vec_split_with, Str};
@ -960,6 +960,32 @@ impl Call {
args,
}
}
pub fn is_match(&self) -> bool {
self.obj
.get_name()
.map(|s| &s[..] == "match")
.unwrap_or(false)
}
pub fn is_assert_cast(&self) -> bool {
self.obj
.get_name()
.map(|s| &s[..] == "assert")
.unwrap_or(false)
&& self
.args
.get_left_or_key("pred")
.map(|pred| pred.is_bin_in())
.unwrap_or(false)
}
pub fn assert_cast_target_type(&self) -> Option<&Expr> {
self.args
.get_left_or_key("pred")
.and_then(|pred| option_enum_unwrap!(pred, Expr::BinOp))
.map(|bin| bin.args[1].as_ref())
}
}
/// e.g. `Data::{x = 1; y = 2}`
@ -3132,7 +3158,11 @@ impl_locational_for_enum!(Expr; Lit, Accessor, Array, Tuple, Dict, Set, Record,
impl Expr {
pub fn is_match_call(&self) -> bool {
matches!(self, Expr::Call(Call{ obj, .. }) if obj.get_name().map(|s| &s[..] == "match").unwrap_or(false))
matches!(self, Expr::Call(call) if call.is_match())
}
pub fn is_bin_in(&self) -> bool {
matches!(self, Expr::BinOp(bin) if bin.op.is(TokenKind::InOp))
}
pub fn is_const_acc(&self) -> bool {

View file

@ -11,6 +11,18 @@ use erg_common::{impl_display_and_error, impl_stream_for_wrapper, switch_lang};
#[derive(Debug)]
pub struct LexError(ErrorCore);
impl From<ErrorCore> for LexError {
fn from(core: ErrorCore) -> Self {
Self(core)
}
}
impl From<LexError> for ErrorCore {
fn from(err: LexError) -> Self {
err.0
}
}
#[derive(Debug)]
pub struct LexErrors(Vec<LexError>);

View file

@ -461,15 +461,30 @@ impl Lexer /*<'a>*/ {
let mut num = mantissa;
debug_power_assert!(self.peek_cur_ch(), ==, Some('e'));
num.push(self.consume().unwrap()); // e
num.push(self.consume().unwrap()); // + | -
while let Some(cur) = self.peek_cur_ch() {
if cur.is_ascii_digit() || cur == '_' {
num.push(self.consume().unwrap());
} else {
break;
if self.peek_cur_ch().is_some() {
num.push(self.consume().unwrap()); // + | -
while let Some(cur) = self.peek_cur_ch() {
if cur.is_ascii_digit() || cur == '_' {
num.push(self.consume().unwrap());
} else {
break;
}
}
Ok(self.emit_token(RatioLit, &num))
} else {
let token = self.emit_token(RatioLit, &num);
Err(LexError::syntax_error(
0,
token.loc(),
switch_lang!(
"japanese" => format!("`{}`は無効な十進数リテラルです", &token.content),
"simplified_chinese" => format!("`{}`是无效的十进制字词", &token.content),
"traditional_chinese" => format!("`{}`是無效的十進製文字", &token.content),
"english" => format!("`{}` is invalid decimal literal", &token.content),
),
None,
))
}
Ok(self.emit_token(RatioLit, &num))
}
/// `_` will be removed at compiletime
@ -851,6 +866,10 @@ impl Iterator for Lexer /*<'a>*/ {
)))
}
}
Some('-') => {
self.consume();
self.accept(Inclusion, "<-")
}
Some('=') => {
self.consume();
self.accept(LessEq, "<=")

View file

@ -471,7 +471,8 @@ impl Parser {
}
}
}
Some(t) if t.is(LSqBr) => {
// x[...] (`x [...]` will interpreted as `x([...])`)
Some(t) if t.is(LSqBr) && acc.col_end().unwrap() == t.col_begin().unwrap() => {
self.skip();
let index = self
.try_reduce_expr(false, false, false)
@ -510,52 +511,6 @@ impl Parser {
Ok(acc)
}
fn validate_const_expr(&mut self, expr: Expr) -> ParseResult<ConstExpr> {
match expr {
Expr::Lit(l) => Ok(ConstExpr::Lit(l)),
Expr::Accessor(Accessor::Ident(local)) => {
let local = ConstLocal::new(local.name.into_token());
Ok(ConstExpr::Accessor(ConstAccessor::Local(local)))
}
Expr::Array(array) => match array {
Array::Normal(arr) => {
let (elems, _, _) = arr.elems.deconstruct();
let mut const_elems = vec![];
for elem in elems.into_iter() {
let const_expr = self.validate_const_expr(elem.expr)?;
const_elems.push(ConstPosArg::new(const_expr));
}
let elems = ConstArgs::new(const_elems, vec![], None);
let const_arr = ConstArray::new(arr.l_sqbr, arr.r_sqbr, elems, None);
Ok(ConstExpr::Array(const_arr))
}
other => {
self.errs.push(ParseError::feature_error(
line!() as usize,
other.loc(),
"???",
));
Err(())
}
},
// TODO: App, Record, BinOp, UnaryOp,
other => {
self.errs.push(ParseError::syntax_error(
0,
other.loc(),
switch_lang!(
"japanese" => "この式はコンパイル時計算できないため、型引数には使用できません",
"simplified_chinese" => "此表达式在编译时不可计算,因此不能用作类型参数",
"traditional_chinese" => "此表達式在編譯時不可計算,因此不能用作類型參數",
"english" => "this expression is not computable at the compile-time, so cannot used as a type-argument",
),
None,
));
Err(())
}
}
}
/// For parsing elements of arrays and tuples
fn try_reduce_elems(&mut self) -> ParseResult<ArrayInner> {
debug_call_info!(self);
@ -950,9 +905,7 @@ impl Parser {
}
}
let defs = RecordAttrs::from(defs);
let class = self
.convert_rhs_to_type_spec(class)
.map_err(|_| self.stack_dec())?;
let class = Self::expr_to_type_spec(class).map_err(|e| self.errs.push(e))?;
self.level -= 1;
Ok(Methods::new(class, vis, defs))
}
@ -1036,9 +989,7 @@ impl Parser {
let t_spec = self
.try_reduce_expr(false, false, false)
.map_err(|_| self.stack_dec())?;
let t_spec = self
.convert_rhs_to_type_spec(t_spec)
.map_err(|_| self.stack_dec())?;
let t_spec = Self::expr_to_type_spec(t_spec).map_err(|e| self.errs.push(e))?;
let expr = Expr::TypeAsc(TypeAscription::new(lhs, op, t_spec));
stack.push(ExprOrOp::Expr(expr));
}
@ -1280,9 +1231,7 @@ impl Parser {
let t_spec = self
.try_reduce_expr(false, in_type_args, in_brace)
.map_err(|_| self.stack_dec())?;
let t_spec = self
.convert_rhs_to_type_spec(t_spec)
.map_err(|_| self.stack_dec())?;
let t_spec = Self::expr_to_type_spec(t_spec).map_err(|e| self.errs.push(e))?;
let expr = Expr::TypeAsc(TypeAscription::new(lhs, op, t_spec));
stack.push(ExprOrOp::Expr(expr));
}
@ -2173,9 +2122,7 @@ impl Parser {
pack: DataPack,
) -> ParseResult<VarDataPackPattern> {
debug_call_info!(self);
let class = self
.convert_rhs_to_type_spec(*pack.class)
.map_err(|_| self.stack_dec())?;
let class = Self::expr_to_type_spec(*pack.class).map_err(|e| self.errs.push(e))?;
let args = self
.convert_record_to_record_pat(pack.args)
.map_err(|_| self.stack_dec())?;
@ -2463,7 +2410,7 @@ impl Parser {
fn convert_kw_arg_to_default_param(&mut self, arg: KwArg) -> ParseResult<ParamSignature> {
debug_call_info!(self);
let pat = ParamPattern::VarName(VarName::new(arg.keyword));
let expr = self.validate_const_expr(arg.expr)?;
let expr = Self::validate_const_expr(arg.expr).map_err(|e| self.errs.push(e))?;
let param = ParamSignature::new(pat, arg.t_spec, Some(expr));
self.level -= 1;
Ok(param)
@ -2672,95 +2619,51 @@ impl Parser {
TypeBoundSpecs::empty(),
))
}
}
fn convert_rhs_to_type_spec(&mut self, rhs: Expr) -> ParseResult<TypeSpec> {
debug_call_info!(self);
match rhs {
Expr::Accessor(acc) => {
let t_spec = self
.convert_accessor_to_type_spec(acc)
.map_err(|_| self.stack_dec())?;
self.level -= 1;
Ok(t_spec)
// The APIs defined below are also used by `ASTLowerer` to interpret expressions as types.
impl Parser {
fn validate_const_expr(expr: Expr) -> Result<ConstExpr, ParseError> {
match expr {
Expr::Lit(l) => Ok(ConstExpr::Lit(l)),
Expr::Accessor(Accessor::Ident(local)) => {
let local = ConstLocal::new(local.name.into_token());
Ok(ConstExpr::Accessor(ConstAccessor::Local(local)))
}
Expr::Call(call) => {
let predecl = self
.convert_call_to_predecl_type_spec(call)
.map_err(|_| self.stack_dec())?;
self.level -= 1;
Ok(TypeSpec::PreDeclTy(predecl))
}
Expr::Lambda(lambda) => {
let lambda = self
.convert_lambda_to_subr_type_spec(lambda)
.map_err(|_| self.stack_dec())?;
self.level -= 1;
Ok(TypeSpec::Subr(lambda))
}
Expr::Array(array) => {
let array = self
.convert_array_to_array_type_spec(array)
.map_err(|_| self.stack_dec())?;
self.level -= 1;
Ok(TypeSpec::Array(array))
}
Expr::Set(set) => {
let set = self
.convert_set_to_set_type_spec(set)
.map_err(|_| self.stack_dec())?;
self.level -= 1;
Ok(TypeSpec::Set(set))
}
Expr::BinOp(bin) => {
if bin.op.kind.is_range_op() {
let op = bin.op;
let mut args = bin.args.into_iter();
let lhs = self
.validate_const_expr(*args.next().unwrap())
.map_err(|_| self.stack_dec())?;
let rhs = self
.validate_const_expr(*args.next().unwrap())
.map_err(|_| self.stack_dec())?;
self.level -= 1;
Ok(TypeSpec::Interval { op, lhs, rhs })
} else if bin.op.kind == TokenKind::AndOp {
let mut args = bin.args.into_iter();
let lhs = self
.convert_rhs_to_type_spec(*args.next().unwrap())
.map_err(|_| self.stack_dec())?;
let rhs = self
.convert_rhs_to_type_spec(*args.next().unwrap())
.map_err(|_| self.stack_dec())?;
self.level -= 1;
Ok(TypeSpec::and(lhs, rhs))
} else if bin.op.kind == TokenKind::OrOp {
let mut args = bin.args.into_iter();
let lhs = self
.convert_rhs_to_type_spec(*args.next().unwrap())
.map_err(|_| self.stack_dec())?;
let rhs = self
.convert_rhs_to_type_spec(*args.next().unwrap())
.map_err(|_| self.stack_dec())?;
self.level -= 1;
Ok(TypeSpec::or(lhs, rhs))
} else {
self.level -= 1;
let err = ParseError::simple_syntax_error(line!() as usize, bin.loc());
self.errs.push(err);
Err(())
Expr::Array(array) => match array {
Array::Normal(arr) => {
let (elems, _, _) = arr.elems.deconstruct();
let mut const_elems = vec![];
for elem in elems.into_iter() {
let const_expr = Self::validate_const_expr(elem.expr)?;
const_elems.push(ConstPosArg::new(const_expr));
}
let elems = ConstArgs::new(const_elems, vec![], None);
let const_arr = ConstArray::new(arr.l_sqbr, arr.r_sqbr, elems, None);
Ok(ConstExpr::Array(const_arr))
}
}
other => {
self.level -= 1;
let err = ParseError::simple_syntax_error(line!() as usize, other.loc());
self.errs.push(err);
Err(())
}
other => Err(ParseError::feature_error(
line!() as usize,
other.loc(),
"???",
)),
},
// TODO: App, Record, BinOp, UnaryOp,
other => Err(ParseError::syntax_error(
line!() as usize,
other.loc(),
switch_lang!(
"japanese" => "この式はコンパイル時計算できないため、型引数には使用できません",
"simplified_chinese" => "此表达式在编译时不可计算,因此不能用作类型参数",
"traditional_chinese" => "此表達式在編譯時不可計算,因此不能用作類型參數",
"english" => "this expression is not computable at the compile-time, so cannot used as a type-argument",
),
None,
)),
}
}
fn convert_accessor_to_type_spec(&mut self, accessor: Accessor) -> ParseResult<TypeSpec> {
debug_call_info!(self);
fn accessor_to_type_spec(accessor: Accessor) -> Result<TypeSpec, ParseError> {
let t_spec = match accessor {
Accessor::Ident(ident) => {
let predecl =
@ -2768,32 +2671,22 @@ impl Parser {
TypeSpec::PreDeclTy(predecl)
}
Accessor::TypeApp(tapp) => {
let spec = self
.convert_rhs_to_type_spec(*tapp.obj)
.map_err(|_| self.stack_dec())?;
let spec = Self::expr_to_type_spec(*tapp.obj)?;
TypeSpec::type_app(spec, tapp.type_args)
}
other => {
self.level -= 1;
let err = ParseError::simple_syntax_error(line!() as usize, other.loc());
self.errs.push(err);
return Err(());
return Err(err);
}
};
self.level -= 1;
Ok(t_spec)
}
fn convert_call_to_predecl_type_spec(&mut self, _call: Call) -> ParseResult<PreDeclTypeSpec> {
debug_call_info!(self);
fn call_to_predecl_type_spec(_call: Call) -> Result<PreDeclTypeSpec, ParseError> {
todo!()
}
fn convert_lambda_to_subr_type_spec(
&mut self,
mut lambda: Lambda,
) -> ParseResult<SubrTypeSpec> {
debug_call_info!(self);
fn lambda_to_subr_type_spec(mut lambda: Lambda) -> Result<SubrTypeSpec, ParseError> {
let bounds = lambda.sig.bounds;
let lparen = lambda.sig.params.parens.map(|(l, _)| l);
let mut non_defaults = vec![];
@ -2838,8 +2731,7 @@ impl Parser {
};
defaults.push(param);
}
let return_t = self.convert_rhs_to_type_spec(lambda.body.remove(0))?;
self.level -= 1;
let return_t = Self::expr_to_type_spec(lambda.body.remove(0))?;
Ok(SubrTypeSpec::new(
bounds,
lparen,
@ -2851,45 +2743,86 @@ impl Parser {
))
}
fn convert_array_to_array_type_spec(&mut self, array: Array) -> ParseResult<ArrayTypeSpec> {
debug_call_info!(self);
fn array_to_array_type_spec(array: Array) -> Result<ArrayTypeSpec, ParseError> {
match array {
Array::Normal(arr) => {
// TODO: add hint
self.errs
.push(ParseError::simple_syntax_error(line!() as usize, arr.loc()));
Err(())
let err = ParseError::simple_syntax_error(line!() as usize, arr.loc());
Err(err)
}
Array::WithLength(arr) => {
let t_spec = self.convert_rhs_to_type_spec(arr.elem.expr)?;
let len = self.validate_const_expr(*arr.len)?;
self.level -= 1;
let t_spec = Self::expr_to_type_spec(arr.elem.expr)?;
let len = Self::validate_const_expr(*arr.len)?;
Ok(ArrayTypeSpec::new(t_spec, len))
}
Array::Comprehension(arr) => {
// TODO: add hint
self.errs
.push(ParseError::simple_syntax_error(line!() as usize, arr.loc()));
Err(())
let err = ParseError::simple_syntax_error(line!() as usize, arr.loc());
Err(err)
}
}
}
fn convert_set_to_set_type_spec(&mut self, set: Set) -> ParseResult<SetTypeSpec> {
debug_call_info!(self);
fn set_to_set_type_spec(set: Set) -> Result<SetTypeSpec, ParseError> {
match set {
Set::Normal(arr) => {
// TODO: add hint
self.errs
.push(ParseError::simple_syntax_error(line!() as usize, arr.loc()));
Err(())
let err = ParseError::simple_syntax_error(line!() as usize, arr.loc());
Err(err)
}
Set::WithLength(set) => {
let t_spec = self.convert_rhs_to_type_spec(set.elem.expr)?;
let len = self.validate_const_expr(*set.len)?;
self.level -= 1;
let t_spec = Self::expr_to_type_spec(set.elem.expr)?;
let len = Self::validate_const_expr(*set.len)?;
Ok(SetTypeSpec::new(t_spec, len))
}
}
}
pub fn expr_to_type_spec(rhs: Expr) -> Result<TypeSpec, ParseError> {
match rhs {
Expr::Accessor(acc) => Self::accessor_to_type_spec(acc),
Expr::Call(call) => {
let predecl = Self::call_to_predecl_type_spec(call)?;
Ok(TypeSpec::PreDeclTy(predecl))
}
Expr::Lambda(lambda) => {
let lambda = Self::lambda_to_subr_type_spec(lambda)?;
Ok(TypeSpec::Subr(lambda))
}
Expr::Array(array) => {
let array = Self::array_to_array_type_spec(array)?;
Ok(TypeSpec::Array(array))
}
Expr::Set(set) => {
let set = Self::set_to_set_type_spec(set)?;
Ok(TypeSpec::Set(set))
}
Expr::BinOp(bin) => {
if bin.op.kind.is_range_op() {
let op = bin.op;
let mut args = bin.args.into_iter();
let lhs = Self::validate_const_expr(*args.next().unwrap())?;
let rhs = Self::validate_const_expr(*args.next().unwrap())?;
Ok(TypeSpec::Interval { op, lhs, rhs })
} else if bin.op.kind == TokenKind::AndOp {
let mut args = bin.args.into_iter();
let lhs = Self::expr_to_type_spec(*args.next().unwrap())?;
let rhs = Self::expr_to_type_spec(*args.next().unwrap())?;
Ok(TypeSpec::and(lhs, rhs))
} else if bin.op.kind == TokenKind::OrOp {
let mut args = bin.args.into_iter();
let lhs = Self::expr_to_type_spec(*args.next().unwrap())?;
let rhs = Self::expr_to_type_spec(*args.next().unwrap())?;
Ok(TypeSpec::or(lhs, rhs))
} else {
let err = ParseError::simple_syntax_error(line!() as usize, bin.loc());
Err(err)
}
}
other => {
let err = ParseError::simple_syntax_error(line!() as usize, other.loc());
Err(err)
}
}
}
}

View file

@ -109,6 +109,8 @@ pub enum TokenKind {
RefMutOp,
/// =
Equal,
/// <-
Inclusion,
/// :=
Walrus,
/// ->
@ -214,9 +216,8 @@ impl TokenKind {
| InfLit => TokenCategory::Literal,
PrePlus | PreMinus | PreBitNot | Mutate | RefOp | RefMutOp => TokenCategory::UnaryOp,
Try => TokenCategory::PostfixOp,
Comma | Colon | DblColon | SupertypeOf | SubtypeOf | Dot | Pipe | Walrus => {
TokenCategory::SpecialBinOp
}
Comma | Colon | DblColon | SupertypeOf | SubtypeOf | Dot | Pipe | Walrus
| Inclusion => TokenCategory::SpecialBinOp,
Equal => TokenCategory::DefOp,
FuncArrow | ProcArrow => TokenCategory::LambdaOp,
Semi | Newline => TokenCategory::Separator,
@ -246,14 +247,14 @@ impl TokenKind {
BitOr => 120, // ||
Closed | LeftOpen | RightOpen | Open => 100, // range operators
Less | Gre | LessEq | GreEq | DblEq | NotEq | InOp | NotInOp | IsOp | IsNotOp => 90, // < > <= >= == != in notin is isnot
AndOp => 80, // and
OrOp => 70, // or
FuncArrow | ProcArrow => 60, // -> =>
Colon | SupertypeOf | SubtypeOf => 50, // : :> <:
Comma => 40, // ,
Equal | Walrus => 20, // = :=
Newline | Semi => 10, // \n ;
LParen | LBrace | LSqBr | Indent => 0, // ( { [ Indent
AndOp => 80, // and
OrOp => 70, // or
FuncArrow | ProcArrow | Inclusion => 60, // -> => <-
Colon | SupertypeOf | SubtypeOf => 50, // : :> <:
Comma => 40, // ,
Equal | Walrus => 20, // = :=
Newline | Semi => 10, // \n ;
LParen | LBrace | LSqBr | Indent => 0, // ( { [ Indent
_ => return None,
};
Some(prec)