erg/crates/erg_parser/tests/tokenize_test.rs

use std::iter::Iterator;

use erg_common::io::Input;

// use erg_compiler::parser;

use erg_parser::error::ParseResult;
use erg_parser::lex::Lexer;
use erg_parser::token::*;
use TokenKind::*;

const FILE1: &str = "tests/test1_basic_syntax.er";
const FILE2: &str = "tests/test2_advanced_syntax.er";
const FILE3: &str = "tests/test3_literal_syntax.er";
const FILE4: &str = "tests/multi_line_str_literal.er";
const FILE5: &str = "tests/for.er";

#[test]
fn test_lexer_for_basic() -> ParseResult<()> {
    let mut lexer = Lexer::new(Input::file(FILE1.into()));
    let newline = "\n";
    let token_array = vec![
        (Newline, newline),
        (Newline, newline),
        (Symbol, "_a"),
        (Assign, "="),
        (NatLit, "1_234"),
        (Plus, "+"),
        (RatioLit, "1113."),
        (Star, "*"),
        (RatioLit, "3_000.2e-4"),
        (Pow, "**"),
        (NatLit, "0003"),
        (Star, "*"),
        (RatioLit, ".4"),
        (Newline, newline),
        (Symbol, "a"),
        (Comma, ","),
        (UBar, "_"),
        (Comma, ","),
        (Symbol, "b"),
        (Assign, "="),
        (Symbol, "tuple"),
        (Newline, newline),
        (Symbol, "f"),
        (Symbol, "x"),
        (Comma, ","),
        (Symbol, "y"),
        (Assign, "="),
        (Newline, newline),
        (Indent, "    "),
        (Symbol, "x"),
        (Plus, "+"),
        (Symbol, "y"),
        (Newline, newline),
        (Dedent, ""),
        (Symbol, "if!"),
        (BoolLit, "True"),
        (Comma, ","),
        (Symbol, "do!"),
        (Colon, ":"),
        (Newline, newline),
        (Indent, "    "),
        (Symbol, "print!"),
        (StrLit, "\"\"\\hello, world\\\"\""),
        (Newline, newline),
        (Newline, newline),
        (NatLit, "10"),
        (Dot, "."),
        (Symbol, "times!"),
        (Symbol, "do!"),
        (Colon, ":"),
        (Newline, newline),
        (Indent, "    "),
        (Symbol, "if!"),
        (Symbol, "x"),
        (Dot, "."),
        (Symbol, "y"),
        (Dot, "."),
        (Symbol, "z"),
        (Comma, ","),
        (Symbol, "do!"),
        (Colon, ":"),
        (Newline, newline),
        (Indent, "    "),
        (Symbol, "print!"),
        (StrLit, "\"\""),
        (Newline, newline),
        (Dedent, ""),
        (Symbol, "Hello"),
        (Assign, "="),
        (Symbol, "S2c"),
        (StrLit, "\"hello\""),
        (Newline, newline),
        (NoneLit, "None"),
        (Newline, newline),
        (Dedent, ""),
        (Dedent, ""),
        (Symbol, "aあ아"),
        (Assign, "="),
        (Newline, newline),
        (Indent, "    "),
        (Newline, newline),
        (StrLit, "\"aaa\""),
        (Newline, newline),
        (Dedent, ""),
        (Symbol, "x"),
        (Semi, ";"),
        (Symbol, "x"),
        (Semi, ";"),
        (Semi, ";"),
        (Symbol, "x"),
        (Semi, ";"),
        (Newline, newline),
        (NatLit, "10"),
        (Closed, ".."),
        (Symbol, "twelve"),
        (Semi, ";"),
        (Newline, newline),
        (EOF, "\0"),
    ];
    let mut tok: Token;
    for (id, i) in token_array.into_iter().enumerate() {
        print!("{id:>03}: ");
        tok = lexer.next().unwrap().unwrap();
        assert_eq!(tok, Token::from_str(i.0, i.1));
        println!("{tok}");
    }
    Ok(())
}

#[test]
fn test_lexer_for_advanced() -> ParseResult<()> {
    let mut lexer = Lexer::new(Input::file(FILE2.into()));
    let newline = "\n";
    let token_array = vec![
        (Newline, newline),
        (Newline, newline),
        (Newline, newline),
        (Newline, newline),
        (Symbol, "fib"),
        (NatLit, "0"),
        (Assign, "="),
        (NatLit, "0"),
        (Newline, newline),
        (Symbol, "fib"),
        (NatLit, "1"),
        (Assign, "="),
        (NatLit, "1"),
        (Newline, newline),
        (Symbol, "fib"),
        (LParen, "("),
        (Symbol, "n"),
        (Colon, ":"),
        (Symbol, "Nat"),
        (RParen, ")"),
        (Colon, ":"),
        (Symbol, "Nat"),
        (Assign, "="),
        (Symbol, "fib"),
        (LParen, "("),
        (Symbol, "n"),
        (Minus, "-"),
        (NatLit, "1"),
        (RParen, ")"),
        (Plus, "+"),
        (Symbol, "fib"),
        (LParen, "("),
        (Symbol, "n"),
        (Minus, "-"),
        (NatLit, "2"),
        (RParen, ")"),
        (Newline, newline),
        (Newline, newline),
        (Newline, newline),
        (Symbol, "t"),
        (Assign, "="),
        (Symbol, "if"),
        (BoolLit, "True"),
        (Colon, ":"),
        (Newline, newline),
        (Indent, "    "),
        (Symbol, "then"),
        (Walrus, ":="),
        (NatLit, "1"),
        (Newline, newline),
        (Symbol, "else"),
        (Walrus, ":="),
        (NatLit, "2"),
        (Newline, newline),
        (Dedent, ""),
        (Symbol, "assert"),
        (Symbol, "t"),
        (DblEq, "=="),
        (NatLit, "1"),
        (Newline, newline),
        (Newline, newline),
        (Newline, newline),
        (Symbol, "math"),
        (Assign, "="),
        (Symbol, "import"),
        (StrLit, "\"math\""),
        (Newline, newline),
        (Newline, newline),
        (LBrace, "{"),
        (Symbol, "pi"),
        (Semi, ";"),
        (RBrace, "}"),
        (Assign, "="),
        (Symbol, "import"),
        (StrLit, "\"math\""),
        (Newline, newline),
        (EOF, "\0"),
    ];
    let mut tok: Token;
    for (id, i) in token_array.into_iter().enumerate() {
        print!("{id:>03}: ");
        tok = lexer.next().unwrap().unwrap();
        assert_eq!(tok, Token::from_str(i.0, i.1));
        println!("{tok}");
    }
    Ok(())
}

#[test]
fn test_lexer_for_literals() -> ParseResult<()> {
    let mut lexer = Lexer::new(Input::file(FILE3.into()));
    let newline = "\n";
    let token_array = vec![
        (Newline, newline),
        (Newline, newline),
        (NatLit, "0"),
        (Comma, ","),
        (NatLit, "1"),
        (Comma, ","),
        (NatLit, "2"),
        (Comma, ","),
        (NatLit, "100_000"),
        (Newline, newline),
        (Newline, newline),
        (Newline, newline),
        (IntLit, "-1"),
        (Comma, ","),
        (IntLit, "-10"),
        (Comma, ","),
        (IntLit, "-100_000"),
        (Newline, newline),
        (Newline, newline),
        (Newline, newline),
        (RatioLit, "0.00"),
        (Comma, ","),
        (RatioLit, "-0.0"),
        (Comma, ","),
        (RatioLit, ".1"),
        (Comma, ","),
        (RatioLit, "400."),
        (Newline, newline),
        (Newline, newline),
        (Newline, newline),
        (StrLit, "\"\""),
        (Comma, ","),
        (StrLit, "\"a\""),
        (Comma, ","),
        (StrLit, "\"こんにちは\""),
        (Comma, ","),
        (StrLit, "\"\"\\\""),
        (Comma, ","),
        (StrLit, "\"\"\'\\\0\r\n    \""),
        (Newline, newline),
        (Newline, newline),
        (Newline, newline),
        (BoolLit, "True"),
        (Comma, ","),
        (BoolLit, "False"),
        (Newline, newline),
        (Newline, newline),
        (Newline, newline),
        (NoneLit, "None"),
        (Newline, newline),
        (Newline, newline),
        (Newline, newline),
        (EllipsisLit, "..."),
        (Newline, newline),
        (Newline, newline),
        (Newline, newline),
        (InfLit, "Inf"),
        (Comma, ","),
        (PreMinus, "-"),
        (InfLit, "Inf"),
        (Newline, newline),
        (Newline, newline),
        (Newline, newline),
        (Symbol, "NotImplemented"),
        (Newline, newline),
        (Newline, newline),
        (Newline, newline),
        (Newline, newline),
        (EOF, "\0"),
    ];
    let mut tok: Token;
    for (id, i) in token_array.into_iter().enumerate() {
        print!("{id:>03}: ");
        tok = lexer.next().unwrap().unwrap();
        assert_eq!(tok, Token::from_str(i.0, i.1));
        println!("{tok}");
    }
    Ok(())
}

#[test]
fn test_lexer_for_multi_line_str_literal() -> ParseResult<()> {
    let mut lexer = Lexer::new(Input::file(FILE4.into()));
    let newline = "\n";
    let token_array = [
        (Newline, newline),
        (Newline, newline),
        (Symbol, "single_a"),
        (Assign, "="),
        (StrLit, "\"line break\naaa\nbbb\nccc\nline break\""),
        (Newline, newline),
        (Symbol, "print!"),
        (Symbol, "single_a"),
        (Newline, newline),
        (Newline, newline),
        (Symbol, "multi_line_a"),
        (Assign, "="),
        (
            StrLit,
            "\"\"\"line break
aaa
bbb
ccc
line break\"\"\"",
        ),
        (Newline, newline),
        (Symbol, "print!"),
        (Symbol, "multi_line_a"),
        (Newline, newline),
        (Newline, newline),
        (Symbol, "single_b"),
        (Assign, "="),
        (
            StrLit,
            "\"ignore line break    Hello,\n    Worldignored line break\"",
        ),
        (Newline, newline),
        (Symbol, "print!"),
        (Symbol, "single_b"),
        (Newline, newline),
        (Newline, newline),
        (Symbol, "multi_line_b"),
        (Assign, "="),
        (
            StrLit,
            "\"\"\"ignore line break    Hello,
    Worldignored line break\"\"\"",
        ),
        (Newline, newline),
        (Symbol, "print!"),
        (Symbol, "multi_line_b"),
        (Newline, newline),
        (Newline, newline),
        (Symbol, "complex_string"),
        (Assign, "="),
        (StrLit, "\"\"\"\\    \0\r\n\\\"\"\""),
        (Newline, newline),
        (Symbol, "print!"),
        (Symbol, "complex_string"),
        (Newline, newline),
        (Newline, newline),
        (Symbol, "quotation_mark"),
        (Assign, "="),
        (StrLit, "\"\"\"\"\"\"\"\"\""),
        (Newline, newline),
        (Symbol, "print!"),
        (Symbol, "quotation_mark"),
        (Newline, newline),
        (Newline, newline),
        (Symbol, "quotation_marks"),
        (Assign, "="),
        (
            StrLit,
            "\"\"\"\"
\"\"
\"\"\"
\"\"\"\"\"
\"\"\"
\"\"\"\"\"\"",
        ),
        (Newline, newline),
        (Symbol, "print!"),
        (Symbol, "quotation_marks"),
        (Newline, newline),
        (EOF, "\0"),
    ];
    let mut tok: Token;
    for (id, i) in token_array.into_iter().enumerate() {
        print!("{id:>03}: ");
        tok = lexer.next().unwrap().unwrap();
        assert_eq!(tok, Token::from_str(i.0, i.1));
        println!("{tok}");
    }
    Ok(())
}

#[test]
fn for_loop() -> ParseResult<()> {
    let mut lexer = Lexer::new(Input::file(FILE5.into()));
    let newline = "\n";
    let token_array = [
        (Symbol, "for!"),
        (NatLit, "0"),
        (Closed, ".."),
        (NatLit, "1"),
        (Comma, ","),
        (Symbol, "i"),
        (ProcArrow, "=>"),
        (Newline, newline),
        (Indent, "    "),
        (Symbol, "for!"),
        (NatLit, "0"),
        (Closed, ".."),
        (NatLit, "1"),
        (Comma, ","),
        (Symbol, "j"),
        (ProcArrow, "=>"),
        (Newline, newline),
        (Indent, "    "),
        (Symbol, "for!"),
        (NatLit, "0"),
        (Closed, ".."),
        (NatLit, "1"),
        (Comma, ","),
        (Symbol, "k"),
        (ProcArrow, "=>"),
        (Newline, newline),
        (Newline, newline),
        (Indent, "    "),
        (Symbol, "print!"),
        (StrLit, "\"hi\""),
        (Newline, newline),
        (Dedent, ""),
        (Dedent, ""),
        (Dedent, ""),
        (EOF, "\0"),
    ];
    let mut tok: Token;
    for (id, i) in token_array.into_iter().enumerate() {
        print!("{id:>03}: ");
        tok = lexer.next().unwrap().unwrap();
        assert_eq!(tok, Token::from_str(i.0, i.1));
        println!("{tok}");
    }
    Ok(())
}

#[test]
fn tesop_te_prec() {
    assert_eq!(Mod.precedence(), Some(170));
    assert_eq!(LParen.precedence(), Some(0));
    assert_eq!(Illegal.precedence(), None);
}