/*
To debug grammar, add `dbg!(&tokens);`, execute with:
```
cargo test --features peg/trace test_fibo -- --nocapture
```
With visualization, see necessary format for temp_trace.txt [here](https://github.com/fasterthanlime/pegviz):
```
cat compiler/parse/temp_trace.txt | pegviz --output ./pegviz.html
```
*/

#[cfg(test)]
mod test_peg_grammar {
    use std::fs;


  #[repr(u8)]
  #[derive(Debug, Clone, Copy, PartialEq, Eq)]
  /// Tokens are full of very dense information to make checking properties about them
  /// very fast.
  /// Some bits have specific meanings: 
  /// * 0b_001*_****: "Identifier-like" things
  /// * 0b_01**_****: "Punctuation"
  ///     * 0b_0100_1***: []{}() INDENT/DEDENT
  ///         * 0b_0100_1**0 [{(INDENT
  ///         * 0b_0100_1**1 ]})DEDENT
  ///     * 0b_011*_**** Operators
  pub enum Token {
      LowercaseIdent      = 0b_0010_0000, 
      UppercaseIdent      = 0b_0011_0011,
      MalformedIdent      = 0b_0010_0001,
  
      KeywordIf           = 0b_0010_0010,
      KeywordThen         = 0b_0010_0011,
      KeywordElse         = 0b_0010_0100,
      KeywordWhen         = 0b_0010_0101,
      KeywordAs           = 0b_0010_0110,
      KeywordIs           = 0b_0010_0111,
      KeywordExpect       = 0b_0010_1000,
      KeywordApp          = 0b_0010_1001,
      KeywordInterface    = 0b_0010_1010,
      KeywordPackages     = 0b_0010_1011,
      KeywordImports      = 0b_0010_1100,
      KeywordProvides     = 0b_0010_1101,
      KeywordTo           = 0b_0010_1110,
      KeywordExposes      = 0b_0010_1111,
      KeywordEffects      = 0b_0011_0000,
      KeywordPlatform     = 0b_0011_0001,
      KeywordRequires     = 0b_0011_0010,
  
      Comma               = 0b_0100_0000,
      Colon               = 0b_0100_0001,
  
      OpenParen           = 0b_0100_1000,
      CloseParen          = 0b_0100_1001,
      OpenCurly           = 0b_0100_1010,
      CloseCurly          = 0b_0100_1011,
      OpenSquare          = 0b_0100_1100,
      CloseSquare         = 0b_0100_1101,
      OpenIndent          = 0b_0100_1110,
      CloseIndent         = 0b_0100_1111,
      SameIndent          = 0b_0101_0000,
  
      OpPlus              = 0b_0110_0000,
      OpMinus             = 0b_0110_0001,
      OpSlash             = 0b_0110_0010,
      OpPercent           = 0b_0110_0011,
      OpCaret             = 0b_0110_0100,
      OpGreaterThan       = 0b_0110_0101,
      OpLessThan          = 0b_0110_0110,
      OpAssignment        = 0b_0110_0111,
      OpPizza             = 0b_0110_1000,
      OpEquals            = 0b_0110_1001,
      OpNotEquals         = 0b_0110_1010,
      OpGreaterThanOrEq   = 0b_0110_1011,
      OpLessThanOrEq      = 0b_0110_1100,
      OpAnd               = 0b_0110_1101,
      OpOr                = 0b_0110_1110,
      OpDoubleSlash       = 0b_0110_1111,
      OpDoublePercent     = 0b_0111_0001,
      OpBackpassing       = 0b_0111_1010,
  
      TodoNextThing       = 0b_1000_0000,
  
      Malformed,
      MalformedOperator,
  
      PrivateTag,
  
      String,
  
      NumberBase,
      Number,
  
      QuestionMark,
  
      Underscore,
  
      Ampersand,
      Pipe,
      Dot,
      Bang,
      LambdaStart,
      Arrow,
      FatArrow,
      Asterisk,
  }
  
  pub struct TokenTable {
      pub tokens: Vec<Token>,
      pub offsets: Vec<usize>,
      pub lengths: Vec<usize>,
  }
  
  pub struct LexState {
      indents: Vec<usize>,
  }
  
  trait ConsumeToken {
      fn token(&mut self, token: Token, offset: usize, length: usize);
  }

  struct TestConsumer{
    tokens: Vec<Token>,
  }

  impl ConsumeToken for TestConsumer {
    fn token(&mut self, token: Token, offset: usize, length: usize){
      self.tokens.push(token);
    }
  }

  fn test_tokenize(code_str: &str) -> Vec<Token> {
    let mut lex_state = LexState{ indents: Vec::new() };
    let mut consumer = TestConsumer{ tokens: Vec::new() };

    tokenize(
      &mut lex_state,
      code_str.as_bytes(),
      &mut consumer
    );

    consumer.tokens
  }
  
  fn tokenize(
      state: &mut LexState,
      bytes: &[u8],
      consumer: &mut impl ConsumeToken,
  ) {
      let mut i = 0;
  
      while i < bytes.len() {
          let bytes = &bytes[i..];
  
          let (token, len) = match bytes[0] {
              b'(' => (Token::OpenParen, 1),
              b')' => (Token::CloseParen, 1),
              b'{' => (Token::OpenCurly, 1),
              b'}' => (Token::CloseCurly, 1),
              b'[' => (Token::OpenSquare, 1),
              b']' => (Token::CloseSquare, 1),
              b',' => (Token::Comma, 1),
              b'_' => lex_underscore(bytes),
              b'@' => lex_private_tag(bytes),
              b'a'..=b'z' => lex_ident(false, bytes),
              b'A'..=b'Z' => lex_ident(true, bytes),
              b'0'..=b'9' => lex_number(bytes),
              b'-' | b':' | b'!' | b'.' | b'*' | b'/' | b'&' |
              b'%' | b'^' | b'+' | b'<' | b'=' | b'>' | b'|' | b'\\' => lex_operator(bytes),
              b' ' => {
                  i += skip_whitespace(bytes);
                  continue;
              }
              b'\n' => {
                  // TODO: add newline to side_table
                  let (new_skip, curr_line_indent) = skip_newlines(bytes);
                  i += new_skip;

                  if let Some(&prev_indent) = state.indents.last() {
                    if curr_line_indent > prev_indent {
                      state.indents.push(curr_line_indent);
                      (Token::OpenIndent, curr_line_indent)
                    } else {
                      i += curr_line_indent;

                      if curr_line_indent == 0 {

                      }

                      if prev_indent == curr_line_indent {
                        consumer.token(Token::SameIndent, i, 0);
                      } else if curr_line_indent < prev_indent {
                        // safe unwrap because we check first
                        while state.indents.last().is_some() && curr_line_indent < *state.indents.last().unwrap() {
                          state.indents.pop();
                          consumer.token(Token::CloseIndent, i, 0);
                        }
                      }

                      continue;
                    }
                  } else if curr_line_indent > 0 {
                    state.indents.push(curr_line_indent);
                    (Token::OpenIndent, curr_line_indent)
                  } else {
                    consumer.token(Token::SameIndent, i, 0);
                    continue;
                  }
                  
              }
              b'#' => {
                  // TODO: add comment to side_table
                  i += skip_comment(bytes);
                  continue;
              }
              b'"' => lex_string(bytes),
              b => todo!("handle {:?}", b as char),
          };
  
          consumer.token(token, i, len);
          i += len;
      }
  }
  
  impl TokenTable {
      pub fn new(text: &str) -> TokenTable {
          let mut tt = TokenTable {
              tokens: Vec::new(),
              offsets: Vec::new(),
              lengths: Vec::new(),
          };
  
          let mut offset = 0;
          let mut state = LexState::new();
  
          // while let Some((token, skip, length)) = Token::lex_single(&mut state, &text.as_bytes()[offset..]) {
          //     tt.tokens.push(token);
          //     offset += skip;
          //     tt.offsets.push(offset);
          //     offset += length;
          //     tt.lengths.push(length);
          // }
  
          tt
      }
  }
  
  impl LexState {
      pub fn new() -> LexState {
          LexState {
              indents: Vec::new(),
          }
      }
  }
  
  fn skip_comment(bytes: &[u8]) -> usize {
      let mut skip = 0;
      while skip < bytes.len() && bytes[skip] != b'\n' {
          skip += 1;
      }
      if skip < bytes.len() && bytes[skip] == b'\n' {
          skip += 1;
      }
      skip
  }
  
  #[derive(Copy, Clone, Eq, PartialEq, PartialOrd, Ord)]
  struct Indent(usize);
  
  fn skip_whitespace(bytes: &[u8]) -> usize {
      debug_assert!(bytes[0] == b' ');
  
      let mut skip = 0;
      while skip < bytes.len() && bytes[skip] == b' ' {
          skip += 1;
      }
      skip
  }
  
  // also skips lines that contain only whitespace
  fn skip_newlines(bytes: &[u8]) -> (usize, usize) {
      let mut skip = 0;
      let mut indent = 0;
  
      while skip < bytes.len() && bytes[skip] == b'\n' {
          skip += indent + 1;

          let spaces = 
          if bytes.len() > skip && bytes[skip] == b' ' {
            skip_whitespace(&bytes[skip..])
          } else {
            0
          };

          if bytes.len() > (skip + spaces) && bytes[skip + spaces] == b'\n' {
            indent = 0;
            skip += spaces;
          } else {
            indent = spaces;
          }
      }
  
      (skip, indent)
  }
  
  fn is_op_continue(ch: u8) -> bool {
      matches!(ch, b'-' | b':' | b'!' | b'.' | b'*' | b'/' | b'&' |
                  b'%' | b'^' | b'+' | b'<' | b'=' | b'>' | b'|' | b'\\')
  }
  
  fn lex_operator(bytes: &[u8]) -> (Token, usize) {
      let mut i = 0;
      while i < bytes.len() && is_op_continue(bytes[i]) {
          i += 1;
      }
      let tok = match &bytes[0..i] {
          b"+" => Token::OpPlus,
          b"-" => Token::OpMinus,
          b"*" => Token::Asterisk,
          b"/" => Token::OpSlash,
          b"%" => Token::OpPercent,
          b"^" => Token::OpCaret,
          b">" => Token::OpGreaterThan,
          b"<" => Token::OpLessThan,
          b"." => Token::Dot,
          b"=" => Token::OpAssignment,
          b":" => Token::Colon,
          b"|" => Token::Pipe,
          b"\\" => Token::LambdaStart,
          b"|>" => Token::OpPizza,
          b"==" => Token::OpEquals,
          b"!" => Token::Bang,
          b"!=" => Token::OpNotEquals,
          b">=" => Token::OpGreaterThanOrEq,
          b"<=" => Token::OpLessThanOrEq,
          b"&&" => Token::OpAnd,
          b"&" => Token::Ampersand,
          b"||" => Token::OpOr,
          b"//" => Token::OpDoubleSlash,
          b"%%" => Token::OpDoublePercent,
          b"->" => Token::Arrow,
          b"<-" => Token::OpBackpassing,
          op => {
              dbg!(std::str::from_utf8(op).unwrap());
              Token::MalformedOperator
          }
      };
      (tok, i)
  }
  
  fn is_ident_continue(ch: u8) -> bool {
      matches!(ch, b'a'..=b'z'|b'A'..=b'Z'|b'0'..=b'9'|b'_')
  }
  
  fn lex_private_tag(bytes: &[u8]) -> (Token, usize) {
      debug_assert!(bytes[0] == b'@');
      let mut i = 1;
      while i < bytes.len() && is_ident_continue(bytes[i]) {
          i += 1;
      }
      (Token::PrivateTag, i)
  }
  
  fn lex_ident(uppercase: bool, bytes: &[u8]) -> (Token, usize) {
      let mut i = 0;
      while i < bytes.len() && is_ident_continue(bytes[i]) {
          i += 1;
      }
      let tok = match &bytes[0..i] {
          b"if" => Token::KeywordIf,
          b"then" => Token::KeywordThen,
          b"else" => Token::KeywordElse,
          b"when" => Token::KeywordWhen,
          b"as" => Token::KeywordAs,
          b"is" => Token::KeywordIs,
          b"expect" => Token::KeywordExpect,
          b"app" => Token::KeywordApp,
          b"interface" => Token::KeywordInterface,
          b"packages" => Token::KeywordPackages,
          b"imports" => Token::KeywordImports,
          b"provides" => Token::KeywordProvides,
          b"to" => Token::KeywordTo,
          b"exposes" => Token::KeywordExposes,
          b"effects" => Token::KeywordEffects,
          b"platform" => Token::KeywordPlatform,
          b"requires" => Token::KeywordRequires,
          ident => {
              if ident.contains(&b'_') {
                  Token::MalformedIdent
              } else if uppercase {
                  Token::UppercaseIdent
              } else {
                  Token::LowercaseIdent
              }
          },
      };
      (tok, i)
  }
  
  fn lex_underscore(bytes: &[u8]) -> (Token, usize) {
      let mut i = 0;
      while i < bytes.len() && is_ident_continue(bytes[i]) {
          i += 1;
      }
      (Token::Underscore, i)
  }
  
  fn is_int_continue(ch: u8) -> bool {
      matches!(ch, b'0'..=b'9' | b'_')
  }
  
  fn lex_number(bytes: &[u8]) -> (Token, usize) {
      let mut i = 0;
      while i < bytes.len() && is_int_continue(bytes[i]) {
          i += 1;
      }
  
      if i < bytes.len() && bytes[i] == b'.' {
          i += 1;
          while i < bytes.len() && is_int_continue(bytes[i]) {
              i += 1;
          }
      }
  
      (Token::Number, i)
  }
  
  fn lex_string(bytes: &[u8]) -> (Token, usize) {
      let mut i = 0;
      assert_eq!(bytes[i], b'"');
      i += 1;
  
      while i < bytes.len() {
          match bytes[i] {
              b'"' => break,
              // TODO: escapes
              _ => i += 1,
          }
      }
  
      assert_eq!(bytes[i], b'"');
      i += 1;
  
      (Token::String, i)
  }

type T = Token;

#[test]
fn test_indent_tokenization_1() {
  let tokens = test_tokenize(r#"showBool = \b ->
    when b is
        True ->
            "True""#);
  
  assert_eq!(
    tokens,
    [T::LowercaseIdent, T::OpAssignment, T::LambdaStart, T::LowercaseIdent, T::Arrow,
    T::OpenIndent, T::KeywordWhen, T::LowercaseIdent, T::KeywordIs,
    T::OpenIndent, T::UppercaseIdent, T::Arrow,
    T::OpenIndent, T::String]
  );
}

#[test]
fn test_indent_tokenization_2() {
  let tokens = test_tokenize(r#"showBool = \b ->
    when b is
        True ->
            "True"
"#);
  
  assert_eq!(
    tokens,
    [T::LowercaseIdent, T::OpAssignment, T::LambdaStart, T::LowercaseIdent, T::Arrow,
    T::OpenIndent, T::KeywordWhen, T::LowercaseIdent, T::KeywordIs,
    T::OpenIndent, T::UppercaseIdent, T::Arrow,
    T::OpenIndent, T::String,
    T::CloseIndent, T::CloseIndent, T::CloseIndent]
  );
}

#[test]
fn test_tokenization_line_with_only_spaces() {
  let tokens = test_tokenize(r#"\key ->
  when dict is
      Empty ->
          4
  
      Node ->
          5"#);
          
  assert_eq!(
    tokens,
    [T::LambdaStart, T::LowercaseIdent, T::Arrow,
    T::OpenIndent, T::KeywordWhen, T::LowercaseIdent, T::KeywordIs,
    T::OpenIndent, T::UppercaseIdent, T::Arrow,
    T::OpenIndent, T::Number,
    T::CloseIndent,
    T::UppercaseIdent, T::Arrow,
    T::OpenIndent, T::Number]
  );
}

// Inspired by https://ziglang.org/documentation/0.7.1/#Grammar
// license information can be found in the LEGAL_DETAILS file in
// the root directory of this distribution.
// Thank you zig contributors!
peg::parser!{
    grammar tokenparser() for [T] {

      pub rule module() =
        header() module_defs()? indented_end()

      pub rule full_expr() = [T::OpenIndent]? op_expr() [T::CloseIndent]?

      
      rule common_expr() =
          closure()
          / expect()
          / if_expr()
          / when()
          / backpass()
          / list()
          / record()
          / record_update()
          / parens_around()
          / [T::Number]
          / [T::NumberBase]
          / [T::String]
          / module_var()
          / tag()
          / accessor_function()
          / defs()
          / annotation()
          / [T::LowercaseIdent]
      pub rule expr() =
          apply()
          / common_expr()
          // / access() // TODO prevent infinite loop

      rule no_apply_expr() =
          common_expr()

        pub rule closure() =
          [T::LambdaStart] args() [T::Arrow] closure_body()

        rule closure_body() =
          [T::OpenIndent] full_expr() ([T::CloseIndent] / end_of_file())
          / [T::SameIndent]? full_expr()

        rule args() =
          (arg() [T::Comma])* arg()

        rule arg() =
          [T::Underscore]
          / ident()
          / record_destructure()


        rule tag() =
          private_tag()
          / [T::UppercaseIdent]

        rule private_tag() = [T::PrivateTag] {}


        rule list() = empty_list()
                    / [T::OpenSquare] (expr() [T::Comma])* expr()? [T::Comma]? [T::CloseSquare] { }
        rule empty_list() = [T::OpenSquare] [T::CloseSquare]


        rule record() =
          empty_record()
          / [T::OpenCurly] __ assigned_fields() __ [T::CloseCurly]

        rule assigned_fields() =
          (__ assigned_field() [T::Comma])* __ assigned_field()? [T::Comma]?

        rule assigned_field() =
          required_value()
          / __ [T::LowercaseIdent]

        rule required_value() =
          __ [T::LowercaseIdent] [T::Colon] full_expr()

        rule empty_record() = [T::OpenCurly] [T::CloseCurly]

        rule record_update() = [T::OpenCurly] expr() [T::Ampersand] assigned_fields() [T::CloseCurly]

        rule record_type() =
          empty_record()
          / [T::OpenCurly] record_field_types_i() [T::CloseCurly]

        rule record_field_types() =
          (record_field_type() [T::Comma] __)* record_field_type() [T::Comma]?

        rule record_field_type() =
          ident() [T::Colon] type_annotation()


        rule parens_around() = [T::OpenParen] full_expr() [T::CloseParen]

        rule if_expr() = [T::KeywordIf] full_expr() [T::KeywordThen] full_expr()
                            [T::KeywordElse] full_expr()

        rule expect() = [T::KeywordExpect] expr()

        pub rule backpass() =
          __ pattern() [T::OpBackpassing] expr()

        rule pattern() =
          [T::LowercaseIdent]
          / [T::Underscore]
          / record_destructure()

        rule record_destructure() =
          empty_record()
          / [T::OpenCurly] (ident() [T::Comma])* ident() [T::Comma]? [T::CloseCurly]

        rule access() =
          expr() [T::Dot] ident()

        rule accessor_function() =
          [T::Dot] ident()

        pub rule header() =
          __ almost_header() header_end()

        pub rule almost_header() =
          app_header()
          / interface_header()
          / platform_header()

        rule app_header() =
          [T::KeywordApp] [T::String] [T::OpenIndent]? packages() imports() provides()// check String to be non-empty?
        
        rule interface_header() =
          [T::KeywordInterface] module_name() [T::OpenIndent]? exposes() imports()

        rule platform_header() =
          [T::KeywordPlatform] [T::String] [T::OpenIndent]? requires() exposes() packages() imports() provides() effects()// check String to be nonempty?

        rule header_end() =
          ([T::CloseIndent]
          / &[T::SameIndent])? // & to not consume the SameIndent
        rule packages() =
          __ [T::KeywordPackages] record() 

        rule imports() =
          __ [T::KeywordImports] imports_list()

        rule imports_list() =
          empty_list()
          / [T::OpenSquare] (imports_entry() [T::Comma])* imports_entry()? [T::Comma]? [T::CloseSquare]

        rule imports_entry() =
          ([T::LowercaseIdent] [T::Dot])?
          module_name()
          ([T::Dot] exposes_list() )?

        rule exposes_list() =
          [T::OpenCurly] (exposes_entry() [T::Comma])* exposes_entry()? [T::Comma]? [T::CloseCurly]
        rule exposes_entry() =
          ident()

        rule provides() =
          __ [T::KeywordProvides] provides_list() ([T::KeywordTo] provides_to())?

        rule provides_to() =
         [T::String]
          / ident()
        
        rule provides_list() =
          empty_list()
          / [T::OpenSquare] exposed_names() [T::CloseSquare]

        rule exposes() =
          __ [T::KeywordExposes] [T::OpenSquare] exposed_names() [T::CloseSquare]

        rule exposed_names() =
          (ident() [T::Comma])* ident()? [T::Comma]?

        rule requires() =
          [T::KeywordRequires] requires_rigids() [T::OpenCurly] typed_ident() [T::CloseCurly]

        rule requires_rigids() =
          empty_record()
          / [T::OpenCurly] (requires_rigid() [T::Comma])* requires_rigid() [T::Comma]? [T::CloseCurly]

        rule requires_rigid() =
          [T::LowercaseIdent] ([T::FatArrow] [T::UppercaseIdent])?

        pub rule typed_ident() =
          [T::LowercaseIdent] [T::Colon] type_annotation()

        pub rule effects() =
          __ [T::KeywordEffects] effect_name() record_type_i()

        rule effect_name() =
          [T::LowercaseIdent] [T::Dot] [T::UppercaseIdent]


        rule module_name() =
          [T::UppercaseIdent] ([T::Dot] [T::UppercaseIdent])*

        rule ident() =
          [T::UppercaseIdent]
          / [T::LowercaseIdent]


        // content of type_annotation without Colon(:)
        pub rule type_annotation() =
          function_type()
          / type_annotation_no_fun()

        rule type_annotation_no_fun() =
          [T::OpenParen] type_annotation_no_fun() [T::CloseParen]
          / tag_union()
          / apply_type()
          / bound_variable()
          / record_type()
          / inferred()
          / wildcard()
        // TODO inline type alias

        rule type_annotation_paren_fun() =
          type_annotation_no_fun()
          / [T::OpenParen] function_type() [T::CloseParen]

        rule tag_union() =
          empty_list()
          / [T::OpenSquare] tags() [T::CloseSquare] type_variable()?

        rule tags() =
          [T::OpenIndent] apply_type() (__ [T::Comma] __ apply_type() __)* [T::Comma]? [T::CloseIndent]
          / (apply_type() [T::Comma])* apply_type() [T::Comma]?
        
        rule type_variable() =
          [T::Underscore]
          / bound_variable()

        rule bound_variable() =
          [T::LowercaseIdent]

        // The `*` type variable, e.g. in (List *)  
        rule wildcard() =
          [T::Asterisk]

        // '_', indicating the compiler should infer the type  
        rule inferred() =
          [T::Underscore]

        rule function_type() =
          ( type_annotation_paren_fun() ([T::Comma] type_annotation_paren_fun())* [T::Arrow])? type_annotation_paren_fun()

        pub rule apply_type() =
          concrete_type() apply_type_args()?
        rule concrete_type() =
          [T::UppercaseIdent] ([T::Dot] [T::UppercaseIdent])*
        rule apply_type_args() =
          type_annotation_no_fun() type_annotation_no_fun()*

        rule _() =
          ([T::OpenIndent]/[T::SameIndent])?

        // TODO write tests for precedence
        pub rule op_expr() = precedence!{ // lowest precedence
            x:(@) _ [T::OpPizza] _ y:@ {}// |>
            --
            x:(@) _ [T::OpAnd] _ y:@ {}
            x:(@) _ [T::OpOr] _ y:@ {}
            --
            x:(@) _ [T::OpEquals] _ y:@ {}
            x:(@) _ [T::OpNotEquals] _ y:@ {}
            x:(@) _ [T::OpLessThan] _ y:@ {}
            x:(@) _ [T::OpGreaterThan] _ y:@ {}
            x:(@) _ [T::OpLessThanOrEq] _ y:@ {}
            x:(@) _ [T::OpGreaterThanOrEq] _ y:@ {}
            --
            x:(@) _ [T::OpPlus] _ y:@ {}
            x:(@) _ [T::OpMinus] _ y:@ {}
            --
            x:(@) _ [T::Asterisk] _ y:@ {}
            x:(@) _ [T::OpSlash] _ y:@ {}
            x:(@) _ [T::OpDoubleSlash] _ y:@ {}
            x:(@) _ [T::OpPercent] _ y:@ {}
            x:(@) _ [T::OpDoublePercent] _ y:@ {}
            --
            x:@ _ [T::OpCaret] _ y:(@) {} // ^
            --
            [T::OpMinus] x:@ {}
            [T::Bang] x:@ {} // !
            --
            expr() {}
        }  // highest precedence

        // TODO DRY?
        rule no_apply_op_expr() = precedence!{ // lowest precedence
          x:(@) _ [T::OpPizza] _ y:@ {}// |>
          --
          x:(@) _ [T::OpAnd] _ y:@ {}
          x:(@) _ [T::OpOr] _ y:@ {}
          --
          x:(@) _ [T::OpEquals] _ y:@ {}
          x:(@) _ [T::OpNotEquals] _ y:@ {}
          x:(@) _ [T::OpLessThan] _ y:@ {}
          x:(@) _ [T::OpGreaterThan] _ y:@ {}
          x:(@) _ [T::OpLessThanOrEq] _ y:@ {}
          x:(@) _ [T::OpGreaterThanOrEq] _ y:@ {}
          --
          x:(@) _ [T::OpPlus] _ y:@ {}
          x:(@) _ [T::OpMinus] _ y:@ {}
          --
          x:(@) _ [T::Asterisk] _ y:@ {}
          x:(@) _ [T::OpSlash] _ y:@ {}
          x:(@) _ [T::OpDoubleSlash] _ y:@ {}
          x:(@) _ [T::OpPercent] _ y:@ {}
          x:(@) _ [T::OpDoublePercent] _ y:@ {}
          --
          x:@ _ [T::OpCaret] _ y:(@) {} // ^
          --
          [T::OpMinus] x:@ {}
          [T::Bang] x:@ {} // !
          --
          no_apply_expr() {}
      }  // highest precedence

        pub rule defs() =
          def() ([T::SameIndent]? def())* [T::SameIndent]? full_expr()

        pub rule def() =
            annotated_body()
            / annotation()
            / body()
            / alias()
            / expect()

        pub rule module_defs() =
          ([T::SameIndent]? def())+

        rule annotation() =
        annotation_pre_colon() [T::Colon] type_annotation()

        rule annotation_pre_colon() =
          apply()
          / tag()
          / ident()

        rule body() =
        ident() [T::OpAssignment] [T::OpenIndent] full_expr() ([T::SameIndent]? full_expr())* ([T::CloseIndent] / end_of_file())
        /  ident() [T::OpAssignment] full_expr() end_of_file()?

        rule annotated_body() =
          annotation() [T::SameIndent] body()

        rule alias() =
          apply_type() [T::Colon] type_annotation()
          
        pub rule when() =
          [T::KeywordWhen] expr() [T::KeywordIs] when_branch()+

        pub rule when_branch() =
          __ matchable() ([T::Pipe] full_expr())* ([T::KeywordIf] full_expr())? [T::Arrow] when_branch_body() 

        rule when_branch_body() =
          [T::OpenIndent] full_expr() ([T::CloseIndent] / end_of_file())
          / full_expr()

        rule matchable() =
          type_annotation_no_fun()
          / expr()

        rule var() =
          [T::LowercaseIdent]
          / module_var()

        rule module_var() =
          module_name() [T::Dot] [T::LowercaseIdent]

        pub rule apply() =
          apply_expr() apply_args()

        pub rule apply_args() =
          [T::OpenIndent] op_expr() single_line_apply_args()? ([T::CloseIndent]/indented_end())
          / no_apply_op_expr()+

        rule single_line_apply_args() =
          ([T::SameIndent] op_expr()) indented_end()
          / ([T::OpenIndent] op_expr() indented_end())
          / ([T::SameIndent] op_expr()) single_line_apply_args()*
          / ([T::OpenIndent] op_expr() single_line_apply_args()* [T::CloseIndent])

        rule apply_expr() =
          var()
          / tag()

        rule end() =
          [T::CloseIndent]
          / &[T::SameIndent] // & to not consume the SameIndent
          / end_of_file()

        rule indented_end() =
          ([T::OpenIndent] / [T::CloseIndent] / [T::SameIndent])* end_of_file()

        // for optionalindents
        // underscore rules do not require parentheses  
        rule __() =
          (
            [T::OpenIndent]
          / [T::CloseIndent]
          / [T::SameIndent]
        )?

        rule end_of_file() =
         ![_]

        rule record_type_i() = 
          [T::OpenIndent] record_type() [T::CloseIndent]?
          / record_type()

        rule record_field_types_i() =
          [T::OpenIndent] record_field_types() [T::CloseIndent]
          / record_field_types()
    }
}

#[test]
fn test_basic_expr() {
    assert_eq!(tokenparser::expr(&[T::OpenSquare, T::CloseSquare]), Ok(()));
    assert_eq!(tokenparser::expr(&[T::OpenCurly, T::CloseCurly]), Ok(()));

    assert_eq!(tokenparser::expr(&[T::OpenParen, T::OpenSquare, T::CloseSquare, T::CloseParen]), Ok(()));

    assert_eq!(tokenparser::expr(&[T::Number]), Ok(()));
    assert_eq!(tokenparser::expr(&[T::String]), Ok(()));

    assert_eq!(tokenparser::expr(&[T::KeywordIf, T::Number, T::KeywordThen, T::Number, T::KeywordElse, T::Number]), Ok(()));

    assert_eq!(tokenparser::expr(&[T::KeywordExpect, T::Number]), Ok(()));

    assert_eq!(tokenparser::expr(&[T::LowercaseIdent, T::OpBackpassing, T::Number]), Ok(()));    
}

#[test]
fn test_app_header_1() {
  let tokens = test_tokenize( r#"app "test-app" packages {} imports [] provides [] to blah"#);
  
  assert_eq!(tokenparser::header(&tokens), Ok(()));
}

#[test]
fn test_app_header_2() {
  let tokens = test_tokenize( r#"
app "test-app"
    packages { pf: "platform" }
    imports []
    provides [ main ] to pf
"#);

  assert_eq!(tokenparser::header(&tokens), Ok(()));
}

#[test]
fn test_interface_header() {
  let tokens = test_tokenize( r#"
interface Foo.Bar.Baz exposes [] imports []"#);

  assert_eq!(tokenparser::header(&tokens), Ok(()));
}

#[test]
fn test_interface_header_2() {
  let tokens = test_tokenize( r#"

  interface Base64.Encode
      exposes [ toBytes ]
      imports [ Bytes.Encode.{ Encoder } ]"#);

  assert_eq!(tokenparser::header(&tokens), Ok(()));
}

#[test]
fn test_platform_header_1() {

    let tokens = test_tokenize( r#"platform "rtfeldman/blah" requires {} { main : {} } exposes [] packages {} imports [] provides [] effects fx.Blah {}"#);
    
    assert_eq!(tokenparser::header(&tokens), Ok(()));
}

#[test]
fn test_platform_header_2() {

    let tokens = test_tokenize( r#"platform "examples/cli"
    requires {}{ main : Task {} [] } # TODO FIXME
    exposes []
    packages {}
    imports [ Task.{ Task } ]
    provides [ mainForHost ]
    effects fx.Effect
        {
            getLine : Effect Str,
            putLine : Str -> Effect {},
            twoArguments : Int, Int -> Effect {}
        }"#);
    
    assert_eq!(tokenparser::header(&tokens), Ok(()));
}

#[test]
fn test_annotated_def() {
  let tokens = test_tokenize( r#"test1 : Bool
test1 =
    example1 == [ 2, 4 ]"#);

  assert_eq!(tokenparser::def(&tokens), Ok(()));
}

#[test]
fn test_record_def_1() {

    let tokens = test_tokenize( r#"x = { content: 4 }"#);

    assert_eq!(tokenparser::def(&tokens), Ok(()));
}

#[test]
fn test_record_def_2() {

  let tokens = test_tokenize( r#"x =
   { content: 4 }"#);

  assert_eq!(tokenparser::def(&tokens), Ok(()));
}

#[test]
fn test_record_def_3() {

  let tokens = test_tokenize( r#"x =
   {
     a: 4,
     b: 5
    }"#);

  assert_eq!(tokenparser::def(&tokens), Ok(()));
}

#[test]
fn test_record_def_4() {

  let tokens = test_tokenize( r#"x =
   {
     a: 4,
       b: 5,
     c: 6,
       }"#);
  
  assert_eq!(tokenparser::def(&tokens), Ok(()));
}

#[test]
fn test_record_def_5() {

  let tokens = test_tokenize( r#"x =
   {
   a: 4,
   }"#);
   
  assert_eq!(tokenparser::def(&tokens), Ok(()));
}

#[test]
fn test_typed_ident() {
  // main : Task {} []
  assert_eq!(tokenparser::typed_ident(&[
    T::LowercaseIdent, T::Colon, T::UppercaseIdent, T::OpenCurly, T::CloseCurly, T::OpenSquare, T::CloseSquare
  ]), Ok(()));
}

#[test]
fn test_order_of_ops() {
  // True || False && True || False
  assert_eq!(tokenparser::full_expr(&[T::UppercaseIdent, T::OpOr, T::UppercaseIdent, T::OpAnd, T::UppercaseIdent, T::OpOr, T::UppercaseIdent]), Ok(()));
}

fn file_to_string(file_path: &str) -> String {
  // it's ok to panic in a test
  fs::read_to_string(file_path).unwrap()
}

fn example_path(sub_path: &str) -> String {
  let examples_dir = "../../examples/".to_string();

  let file_path = examples_dir + sub_path;

  file_to_string(
    &file_path
  )
}

#[test]
fn test_hello() {
  let tokens = test_tokenize(&example_path("hello-world/Hello.roc"));
  
  assert_eq!(tokenparser::module(&tokens), Ok(()));
}

#[test]
fn test_fibo() {
  let tokens = test_tokenize(&example_path("fib/Fib.roc"));
  
  assert_eq!(tokenparser::module(&tokens), Ok(()));
}

#[test]
fn test_annotation() {
  let tokens = test_tokenize( r#"ConsList a : [ Cons a (ConsList a), Nil ]"#);

  assert_eq!(tokenparser::def(&tokens), Ok(()));
}

#[test]
fn test_apply_type() {
  let tokens = test_tokenize( r#"Cons a (ConsList a)"#);

  assert_eq!(tokenparser::apply_type(&tokens), Ok(()));
}

#[test]
fn test_apply_expect_fail_1() {
  assert!(tokenparser::apply(&[
    T::LowercaseIdent, T::LowercaseIdent,T::CloseIndent, T::UppercaseIdent
  ]).is_err());
}

#[test]
fn test_apply_expect_fail_2() {
  let tokens = test_tokenize( r#"eval a
b"#);

  assert!(tokenparser::apply(&tokens).is_err());
}

#[test]
fn test_backpass_expect_fail() {
  let tokens = test_tokenize( r#"lastName <- 4
5"#);

  assert!(tokenparser::backpass(&tokens).is_err());
}

#[test]
fn test_when_1() {
  let tokens = test_tokenize( r#"when list is
  Cons _ rest ->
      1 + len rest

  Nil ->
      0"#);
  
  assert_eq!(tokenparser::when(&tokens), Ok(()));
}

#[test]
fn test_when_2() {
  let tokens = test_tokenize( r#"when list is
  Nil ->
      Cons a

  Nil ->
      Nil"#);

  assert_eq!(tokenparser::when(&tokens), Ok(()));
}

#[test]
fn test_cons_list() {
  let tokens = test_tokenize(&example_path("effect/ConsList.roc"));

  assert_eq!(tokenparser::module(&tokens), Ok(()));
}

#[test]
fn test_base64() {
  let tokens = test_tokenize(&example_path("benchmarks/Base64.roc"));

  assert_eq!(tokenparser::module(&tokens), Ok(()));
}

#[test]
fn test_base64_test() {
  let tokens = test_tokenize(&example_path("benchmarks/TestBase64.roc"));
  
  assert_eq!(tokenparser::module(&tokens), Ok(()));
}

#[test]
fn test_when_branch() {
  let tokens = test_tokenize(r#"Ok path -> path"#);

  assert_eq!(tokenparser::when_branch(&tokens), Ok(()));
}
#[test]
fn test_def_in_def() {
  let tokens = test_tokenize(r#"example =
  cost = 1

  cost"#);
  assert_eq!(tokenparser::def(&tokens), Ok(()));
}

#[test]
fn test_backpass_in_def() {
  let tokens = test_tokenize(r#"main =
  lastName <- 4
  Stdout.line "Hi!""#);

  assert_eq!(tokenparser::def(&tokens), Ok(()));
}

#[test]
fn test_astar_test() {
  let tokens = test_tokenize(&example_path("benchmarks/TestAStar.roc"));

  assert_eq!(tokenparser::module(&tokens), Ok(()));
}

#[test]
fn test_cli_echo() {
  let tokens = test_tokenize(&example_path("cli/Echo.roc"));
  dbg!(&tokens);
  assert_eq!(tokenparser::module(&tokens), Ok(()));
}

#[test]
fn test_pizza() {
  let tokens = test_tokenize(r#"closure = \_ ->
  Task.succeed {}
      |> Task.map (\_ -> x)"#);
  
  assert_eq!(tokenparser::def(&tokens), Ok(()));
}

#[test]
fn test_closure_file() {
  let tokens = test_tokenize(&example_path("benchmarks/Closure.roc"));

  assert_eq!(tokenparser::module(&tokens), Ok(()));
}

#[test]
fn test_def_with_indents() {
  let tokens = test_tokenize(r#"main =
  Task.after
      Task.getInt
      \n ->
          queens n"#);

  assert_eq!(tokenparser::def(&tokens), Ok(()));
}

#[test]
fn test_nqueens() {
  let tokens = test_tokenize(&example_path("benchmarks/NQueens.roc"));
  dbg!(&tokens);
  assert_eq!(tokenparser::module(&tokens), Ok(()));
}

// TODO fix infinite loop
/*#[test]
fn test_quicksort() {
  let tokens = test_tokenize(&example_path("benchmarks/Quicksort.roc"));

  assert_eq!(tokenparser::module(&tokens), Ok(()));
}*/

#[test]
fn test_indented_closure_apply() {
  let tokens = test_tokenize(r#"
  effect
  \result -> result"#);

  assert_eq!(tokenparser::apply_args(&tokens), Ok(()));
}

#[test]
fn test_task() {
  let tokens = test_tokenize(&example_path("benchmarks/platform/Task.roc"));
  dbg!(&tokens);
  assert_eq!(tokenparser::module(&tokens), Ok(()));
}

#[test]
fn test_pizza_line() {
  let tokens = test_tokenize(r#"unoptimized
      |> Num.toStr
      |> Task.putLine"#);

  assert_eq!(tokenparser::full_expr(&tokens), Ok(()));
}

#[test]
fn test_defs_w_apply() {
  let tokens = test_tokenize(r#"unoptimized = eval e

42"#);
  dbg!(&tokens);
  assert_eq!(tokenparser::defs(&tokens), Ok(()));
}

// TODO fix infinite loop
/*
#[test]
fn test_cfold() {
  let tokens = test_tokenize(&example_path("benchmarks/CFold.roc"));
  dbg!(&tokens);
  assert_eq!(tokenparser::module(&tokens), Ok(()));
}*/

#[test]
fn test_multi_defs() {
  let tokens = test_tokenize(r#"main =
    tree : I64
    tree = 0

    tree"#);

  assert_eq!(tokenparser::def(&tokens), Ok(()));
}

#[test]
fn test_rbtree_insert() {
  let tokens = test_tokenize(&example_path("benchmarks/RBTreeInsert.roc"));

  assert_eq!(tokenparser::module(&tokens), Ok(()));
}

#[test]
fn test_closure_1() {
  let tokens = test_tokenize(r#"\key ->
      when dict is
          Empty ->
              4

          Node ->
              5"#);

  dbg!(&tokens);
  assert_eq!(tokenparser::closure(&tokens), Ok(()));
}


#[test]
fn test_closure_2() {
  let tokens = test_tokenize(r#"\key ->
      when dict is
          Empty ->
              Node Red

          Node nColor ->
              when key is  
                  GT ->
                      balance nColor"#);

  assert_eq!(tokenparser::closure(&tokens), Ok(()));
}

#[test]
fn test_nested_apply() {
  let tokens = test_tokenize(r#"after = \effect ->
  Effect.after
      transform a

map : Str"#);
  dbg!(&tokens);

  assert_eq!(tokenparser::module_defs(&tokens), Ok(()));
}

#[test]
fn test_deep_indented_defs() {
  let tokens = test_tokenize(r#"after = \effect ->
  after
      \result ->
          transform a

map : Str"#);
  dbg!(&tokens);

  assert_eq!(tokenparser::module_defs(&tokens), Ok(()));
}

// TODO fix infinite loop
/*#[test]
fn test_rbtree_ck() {
  let tokens = test_tokenize(&example_path("benchmarks/RBTreeCk.roc"));

  assert_eq!(tokenparser::module(&tokens), Ok(()));
}*/



}