From 028dbd48930f85215c13eefebdbfeeb07f6e995a Mon Sep 17 00:00:00 2001 From: pedrocarlo Date: Sun, 14 Dec 2025 01:37:22 -0300 Subject: [PATCH] add raw keyword for expect tests to not trim anything from the block --- turso-test-runner/src/parser/lexer.rs | 20 ++++++-- turso-test-runner/src/parser/mod.rs | 69 ++++++++++++++++++++++++--- 2 files changed, 77 insertions(+), 12 deletions(-) diff --git a/turso-test-runner/src/parser/lexer.rs b/turso-test-runner/src/parser/lexer.rs index 49572dbe6..5dd6e09b7 100644 --- a/turso-test-runner/src/parser/lexer.rs +++ b/turso-test-runner/src/parser/lexer.rs @@ -3,6 +3,7 @@ use miette::{Diagnostic, SourceSpan}; use std::fmt; /// Extract block content between braces, handling nested braces +/// Note: Does NOT trim content - trimming is handled by the parser based on context fn extract_block_content(lexer: &mut Lexer<'_, Token>) -> Option { let remainder = lexer.remainder(); let mut depth = 1; @@ -13,7 +14,7 @@ fn extract_block_content(lexer: &mut Lexer<'_, Token>) -> Option { '}' => { depth -= 1; if depth == 0 { - let content = remainder[..idx].trim().to_string(); + let content = remainder[..idx].to_string(); // Bump past the content and the closing brace lexer.bump(idx + 1); return Some(content); @@ -67,6 +68,10 @@ pub enum Token { #[token("unordered")] Unordered, + /// `raw` modifier (preserves whitespace in expect blocks) + #[token("raw")] + Raw, + /// `readonly` modifier #[token("readonly")] Readonly, @@ -121,6 +126,7 @@ impl fmt::Display for Token { Token::Error => write!(f, "error"), Token::Pattern => write!(f, "pattern"), Token::Unordered => write!(f, "unordered"), + Token::Raw => write!(f, "raw"), Token::Readonly => write!(f, "readonly"), Token::Memory => write!(f, ":memory:"), Token::TempFile => write!(f, ":temp:"), @@ -259,9 +265,10 @@ mod tests { assert_eq!(tokens.len(), 3); assert_eq!(tokens[0].token, Token::Setup); assert_eq!(tokens[1].token, Token::Identifier("users".to_string())); + // Block content is not trimmed by the lexer (parser handles trimming) assert_eq!( tokens[2].token, - Token::BlockContent("CREATE TABLE users (id INTEGER);".to_string()) + Token::BlockContent(" CREATE TABLE users (id INTEGER); ".to_string()) ); } @@ -287,9 +294,10 @@ mod tests { non_newline[5].token, Token::Identifier("select-1".to_string()) ); + // Block content is not trimmed by the lexer (parser handles trimming) assert_eq!( non_newline[6].token, - Token::BlockContent("SELECT 1;".to_string()) + Token::BlockContent(" SELECT 1; ".to_string()) ); } @@ -298,9 +306,10 @@ mod tests { let tokens = tokenize("expect error { no such table }").unwrap(); assert_eq!(tokens[0].token, Token::Expect); assert_eq!(tokens[1].token, Token::Error); + // Block content is not trimmed by the lexer (parser handles trimming) assert_eq!( tokens[2].token, - Token::BlockContent("no such table".to_string()) + Token::BlockContent(" no such table ".to_string()) ); let tokens = tokenize("expect pattern { ^\\d+$ }").unwrap(); @@ -318,9 +327,10 @@ mod tests { assert_eq!(tokens[0].token, Token::Test); assert_eq!(tokens[1].token, Token::Identifier("nested".to_string())); // The json_object call has parens but no braces, should work fine + // Block content is not trimmed by the lexer (parser handles trimming) assert_eq!( tokens[2].token, - Token::BlockContent("SELECT json_object('a', 1);".to_string()) + Token::BlockContent(" SELECT json_object('a', 1); ".to_string()) ); } diff --git a/turso-test-runner/src/parser/mod.rs b/turso-test-runner/src/parser/mod.rs index 1c7841156..cfac6a560 100644 --- a/turso-test-runner/src/parser/mod.rs +++ b/turso-test-runner/src/parser/mod.rs @@ -114,7 +114,7 @@ impl Parser { self.expect_token(Token::Setup)?; let name = self.expect_identifier()?; - let content = self.expect_block_content()?; + let content = self.expect_block_content()?.trim().to_string(); Ok((name, content)) } @@ -148,7 +148,7 @@ impl Parser { // Parse test self.expect_token(Token::Test)?; let (name, name_span) = self.expect_identifier_with_span()?; - let sql = self.expect_block_content()?; + let sql = self.expect_block_content()?.trim().to_string(); self.skip_newlines_and_comments(); @@ -171,7 +171,7 @@ impl Parser { match self.peek() { Some(Token::Error) => { self.advance(); - let content = self.expect_block_content()?; + let content = self.expect_block_content()?.trim().to_string(); let pattern = if content.is_empty() { None } else { @@ -181,7 +181,7 @@ impl Parser { } Some(Token::Pattern) => { self.advance(); - let content = self.expect_block_content()?; + let content = self.expect_block_content()?.trim().to_string(); Ok(Expectation::Pattern(content)) } Some(Token::Unordered) => { @@ -189,16 +189,28 @@ impl Parser { let content = self.expect_block_content()?; // Trim each line to handle indentation in expect blocks let rows = content + .trim() .lines() .map(|s| s.trim().to_string()) .filter(|s| !s.is_empty()) .collect(); Ok(Expectation::Unordered(rows)) } + Some(Token::Raw) => { + self.advance(); + let content = self.expect_block_content()?; + // Raw mode: preserve whitespace exactly, only split on newlines + // We still strip the leading/trailing newlines from the block itself + let content = content.strip_prefix('\n').unwrap_or(&content); + let content = content.strip_suffix('\n').unwrap_or(content); + let rows = content.lines().map(|s| s.to_string()).collect(); + Ok(Expectation::Exact(rows)) + } Some(Token::BlockContent(_)) => { let content = self.expect_block_content()?; // Trim each line to handle indentation in expect blocks let rows = content + .trim() .lines() .map(|s| s.trim().to_string()) .filter(|s| !s.is_empty()) @@ -300,9 +312,10 @@ impl Parser { } fn error(&self, message: String) -> ParseError { - let span = self.tokens.get(self.pos).map(|token| { - SourceSpan::new(token.span.start.into(), token.span.len()) - }); + let span = self + .tokens + .get(self.pos) + .map(|token| SourceSpan::new(token.span.start.into(), token.span.len())); ParseError::SyntaxError { message, @@ -551,6 +564,48 @@ expect { assert_eq!(file.tests[0].skip, Some("known bug".to_string())); } + #[test] + fn test_parse_expect_raw() { + // Using explicit string to control whitespace precisely + // The content " hello " has 2 leading and 2 trailing spaces + let input = "@database :memory:\n\ntest select-spaces {\n SELECT 1;\n}\nexpect raw {\n hello \n}\n"; + + let file = parse(input).unwrap(); + // Raw mode preserves leading/trailing whitespace + assert!(matches!( + &file.tests[0].expectation, + Expectation::Exact(rows) if rows == &vec![" hello ".to_string()] + )); + } + + #[test] + fn test_parse_expect_raw_vs_normal() { + // Normal mode trims whitespace + let input_normal = r#" +@database :memory: + +test select-1 { + SELECT 1; +} +expect { + hello world +} +"#; + let file_normal = parse(input_normal).unwrap(); + assert!(matches!( + &file_normal.tests[0].expectation, + Expectation::Exact(rows) if rows == &vec!["hello world".to_string()] + )); + + // Raw mode preserves whitespace (4 leading spaces, 2 trailing) + let input_raw = "@database :memory:\n\ntest select-1 {\n SELECT 1;\n}\nexpect raw {\n hello world \n}\n"; + let file_raw = parse(input_raw).unwrap(); + assert!(matches!( + &file_raw.tests[0].expectation, + Expectation::Exact(rows) if rows == &vec![" hello world ".to_string()] + )); + } + #[test] fn test_validation_no_database() { let input = r#"