diff --git a/src/sqlast/mod.rs b/src/sqlast/mod.rs index fe30586b..eef10fcd 100644 --- a/src/sqlast/mod.rs +++ b/src/sqlast/mod.rs @@ -249,6 +249,9 @@ pub enum SQLStatement { name: SQLObjectName, /// Optional schema columns: Vec, + external: bool, + file_format: Option, + location: Option, }, /// ALTER TABLE SQLAlterTable { @@ -361,7 +364,30 @@ impl ToString for SQLStatement { query.to_string() ) } - SQLStatement::SQLCreateTable { name, columns } => format!( + SQLStatement::SQLCreateTable { + name, + columns, + external, + file_format, + location, + } if *external => format!( + "CREATE EXTERNAL TABLE {} ({}) STORED AS {} LOCATION '{}'", + name.to_string(), + columns + .iter() + .map(|c| c.to_string()) + .collect::>() + .join(", "), + file_format.as_ref().map(|f| f.to_string()).unwrap(), + location.as_ref().unwrap() + ), + SQLStatement::SQLCreateTable { + name, + columns, + external: _, + file_format: _, + location: _, + } => format!( "CREATE TABLE {} ({})", name.to_string(), columns @@ -429,3 +455,53 @@ impl ToString for SQLColumnDef { s } } + +/// External table's available file format +#[derive(Debug, Clone, PartialEq)] +pub enum FileFormat { + TEXTFILE, + SEQUENCEFILE, + ORC, + PARQUET, + AVRO, + RCFILE, + JSONFILE, +} + +impl ToString for FileFormat { + fn to_string(&self) -> String { + use self::FileFormat::*; + match self { + TEXTFILE => "TEXTFILE".to_string(), + SEQUENCEFILE => "SEQUENCEFILE".to_string(), + ORC => "ORC".to_string(), + PARQUET => "PARQUET".to_string(), + AVRO => "AVRO".to_string(), + RCFILE => "RCFILE".to_string(), + JSONFILE => "TEXTFILE".to_string(), + } + } +} + +use sqlparser::ParserError; +use std::str::FromStr; +impl FromStr for FileFormat { + type Err = ParserError; + + fn from_str(s: &str) -> Result { + use self::FileFormat::*; + match s { + "TEXTFILE" => Ok(TEXTFILE), + "SEQUENCEFILE" => Ok(SEQUENCEFILE), + "ORC" => Ok(ORC), + "PARQUET" => Ok(PARQUET), + "AVRO" => Ok(AVRO), + "RCFILE" => Ok(RCFILE), + "JSONFILE" => Ok(JSONFILE), + _ => Err(ParserError::ParserError(format!( + "Unexpected file format: {}", + s + ))), + } + } +} diff --git a/src/sqlparser.rs b/src/sqlparser.rs index 22d0e70c..624805a1 100644 --- a/src/sqlparser.rs +++ b/src/sqlparser.rs @@ -620,6 +620,8 @@ impl Parser { } else if self.parse_keyword("MATERIALIZED") || self.parse_keyword("VIEW") { self.prev_token(); self.parse_create_view() + } else if self.parse_keyword("EXTERNAL") { + self.parse_create_external_table() } else { parser_err!(format!( "Unexpected token after CREATE: {:?}", @@ -628,6 +630,26 @@ impl Parser { } } + pub fn parse_create_external_table(&mut self) -> Result { + self.expect_keyword("TABLE")?; + let table_name = self.parse_object_name()?; + let columns = self.parse_columns()?; + self.expect_keyword("STORED")?; + self.expect_keyword("AS")?; + let file_format = self.parse_identifier()?.parse::()?; + + self.expect_keyword("LOCATION")?; + let location = self.parse_literal_string()?; + + Ok(SQLStatement::SQLCreateTable { + name: table_name, + columns, + external: true, + file_format: Some(file_format), + location: Some(location), + }) + } + pub fn parse_create_view(&mut self) -> Result { let materialized = self.parse_keyword("MATERIALIZED"); self.expect_keyword("VIEW")?; @@ -650,62 +672,74 @@ impl Parser { pub fn parse_create_table(&mut self) -> Result { let table_name = self.parse_object_name()?; // parse optional column list (schema) - let mut columns = vec![]; - if self.consume_token(&Token::LParen) { - loop { - match self.next_token() { - Some(Token::SQLWord(column_name)) => { - let data_type = self.parse_data_type()?; - let is_primary = self.parse_keywords(vec!["PRIMARY", "KEY"]); - let is_unique = self.parse_keyword("UNIQUE"); - let default = if self.parse_keyword("DEFAULT") { - let expr = self.parse_default_expr(0)?; - Some(expr) - } else { - None - }; - let allow_null = if self.parse_keywords(vec!["NOT", "NULL"]) { - false - } else if self.parse_keyword("NULL") { - true - } else { - true - }; - debug!("default: {:?}", default); + let columns = self.parse_columns()?; - columns.push(SQLColumnDef { - name: column_name.as_sql_ident(), - data_type: data_type, - allow_null, - is_primary, - is_unique, - default, - }); - match self.next_token() { - Some(Token::Comma) => {} - Some(Token::RParen) => { - break; - } - other => { - return parser_err!(format!( - "Expected ',' or ')' after column definition but found {:?}", - other - )); - } - } - } - unexpected => { - return parser_err!(format!("Expected column name, got {:?}", unexpected)); - } - } - } - } Ok(SQLStatement::SQLCreateTable { name: table_name, columns, + external: false, + file_format: None, + location: None, }) } + fn parse_columns(&mut self) -> Result, ParserError> { + let mut columns = vec![]; + if !self.consume_token(&Token::LParen) { + return Ok(columns); + } + + loop { + match self.next_token() { + Some(Token::SQLWord(column_name)) => { + let data_type = self.parse_data_type()?; + let is_primary = self.parse_keywords(vec!["PRIMARY", "KEY"]); + let is_unique = self.parse_keyword("UNIQUE"); + let default = if self.parse_keyword("DEFAULT") { + let expr = self.parse_default_expr(0)?; + Some(expr) + } else { + None + }; + let allow_null = if self.parse_keywords(vec!["NOT", "NULL"]) { + false + } else if self.parse_keyword("NULL") { + true + } else { + true + }; + debug!("default: {:?}", default); + + columns.push(SQLColumnDef { + name: column_name.as_sql_ident(), + data_type, + allow_null, + is_primary, + is_unique, + default, + }); + match self.next_token() { + Some(Token::Comma) => {} + Some(Token::RParen) => { + break; + } + other => { + return parser_err!(format!( + "Expected ',' or ')' after column definition but found {:?}", + other + )); + } + } + } + unexpected => { + return parser_err!(format!("Expected column name, got {:?}", unexpected)); + } + } + } + + Ok(columns) + } + pub fn parse_table_key(&mut self, constraint_name: SQLIdent) -> Result { let is_primary_key = self.parse_keywords(vec!["PRIMARY", "KEY"]); let is_unique_key = self.parse_keywords(vec!["UNIQUE", "KEY"]); diff --git a/tests/sqlparser_generic.rs b/tests/sqlparser_generic.rs index b3e418a0..9946448f 100644 --- a/tests/sqlparser_generic.rs +++ b/tests/sqlparser_generic.rs @@ -434,7 +434,13 @@ fn parse_create_table() { lng double)", ); match ast { - SQLStatement::SQLCreateTable { name, columns } => { + SQLStatement::SQLCreateTable { + name, + columns, + external: _, + file_format: _, + location: _, + } => { assert_eq!("uk_cities", name.to_string()); assert_eq!(3, columns.len()); @@ -457,6 +463,57 @@ fn parse_create_table() { } } +#[test] +fn parse_create_external_table() { + let sql = String::from( + "CREATE EXTERNAL TABLE uk_cities (\ + name VARCHAR(100) NOT NULL,\ + lat DOUBLE NULL,\ + lng DOUBLE NULL)\ + STORED AS TEXTFILE LOCATION '/tmp/example.csv", + ); + let ast = one_statement_parses_to( + &sql, + "CREATE EXTERNAL TABLE uk_cities (\ + name character varying(100) NOT NULL, \ + lat double, \ + lng double) \ + STORED AS TEXTFILE LOCATION '/tmp/example.csv'", + ); + match ast { + SQLStatement::SQLCreateTable { + name, + columns, + external, + file_format, + location, + } => { + assert_eq!("uk_cities", name.to_string()); + assert_eq!(3, columns.len()); + + let c_name = &columns[0]; + assert_eq!("name", c_name.name); + assert_eq!(SQLType::Varchar(Some(100)), c_name.data_type); + assert_eq!(false, c_name.allow_null); + + let c_lat = &columns[1]; + assert_eq!("lat", c_lat.name); + assert_eq!(SQLType::Double, c_lat.data_type); + assert_eq!(true, c_lat.allow_null); + + let c_lng = &columns[2]; + assert_eq!("lng", c_lng.name); + assert_eq!(SQLType::Double, c_lng.data_type); + assert_eq!(true, c_lng.allow_null); + + assert!(external); + assert_eq!(FileFormat::TEXTFILE, file_format.unwrap()); + assert_eq!("/tmp/example.csv", location.unwrap()); + } + _ => assert!(false), + } +} + #[test] fn parse_scalar_function_in_projection() { let sql = "SELECT sqrt(id) FROM foo"; diff --git a/tests/sqlparser_postgres.rs b/tests/sqlparser_postgres.rs index 80e57176..4c35a3cf 100644 --- a/tests/sqlparser_postgres.rs +++ b/tests/sqlparser_postgres.rs @@ -163,7 +163,13 @@ fn parse_create_table_with_defaults() { active integer NOT NULL)", ); match one_statement_parses_to(&sql, "") { - SQLStatement::SQLCreateTable { name, columns } => { + SQLStatement::SQLCreateTable { + name, + columns, + external: _, + file_format: _, + location: _, + } => { assert_eq!("public.customer", name.to_string()); assert_eq!(10, columns.len()); @@ -204,7 +210,13 @@ fn parse_create_table_from_pg_dump() { active integer )"); match one_statement_parses_to(&sql, "") { - SQLStatement::SQLCreateTable { name, columns } => { + SQLStatement::SQLCreateTable { + name, + columns, + external: _, + file_format: _, + location: _, + } => { assert_eq!("public.customer", name.to_string()); let c_customer_id = &columns[0]; @@ -261,7 +273,13 @@ fn parse_create_table_with_inherit() { )", ); match verified_stmt(&sql) { - SQLStatement::SQLCreateTable { name, columns } => { + SQLStatement::SQLCreateTable { + name, + columns, + external: _, + file_format: _, + location: _, + } => { assert_eq!("bazaar.settings", name.to_string()); let c_name = &columns[0];