Merge pull request #46 from zhzy0077/feature/external_table

Thanks @zhzy0077 !

src/sqlast/mod.rs

@@ -249,6 +249,9 @@ pub enum SQLStatement {
         name: SQLObjectName,
         /// Optional schema
         columns: Vec<SQLColumnDef>,
+        external: bool,
+        file_format: Option<FileFormat>,
+        location: Option<String>,
     },
     /// ALTER TABLE
     SQLAlterTable {
@@ -361,7 +364,30 @@ impl ToString for SQLStatement {
                     query.to_string()
                 )
             }
-            SQLStatement::SQLCreateTable { name, columns } => format!(
+            SQLStatement::SQLCreateTable {
+                name,
+                columns,
+                external,
+                file_format,
+                location,
+            } if *external => format!(
+                "CREATE EXTERNAL TABLE {} ({}) STORED AS {} LOCATION '{}'",
+                name.to_string(),
+                columns
+                    .iter()
+                    .map(|c| c.to_string())
+                    .collect::<Vec<String>>()
+                    .join(", "),
+                file_format.as_ref().map(|f| f.to_string()).unwrap(),
+                location.as_ref().unwrap()
+            ),
+            SQLStatement::SQLCreateTable {
+                name,
+                columns,
+                external: _,
+                file_format: _,
+                location: _,
+            } => format!(
                 "CREATE TABLE {} ({})",
                 name.to_string(),
                 columns
@@ -429,3 +455,53 @@ impl ToString for SQLColumnDef {
         s
     }
 }
+
+/// External table's available file format
+#[derive(Debug, Clone, PartialEq)]
+pub enum FileFormat {
+    TEXTFILE,
+    SEQUENCEFILE,
+    ORC,
+    PARQUET,
+    AVRO,
+    RCFILE,
+    JSONFILE,
+}
+
+impl ToString for FileFormat {
+    fn to_string(&self) -> String {
+        use self::FileFormat::*;
+        match self {
+            TEXTFILE => "TEXTFILE".to_string(),
+            SEQUENCEFILE => "SEQUENCEFILE".to_string(),
+            ORC => "ORC".to_string(),
+            PARQUET => "PARQUET".to_string(),
+            AVRO => "AVRO".to_string(),
+            RCFILE => "RCFILE".to_string(),
+            JSONFILE => "TEXTFILE".to_string(),
+        }
+    }
+}
+
+use sqlparser::ParserError;
+use std::str::FromStr;
+impl FromStr for FileFormat {
+    type Err = ParserError;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        use self::FileFormat::*;
+        match s {
+            "TEXTFILE" => Ok(TEXTFILE),
+            "SEQUENCEFILE" => Ok(SEQUENCEFILE),
+            "ORC" => Ok(ORC),
+            "PARQUET" => Ok(PARQUET),
+            "AVRO" => Ok(AVRO),
+            "RCFILE" => Ok(RCFILE),
+            "JSONFILE" => Ok(JSONFILE),
+            _ => Err(ParserError::ParserError(format!(
+                "Unexpected file format: {}",
+                s
+            ))),
+        }
+    }
+}

src/sqlparser.rs

@@ -620,6 +620,8 @@ impl Parser {
         } else if self.parse_keyword("MATERIALIZED") || self.parse_keyword("VIEW") {
             self.prev_token();
             self.parse_create_view()
+        } else if self.parse_keyword("EXTERNAL") {
+            self.parse_create_external_table()
         } else {
             parser_err!(format!(
                 "Unexpected token after CREATE: {:?}",
@@ -628,6 +630,26 @@ impl Parser {
         }
     }
 
+    pub fn parse_create_external_table(&mut self) -> Result<SQLStatement, ParserError> {
+        self.expect_keyword("TABLE")?;
+        let table_name = self.parse_object_name()?;
+        let columns = self.parse_columns()?;
+        self.expect_keyword("STORED")?;
+        self.expect_keyword("AS")?;
+        let file_format = self.parse_identifier()?.parse::<FileFormat>()?;
+
+        self.expect_keyword("LOCATION")?;
+        let location = self.parse_literal_string()?;
+
+        Ok(SQLStatement::SQLCreateTable {
+            name: table_name,
+            columns,
+            external: true,
+            file_format: Some(file_format),
+            location: Some(location),
+        })
+    }
+
     pub fn parse_create_view(&mut self) -> Result<SQLStatement, ParserError> {
         let materialized = self.parse_keyword("MATERIALIZED");
         self.expect_keyword("VIEW")?;
@@ -650,62 +672,74 @@ impl Parser {
     pub fn parse_create_table(&mut self) -> Result<SQLStatement, ParserError> {
         let table_name = self.parse_object_name()?;
         // parse optional column list (schema)
-        let mut columns = vec![];
+        let columns = self.parse_columns()?;
-        if self.consume_token(&Token::LParen) {
-            loop {
-                match self.next_token() {
-                    Some(Token::SQLWord(column_name)) => {
-                        let data_type = self.parse_data_type()?;
-                        let is_primary = self.parse_keywords(vec!["PRIMARY", "KEY"]);
-                        let is_unique = self.parse_keyword("UNIQUE");
-                        let default = if self.parse_keyword("DEFAULT") {
-                            let expr = self.parse_default_expr(0)?;
-                            Some(expr)
-                        } else {
-                            None
-                        };
-                        let allow_null = if self.parse_keywords(vec!["NOT", "NULL"]) {
-                            false
-                        } else if self.parse_keyword("NULL") {
-                            true
-                        } else {
-                            true
-                        };
-                        debug!("default: {:?}", default);
 
-                        columns.push(SQLColumnDef {
-                            name: column_name.as_sql_ident(),
-                            data_type: data_type,
-                            allow_null,
-                            is_primary,
-                            is_unique,
-                            default,
-                        });
-                        match self.next_token() {
-                            Some(Token::Comma) => {}
-                            Some(Token::RParen) => {
-                                break;
-                            }
-                            other => {
-                                return parser_err!(format!(
-                                    "Expected ',' or ')' after column definition but found {:?}",
-                                    other
-                                ));
-                            }
-                        }
-                    }
-                    unexpected => {
-                        return parser_err!(format!("Expected column name, got {:?}", unexpected));
-                    }
-                }
-            }
-        }
         Ok(SQLStatement::SQLCreateTable {
             name: table_name,
             columns,
+            external: false,
+            file_format: None,
+            location: None,
         })
     }
 
+    fn parse_columns(&mut self) -> Result<Vec<SQLColumnDef>, ParserError> {
+        let mut columns = vec![];
+        if !self.consume_token(&Token::LParen) {
+            return Ok(columns);
+        }
+
+        loop {
+            match self.next_token() {
+                Some(Token::SQLWord(column_name)) => {
+                    let data_type = self.parse_data_type()?;
+                    let is_primary = self.parse_keywords(vec!["PRIMARY", "KEY"]);
+                    let is_unique = self.parse_keyword("UNIQUE");
+                    let default = if self.parse_keyword("DEFAULT") {
+                        let expr = self.parse_default_expr(0)?;
+                        Some(expr)
+                    } else {
+                        None
+                    };
+                    let allow_null = if self.parse_keywords(vec!["NOT", "NULL"]) {
+                        false
+                    } else if self.parse_keyword("NULL") {
+                        true
+                    } else {
+                        true
+                    };
+                    debug!("default: {:?}", default);
+
+                    columns.push(SQLColumnDef {
+                        name: column_name.as_sql_ident(),
+                        data_type,
+                        allow_null,
+                        is_primary,
+                        is_unique,
+                        default,
+                    });
+                    match self.next_token() {
+                        Some(Token::Comma) => {}
+                        Some(Token::RParen) => {
+                            break;
+                        }
+                        other => {
+                            return parser_err!(format!(
+                                "Expected ',' or ')' after column definition but found {:?}",
+                                other
+                            ));
+                        }
+                    }
+                }
+                unexpected => {
+                    return parser_err!(format!("Expected column name, got {:?}", unexpected));
+                }
+            }
+        }
+
+        Ok(columns)
+    }
+
     pub fn parse_table_key(&mut self, constraint_name: SQLIdent) -> Result<TableKey, ParserError> {
         let is_primary_key = self.parse_keywords(vec!["PRIMARY", "KEY"]);
         let is_unique_key = self.parse_keywords(vec!["UNIQUE", "KEY"]);

tests/sqlparser_generic.rs

@@ -434,7 +434,13 @@ fn parse_create_table() {
          lng double)",
     );
     match ast {
-        SQLStatement::SQLCreateTable { name, columns } => {
+        SQLStatement::SQLCreateTable {
+            name,
+            columns,
+            external: _,
+            file_format: _,
+            location: _,
+        } => {
             assert_eq!("uk_cities", name.to_string());
             assert_eq!(3, columns.len());
 
@@ -457,6 +463,57 @@ fn parse_create_table() {
     }
 }
 
+#[test]
+fn parse_create_external_table() {
+    let sql = String::from(
+        "CREATE EXTERNAL TABLE uk_cities (\
+         name VARCHAR(100) NOT NULL,\
+         lat DOUBLE NULL,\
+         lng DOUBLE NULL)\
+         STORED AS TEXTFILE LOCATION '/tmp/example.csv",
+    );
+    let ast = one_statement_parses_to(
+        &sql,
+        "CREATE EXTERNAL TABLE uk_cities (\
+         name character varying(100) NOT NULL, \
+         lat double, \
+         lng double) \
+         STORED AS TEXTFILE LOCATION '/tmp/example.csv'",
+    );
+    match ast {
+        SQLStatement::SQLCreateTable {
+            name,
+            columns,
+            external,
+            file_format,
+            location,
+        } => {
+            assert_eq!("uk_cities", name.to_string());
+            assert_eq!(3, columns.len());
+
+            let c_name = &columns[0];
+            assert_eq!("name", c_name.name);
+            assert_eq!(SQLType::Varchar(Some(100)), c_name.data_type);
+            assert_eq!(false, c_name.allow_null);
+
+            let c_lat = &columns[1];
+            assert_eq!("lat", c_lat.name);
+            assert_eq!(SQLType::Double, c_lat.data_type);
+            assert_eq!(true, c_lat.allow_null);
+
+            let c_lng = &columns[2];
+            assert_eq!("lng", c_lng.name);
+            assert_eq!(SQLType::Double, c_lng.data_type);
+            assert_eq!(true, c_lng.allow_null);
+
+            assert!(external);
+            assert_eq!(FileFormat::TEXTFILE, file_format.unwrap());
+            assert_eq!("/tmp/example.csv", location.unwrap());
+        }
+        _ => assert!(false),
+    }
+}
+
 #[test]
 fn parse_scalar_function_in_projection() {
     let sql = "SELECT sqrt(id) FROM foo";

tests/sqlparser_postgres.rs

@@ -163,7 +163,13 @@ fn parse_create_table_with_defaults() {
             active integer NOT NULL)",
     );
     match one_statement_parses_to(&sql, "") {
-        SQLStatement::SQLCreateTable { name, columns } => {
+        SQLStatement::SQLCreateTable {
+            name,
+            columns,
+            external: _,
+            file_format: _,
+            location: _,
+        } => {
             assert_eq!("public.customer", name.to_string());
             assert_eq!(10, columns.len());
 
@@ -204,7 +210,13 @@ fn parse_create_table_from_pg_dump() {
             active integer
         )");
     match one_statement_parses_to(&sql, "") {
-        SQLStatement::SQLCreateTable { name, columns } => {
+        SQLStatement::SQLCreateTable {
+            name,
+            columns,
+            external: _,
+            file_format: _,
+            location: _,
+        } => {
             assert_eq!("public.customer", name.to_string());
 
             let c_customer_id = &columns[0];
@@ -261,7 +273,13 @@ fn parse_create_table_with_inherit() {
          )",
     );
     match verified_stmt(&sql) {
-        SQLStatement::SQLCreateTable { name, columns } => {
+        SQLStatement::SQLCreateTable {
+            name,
+            columns,
+            external: _,
+            file_format: _,
+            location: _,
+        } => {
             assert_eq!("bazaar.settings", name.to_string());
 
             let c_name = &columns[0];