mirrors/datafusion-sqlparse
1
0
Fork 0
mirror of https://github.com/apache/datafusion-sqlparser-rs.git synced 2025-08-30 18:57:21 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Merge pull request #46 from zhzy0077/feature/external_table

Browse source 
Thanks @zhzy0077 !
This commit is contained in:
Andy Grove 2019-04-13 11:32:38 -06:00 • committed by GitHub
commit d9591cd999
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 239 additions and 54 deletions
Download patch file Download diff file Expand all files Collapse all files

78
src/sqlast/mod.rs
View file

@ -249,6 +249,9 @@ pub enum SQLStatement {
name: SQLObjectName,
/// Optional schema
columns: Vec<SQLColumnDef>,
external: bool,
file_format: Option<FileFormat>,
location: Option<String>,
},
/// ALTER TABLE
SQLAlterTable {
@ -361,7 +364,30 @@ impl ToString for SQLStatement {
query.to_string()
)
}
SQLStatement::SQLCreateTable { name, columns } => format!(
SQLStatement::SQLCreateTable {
name,
columns,
external,
file_format,
location,
} if *external => format!(
"CREATE EXTERNAL TABLE {} ({}) STORED AS {} LOCATION '{}'",
name.to_string(),
columns
.iter()
.map(|c| c.to_string())
.collect::<Vec<String>>()
.join(", "),
file_format.as_ref().map(|f| f.to_string()).unwrap(),
location.as_ref().unwrap()
),
SQLStatement::SQLCreateTable {
name,
columns,
external: _,
file_format: _,
location: _,
} => format!(
"CREATE TABLE {} ({})",
name.to_string(),
columns
@ -429,3 +455,53 @@ impl ToString for SQLColumnDef {
s
}
}
/// External table's available file format
#[derive(Debug, Clone, PartialEq)]
pub enum FileFormat {
TEXTFILE,
SEQUENCEFILE,
ORC,
PARQUET,
AVRO,
RCFILE,
JSONFILE,
}
impl ToString for FileFormat {
fn to_string(&self) -> String {
use self::FileFormat::*;
match self {
TEXTFILE => "TEXTFILE".to_string(),
SEQUENCEFILE => "SEQUENCEFILE".to_string(),
ORC => "ORC".to_string(),
PARQUET => "PARQUET".to_string(),
AVRO => "AVRO".to_string(),
RCFILE => "RCFILE".to_string(),
JSONFILE => "TEXTFILE".to_string(),
}
}
}
use sqlparser::ParserError;
use std::str::FromStr;
impl FromStr for FileFormat {
type Err = ParserError;
fn from_str(s: &str) -> Result<Self, Self::Err> {
use self::FileFormat::*;
match s {
"TEXTFILE" => Ok(TEXTFILE),
"SEQUENCEFILE" => Ok(SEQUENCEFILE),
"ORC" => Ok(ORC),
"PARQUET" => Ok(PARQUET),
"AVRO" => Ok(AVRO),
"RCFILE" => Ok(RCFILE),
"JSONFILE" => Ok(JSONFILE),
_ => Err(ParserError::ParserError(format!(
"Unexpected file format: {}",
s
))),
}
}
}

132
src/sqlparser.rs
View file

@ -620,6 +620,8 @@ impl Parser {
} else if self.parse_keyword("MATERIALIZED") || self.parse_keyword("VIEW") {
self.prev_token();
self.parse_create_view()
} else if self.parse_keyword("EXTERNAL") {
self.parse_create_external_table()
} else {
parser_err!(format!(
"Unexpected token after CREATE: {:?}",
@ -628,6 +630,26 @@ impl Parser {
}
}
pub fn parse_create_external_table(&mut self) -> Result<SQLStatement, ParserError> {
self.expect_keyword("TABLE")?;
let table_name = self.parse_object_name()?;
let columns = self.parse_columns()?;
self.expect_keyword("STORED")?;
self.expect_keyword("AS")?;
let file_format = self.parse_identifier()?.parse::<FileFormat>()?;
self.expect_keyword("LOCATION")?;
let location = self.parse_literal_string()?;
Ok(SQLStatement::SQLCreateTable {
name: table_name,
columns,
external: true,
file_format: Some(file_format),
location: Some(location),
})
}
pub fn parse_create_view(&mut self) -> Result<SQLStatement, ParserError> {
let materialized = self.parse_keyword("MATERIALIZED");
self.expect_keyword("VIEW")?;
@ -650,62 +672,74 @@ impl Parser {
pub fn parse_create_table(&mut self) -> Result<SQLStatement, ParserError> {
let table_name = self.parse_object_name()?;
// parse optional column list (schema)
let mut columns = vec![];
if self.consume_token(&Token::LParen) {
loop {
match self.next_token() {
Some(Token::SQLWord(column_name)) => {
let data_type = self.parse_data_type()?;
let is_primary = self.parse_keywords(vec!["PRIMARY", "KEY"]);
let is_unique = self.parse_keyword("UNIQUE");
let default = if self.parse_keyword("DEFAULT") {
let expr = self.parse_default_expr(0)?;
Some(expr)
} else {
None
};
let allow_null = if self.parse_keywords(vec!["NOT", "NULL"]) {
false
} else if self.parse_keyword("NULL") {
true
} else {
true
};
debug!("default: {:?}", default);
let columns = self.parse_columns()?;
columns.push(SQLColumnDef {
name: column_name.as_sql_ident(),
data_type: data_type,
allow_null,
is_primary,
is_unique,
default,
});
match self.next_token() {
Some(Token::Comma) => {}
Some(Token::RParen) => {
break;
}
other => {
return parser_err!(format!(
"Expected ',' or ')' after column definition but found {:?}",
other
));
}
}
}
unexpected => {
return parser_err!(format!("Expected column name, got {:?}", unexpected));
}
}
}
}
Ok(SQLStatement::SQLCreateTable {
name: table_name,
columns,
external: false,
file_format: None,
location: None,
})
}
fn parse_columns(&mut self) -> Result<Vec<SQLColumnDef>, ParserError> {
let mut columns = vec![];
if !self.consume_token(&Token::LParen) {
return Ok(columns);
}
loop {
match self.next_token() {
Some(Token::SQLWord(column_name)) => {
let data_type = self.parse_data_type()?;
let is_primary = self.parse_keywords(vec!["PRIMARY", "KEY"]);
let is_unique = self.parse_keyword("UNIQUE");
let default = if self.parse_keyword("DEFAULT") {
let expr = self.parse_default_expr(0)?;
Some(expr)
} else {
None
};
let allow_null = if self.parse_keywords(vec!["NOT", "NULL"]) {
false
} else if self.parse_keyword("NULL") {
true
} else {
true
};
debug!("default: {:?}", default);
columns.push(SQLColumnDef {
name: column_name.as_sql_ident(),
data_type,
allow_null,
is_primary,
is_unique,
default,
});
match self.next_token() {
Some(Token::Comma) => {}
Some(Token::RParen) => {
break;
}
other => {
return parser_err!(format!(
"Expected ',' or ')' after column definition but found {:?}",
other
));
}
}
}
unexpected => {
return parser_err!(format!("Expected column name, got {:?}", unexpected));
}
}
}
Ok(columns)
}
pub fn parse_table_key(&mut self, constraint_name: SQLIdent) -> Result<TableKey, ParserError> {
let is_primary_key = self.parse_keywords(vec!["PRIMARY", "KEY"]);
let is_unique_key = self.parse_keywords(vec!["UNIQUE", "KEY"]);

59
tests/sqlparser_generic.rs
View file

@ -434,7 +434,13 @@ fn parse_create_table() {
lng double)",
);
match ast {
SQLStatement::SQLCreateTable { name, columns } => {
SQLStatement::SQLCreateTable {
name,
columns,
external: _,
file_format: _,
location: _,
} => {
assert_eq!("uk_cities", name.to_string());
assert_eq!(3, columns.len());
@ -457,6 +463,57 @@ fn parse_create_table() {
}
}
#[test]
fn parse_create_external_table() {
let sql = String::from(
"CREATE EXTERNAL TABLE uk_cities (\
name VARCHAR(100) NOT NULL,\
lat DOUBLE NULL,\
lng DOUBLE NULL)\
STORED AS TEXTFILE LOCATION '/tmp/example.csv",
);
let ast = one_statement_parses_to(
&sql,
"CREATE EXTERNAL TABLE uk_cities (\
name character varying(100) NOT NULL, \
lat double, \
lng double) \
STORED AS TEXTFILE LOCATION '/tmp/example.csv'",
);
match ast {
SQLStatement::SQLCreateTable {
name,
columns,
external,
file_format,
location,
} => {
assert_eq!("uk_cities", name.to_string());
assert_eq!(3, columns.len());
let c_name = &columns[0];
assert_eq!("name", c_name.name);
assert_eq!(SQLType::Varchar(Some(100)), c_name.data_type);
assert_eq!(false, c_name.allow_null);
let c_lat = &columns[1];
assert_eq!("lat", c_lat.name);
assert_eq!(SQLType::Double, c_lat.data_type);
assert_eq!(true, c_lat.allow_null);
let c_lng = &columns[2];
assert_eq!("lng", c_lng.name);
assert_eq!(SQLType::Double, c_lng.data_type);
assert_eq!(true, c_lng.allow_null);
assert!(external);
assert_eq!(FileFormat::TEXTFILE, file_format.unwrap());
assert_eq!("/tmp/example.csv", location.unwrap());
}
_ => assert!(false),
}
}
#[test]
fn parse_scalar_function_in_projection() {
let sql = "SELECT sqrt(id) FROM foo";

24
tests/sqlparser_postgres.rs
View file

@ -163,7 +163,13 @@ fn parse_create_table_with_defaults() {
active integer NOT NULL)",
);
match one_statement_parses_to(&sql, "") {
SQLStatement::SQLCreateTable { name, columns } => {
SQLStatement::SQLCreateTable {
name,
columns,
external: _,
file_format: _,
location: _,
} => {
assert_eq!("public.customer", name.to_string());
assert_eq!(10, columns.len());
@ -204,7 +210,13 @@ fn parse_create_table_from_pg_dump() {
active integer
)");
match one_statement_parses_to(&sql, "") {
SQLStatement::SQLCreateTable { name, columns } => {
SQLStatement::SQLCreateTable {
name,
columns,
external: _,
file_format: _,
location: _,
} => {
assert_eq!("public.customer", name.to_string());
let c_customer_id = &columns[0];
@ -261,7 +273,13 @@ fn parse_create_table_with_inherit() {
)",
);
match verified_stmt(&sql) {
SQLStatement::SQLCreateTable { name, columns } => {
SQLStatement::SQLCreateTable {
name,
columns,
external: _,
file_format: _,
location: _,
} => {
assert_eq!("bazaar.settings", name.to_string());
let c_name = &columns[0];