mirror of
https://github.com/apache/datafusion-sqlparser-rs.git
synced 2025-10-09 21:42:05 +00:00
Improve parsing of JSON accesses on Postgres and Snowflake (#1215)
Co-authored-by: Ifeanyi Ubah <ify1992@yahoo.com>
This commit is contained in:
Joey Hain
GitHub
parent
0606024353
commit
4bfa399919
7 changed files with 432 additions and 199 deletions
|
|
@ -1375,25 +1375,25 @@ fn pg_and_generic() -> TestedDialects {
|
|||
|
||||
#[test]
|
||||
fn parse_json_ops_without_colon() {
|
||||
use self::JsonOperator;
|
||||
let binary_ops = &[
|
||||
("->", JsonOperator::Arrow, all_dialects()),
|
||||
("->>", JsonOperator::LongArrow, all_dialects()),
|
||||
("#>", JsonOperator::HashArrow, pg_and_generic()),
|
||||
("#>>", JsonOperator::HashLongArrow, pg_and_generic()),
|
||||
("@>", JsonOperator::AtArrow, all_dialects()),
|
||||
("<@", JsonOperator::ArrowAt, all_dialects()),
|
||||
("#-", JsonOperator::HashMinus, pg_and_generic()),
|
||||
("@?", JsonOperator::AtQuestion, all_dialects()),
|
||||
("@@", JsonOperator::AtAt, all_dialects()),
|
||||
use self::BinaryOperator::*;
|
||||
let binary_ops = [
|
||||
("->", Arrow, all_dialects()),
|
||||
("->>", LongArrow, all_dialects()),
|
||||
("#>", HashArrow, pg_and_generic()),
|
||||
("#>>", HashLongArrow, pg_and_generic()),
|
||||
("@>", AtArrow, all_dialects()),
|
||||
("<@", ArrowAt, all_dialects()),
|
||||
("#-", HashMinus, pg_and_generic()),
|
||||
("@?", AtQuestion, all_dialects()),
|
||||
("@@", AtAt, all_dialects()),
|
||||
];
|
||||
|
||||
for (str_op, op, dialects) in binary_ops {
|
||||
let select = dialects.verified_only_select(&format!("SELECT a {} b", &str_op));
|
||||
assert_eq!(
|
||||
SelectItem::UnnamedExpr(Expr::JsonAccess {
|
||||
SelectItem::UnnamedExpr(Expr::BinaryOp {
|
||||
left: Box::new(Expr::Identifier(Ident::new("a"))),
|
||||
operator: *op,
|
||||
op,
|
||||
right: Box::new(Expr::Identifier(Ident::new("b"))),
|
||||
}),
|
||||
select.projection[0]
|
||||
|
|
|
|||
|
|
@ -2235,9 +2235,9 @@ fn test_json() {
|
|||
let sql = "SELECT params ->> 'name' FROM events";
|
||||
let select = pg().verified_only_select(sql);
|
||||
assert_eq!(
|
||||
SelectItem::UnnamedExpr(Expr::JsonAccess {
|
||||
SelectItem::UnnamedExpr(Expr::BinaryOp {
|
||||
left: Box::new(Expr::Identifier(Ident::new("params"))),
|
||||
operator: JsonOperator::LongArrow,
|
||||
op: BinaryOperator::LongArrow,
|
||||
right: Box::new(Expr::Value(Value::SingleQuotedString("name".to_string()))),
|
||||
}),
|
||||
select.projection[0]
|
||||
|
|
@ -2246,9 +2246,9 @@ fn test_json() {
|
|||
let sql = "SELECT params -> 'name' FROM events";
|
||||
let select = pg().verified_only_select(sql);
|
||||
assert_eq!(
|
||||
SelectItem::UnnamedExpr(Expr::JsonAccess {
|
||||
SelectItem::UnnamedExpr(Expr::BinaryOp {
|
||||
left: Box::new(Expr::Identifier(Ident::new("params"))),
|
||||
operator: JsonOperator::Arrow,
|
||||
op: BinaryOperator::Arrow,
|
||||
right: Box::new(Expr::Value(Value::SingleQuotedString("name".to_string()))),
|
||||
}),
|
||||
select.projection[0]
|
||||
|
|
@ -2257,15 +2257,55 @@ fn test_json() {
|
|||
let sql = "SELECT info -> 'items' ->> 'product' FROM orders";
|
||||
let select = pg().verified_only_select(sql);
|
||||
assert_eq!(
|
||||
SelectItem::UnnamedExpr(Expr::JsonAccess {
|
||||
left: Box::new(Expr::Identifier(Ident::new("info"))),
|
||||
operator: JsonOperator::Arrow,
|
||||
right: Box::new(Expr::JsonAccess {
|
||||
left: Box::new(Expr::Value(Value::SingleQuotedString("items".to_string()))),
|
||||
operator: JsonOperator::LongArrow,
|
||||
right: Box::new(Expr::Value(Value::SingleQuotedString(
|
||||
"product".to_string()
|
||||
)))
|
||||
SelectItem::UnnamedExpr(Expr::BinaryOp {
|
||||
left: Box::new(Expr::BinaryOp {
|
||||
left: Box::new(Expr::Identifier(Ident::new("info"))),
|
||||
op: BinaryOperator::Arrow,
|
||||
right: Box::new(Expr::Value(Value::SingleQuotedString("items".to_string())))
|
||||
}),
|
||||
op: BinaryOperator::LongArrow,
|
||||
right: Box::new(Expr::Value(Value::SingleQuotedString(
|
||||
"product".to_string()
|
||||
))),
|
||||
}),
|
||||
select.projection[0]
|
||||
);
|
||||
|
||||
// the RHS can be a number (array element access)
|
||||
let sql = "SELECT obj -> 42";
|
||||
let select = pg().verified_only_select(sql);
|
||||
assert_eq!(
|
||||
SelectItem::UnnamedExpr(Expr::BinaryOp {
|
||||
left: Box::new(Expr::Identifier(Ident::new("obj"))),
|
||||
op: BinaryOperator::Arrow,
|
||||
right: Box::new(Expr::Value(number("42"))),
|
||||
}),
|
||||
select.projection[0]
|
||||
);
|
||||
|
||||
// the RHS can be an identifier
|
||||
let sql = "SELECT obj -> key";
|
||||
let select = pg().verified_only_select(sql);
|
||||
assert_eq!(
|
||||
SelectItem::UnnamedExpr(Expr::BinaryOp {
|
||||
left: Box::new(Expr::Identifier(Ident::new("obj"))),
|
||||
op: BinaryOperator::Arrow,
|
||||
right: Box::new(Expr::Identifier(Ident::new("key"))),
|
||||
}),
|
||||
select.projection[0]
|
||||
);
|
||||
|
||||
// -> operator has lower precedence than arithmetic ops
|
||||
let sql = "SELECT obj -> 3 * 2";
|
||||
let select = pg().verified_only_select(sql);
|
||||
assert_eq!(
|
||||
SelectItem::UnnamedExpr(Expr::BinaryOp {
|
||||
left: Box::new(Expr::Identifier(Ident::new("obj"))),
|
||||
op: BinaryOperator::Arrow,
|
||||
right: Box::new(Expr::BinaryOp {
|
||||
left: Box::new(Expr::Value(number("3"))),
|
||||
op: BinaryOperator::Multiply,
|
||||
right: Box::new(Expr::Value(number("2"))),
|
||||
}),
|
||||
}),
|
||||
select.projection[0]
|
||||
|
|
@ -2274,9 +2314,9 @@ fn test_json() {
|
|||
let sql = "SELECT info #> '{a,b,c}' FROM orders";
|
||||
let select = pg().verified_only_select(sql);
|
||||
assert_eq!(
|
||||
SelectItem::UnnamedExpr(Expr::JsonAccess {
|
||||
SelectItem::UnnamedExpr(Expr::BinaryOp {
|
||||
left: Box::new(Expr::Identifier(Ident::new("info"))),
|
||||
operator: JsonOperator::HashArrow,
|
||||
op: BinaryOperator::HashArrow,
|
||||
right: Box::new(Expr::Value(Value::SingleQuotedString(
|
||||
"{a,b,c}".to_string()
|
||||
))),
|
||||
|
|
@ -2287,9 +2327,9 @@ fn test_json() {
|
|||
let sql = "SELECT info #>> '{a,b,c}' FROM orders";
|
||||
let select = pg().verified_only_select(sql);
|
||||
assert_eq!(
|
||||
SelectItem::UnnamedExpr(Expr::JsonAccess {
|
||||
SelectItem::UnnamedExpr(Expr::BinaryOp {
|
||||
left: Box::new(Expr::Identifier(Ident::new("info"))),
|
||||
operator: JsonOperator::HashLongArrow,
|
||||
op: BinaryOperator::HashLongArrow,
|
||||
right: Box::new(Expr::Value(Value::SingleQuotedString(
|
||||
"{a,b,c}".to_string()
|
||||
))),
|
||||
|
|
@ -2300,9 +2340,9 @@ fn test_json() {
|
|||
let sql = "SELECT info FROM orders WHERE info @> '{\"a\": 1}'";
|
||||
let select = pg().verified_only_select(sql);
|
||||
assert_eq!(
|
||||
Expr::JsonAccess {
|
||||
Expr::BinaryOp {
|
||||
left: Box::new(Expr::Identifier(Ident::new("info"))),
|
||||
operator: JsonOperator::AtArrow,
|
||||
op: BinaryOperator::AtArrow,
|
||||
right: Box::new(Expr::Value(Value::SingleQuotedString(
|
||||
"{\"a\": 1}".to_string()
|
||||
))),
|
||||
|
|
@ -2313,11 +2353,11 @@ fn test_json() {
|
|||
let sql = "SELECT info FROM orders WHERE '{\"a\": 1}' <@ info";
|
||||
let select = pg().verified_only_select(sql);
|
||||
assert_eq!(
|
||||
Expr::JsonAccess {
|
||||
Expr::BinaryOp {
|
||||
left: Box::new(Expr::Value(Value::SingleQuotedString(
|
||||
"{\"a\": 1}".to_string()
|
||||
))),
|
||||
operator: JsonOperator::ArrowAt,
|
||||
op: BinaryOperator::ArrowAt,
|
||||
right: Box::new(Expr::Identifier(Ident::new("info"))),
|
||||
},
|
||||
select.selection.unwrap(),
|
||||
|
|
@ -2326,9 +2366,9 @@ fn test_json() {
|
|||
let sql = "SELECT info #- ARRAY['a', 'b'] FROM orders";
|
||||
let select = pg().verified_only_select(sql);
|
||||
assert_eq!(
|
||||
SelectItem::UnnamedExpr(Expr::JsonAccess {
|
||||
SelectItem::UnnamedExpr(Expr::BinaryOp {
|
||||
left: Box::new(Expr::Identifier(Ident::from("info"))),
|
||||
operator: JsonOperator::HashMinus,
|
||||
op: BinaryOperator::HashMinus,
|
||||
right: Box::new(Expr::Array(Array {
|
||||
elem: vec![
|
||||
Expr::Value(Value::SingleQuotedString("a".to_string())),
|
||||
|
|
@ -2343,9 +2383,9 @@ fn test_json() {
|
|||
let sql = "SELECT info FROM orders WHERE info @? '$.a'";
|
||||
let select = pg().verified_only_select(sql);
|
||||
assert_eq!(
|
||||
Expr::JsonAccess {
|
||||
Expr::BinaryOp {
|
||||
left: Box::new(Expr::Identifier(Ident::from("info"))),
|
||||
operator: JsonOperator::AtQuestion,
|
||||
op: BinaryOperator::AtQuestion,
|
||||
right: Box::new(Expr::Value(Value::SingleQuotedString("$.a".to_string())),),
|
||||
},
|
||||
select.selection.unwrap(),
|
||||
|
|
@ -2354,9 +2394,9 @@ fn test_json() {
|
|||
let sql = "SELECT info FROM orders WHERE info @@ '$.a'";
|
||||
let select = pg().verified_only_select(sql);
|
||||
assert_eq!(
|
||||
Expr::JsonAccess {
|
||||
Expr::BinaryOp {
|
||||
left: Box::new(Expr::Identifier(Ident::from("info"))),
|
||||
operator: JsonOperator::AtAt,
|
||||
op: BinaryOperator::AtAt,
|
||||
right: Box::new(Expr::Value(Value::SingleQuotedString("$.a".to_string())),),
|
||||
},
|
||||
select.selection.unwrap(),
|
||||
|
|
|
|||
|
|
@ -183,71 +183,167 @@ fn parse_lateral_flatten() {
|
|||
snowflake().verified_only_select(r#"SELECT emp.employee_ID, emp.last_name, index, value AS project_name FROM employees AS emp, LATERAL FLATTEN(INPUT => emp.project_names) AS proj_names"#);
|
||||
}
|
||||
|
||||
// https://docs.snowflake.com/en/user-guide/querying-semistructured
|
||||
#[test]
|
||||
fn parse_json_using_colon() {
|
||||
fn parse_semi_structured_data_traversal() {
|
||||
// most basic case
|
||||
let sql = "SELECT a:b FROM t";
|
||||
let select = snowflake().verified_only_select(sql);
|
||||
assert_eq!(
|
||||
SelectItem::UnnamedExpr(Expr::JsonAccess {
|
||||
left: Box::new(Expr::Identifier(Ident::new("a"))),
|
||||
operator: JsonOperator::Colon,
|
||||
right: Box::new(Expr::Value(Value::UnQuotedString("b".to_string()))),
|
||||
value: Box::new(Expr::Identifier(Ident::new("a"))),
|
||||
path: JsonPath {
|
||||
path: vec![JsonPathElem::Dot {
|
||||
key: "b".to_owned(),
|
||||
quoted: false
|
||||
}]
|
||||
},
|
||||
}),
|
||||
select.projection[0]
|
||||
);
|
||||
|
||||
let sql = "SELECT a:type FROM t";
|
||||
// identifier can be quoted
|
||||
let sql = r#"SELECT a:"my long object key name" FROM t"#;
|
||||
let select = snowflake().verified_only_select(sql);
|
||||
assert_eq!(
|
||||
SelectItem::UnnamedExpr(Expr::JsonAccess {
|
||||
left: Box::new(Expr::Identifier(Ident::new("a"))),
|
||||
operator: JsonOperator::Colon,
|
||||
right: Box::new(Expr::Value(Value::UnQuotedString("type".to_string()))),
|
||||
value: Box::new(Expr::Identifier(Ident::new("a"))),
|
||||
path: JsonPath {
|
||||
path: vec![JsonPathElem::Dot {
|
||||
key: "my long object key name".to_owned(),
|
||||
quoted: true
|
||||
}]
|
||||
},
|
||||
}),
|
||||
select.projection[0]
|
||||
);
|
||||
|
||||
let sql = "SELECT a:location FROM t";
|
||||
// expressions are allowed in bracket notation
|
||||
let sql = r#"SELECT a[2 + 2] FROM t"#;
|
||||
let select = snowflake().verified_only_select(sql);
|
||||
assert_eq!(
|
||||
SelectItem::UnnamedExpr(Expr::JsonAccess {
|
||||
left: Box::new(Expr::Identifier(Ident::new("a"))),
|
||||
operator: JsonOperator::Colon,
|
||||
right: Box::new(Expr::Value(Value::UnQuotedString("location".to_string()))),
|
||||
}),
|
||||
select.projection[0]
|
||||
);
|
||||
|
||||
let sql = "SELECT a:date FROM t";
|
||||
let select = snowflake().verified_only_select(sql);
|
||||
assert_eq!(
|
||||
SelectItem::UnnamedExpr(Expr::JsonAccess {
|
||||
left: Box::new(Expr::Identifier(Ident::new("a"))),
|
||||
operator: JsonOperator::Colon,
|
||||
right: Box::new(Expr::Value(Value::UnQuotedString("date".to_string()))),
|
||||
value: Box::new(Expr::Identifier(Ident::new("a"))),
|
||||
path: JsonPath {
|
||||
path: vec![JsonPathElem::Bracket {
|
||||
key: Expr::BinaryOp {
|
||||
left: Box::new(Expr::Value(number("2"))),
|
||||
op: BinaryOperator::Plus,
|
||||
right: Box::new(Expr::Value(number("2")))
|
||||
},
|
||||
}]
|
||||
},
|
||||
}),
|
||||
select.projection[0]
|
||||
);
|
||||
|
||||
snowflake().verified_stmt("SELECT a:b::INT FROM t");
|
||||
|
||||
let sql = "SELECT a:start, a:end FROM t";
|
||||
// unquoted keywords are permitted in the object key
|
||||
let sql = "SELECT a:select, a:from FROM t";
|
||||
let select = snowflake().verified_only_select(sql);
|
||||
assert_eq!(
|
||||
vec![
|
||||
SelectItem::UnnamedExpr(Expr::JsonAccess {
|
||||
left: Box::new(Expr::Identifier(Ident::new("a"))),
|
||||
operator: JsonOperator::Colon,
|
||||
right: Box::new(Expr::Value(Value::UnQuotedString("start".to_string()))),
|
||||
value: Box::new(Expr::Identifier(Ident::new("a"))),
|
||||
path: JsonPath {
|
||||
path: vec![JsonPathElem::Dot {
|
||||
key: "select".to_owned(),
|
||||
quoted: false
|
||||
}]
|
||||
},
|
||||
}),
|
||||
SelectItem::UnnamedExpr(Expr::JsonAccess {
|
||||
left: Box::new(Expr::Identifier(Ident::new("a"))),
|
||||
operator: JsonOperator::Colon,
|
||||
right: Box::new(Expr::Value(Value::UnQuotedString("end".to_string()))),
|
||||
value: Box::new(Expr::Identifier(Ident::new("a"))),
|
||||
path: JsonPath {
|
||||
path: vec![JsonPathElem::Dot {
|
||||
key: "from".to_owned(),
|
||||
quoted: false
|
||||
}]
|
||||
},
|
||||
})
|
||||
],
|
||||
select.projection
|
||||
);
|
||||
|
||||
// multiple levels can be traversed
|
||||
// https://docs.snowflake.com/en/user-guide/querying-semistructured#dot-notation
|
||||
let sql = r#"SELECT a:foo."bar".baz"#;
|
||||
let select = snowflake().verified_only_select(sql);
|
||||
assert_eq!(
|
||||
vec![SelectItem::UnnamedExpr(Expr::JsonAccess {
|
||||
value: Box::new(Expr::Identifier(Ident::new("a"))),
|
||||
path: JsonPath {
|
||||
path: vec![
|
||||
JsonPathElem::Dot {
|
||||
key: "foo".to_owned(),
|
||||
quoted: false,
|
||||
},
|
||||
JsonPathElem::Dot {
|
||||
key: "bar".to_owned(),
|
||||
quoted: true,
|
||||
},
|
||||
JsonPathElem::Dot {
|
||||
key: "baz".to_owned(),
|
||||
quoted: false,
|
||||
}
|
||||
]
|
||||
},
|
||||
})],
|
||||
select.projection
|
||||
);
|
||||
|
||||
// dot and bracket notation can be mixed (starting with : case)
|
||||
// https://docs.snowflake.com/en/user-guide/querying-semistructured#dot-notation
|
||||
let sql = r#"SELECT a:foo[0].bar"#;
|
||||
let select = snowflake().verified_only_select(sql);
|
||||
assert_eq!(
|
||||
vec![SelectItem::UnnamedExpr(Expr::JsonAccess {
|
||||
value: Box::new(Expr::Identifier(Ident::new("a"))),
|
||||
path: JsonPath {
|
||||
path: vec![
|
||||
JsonPathElem::Dot {
|
||||
key: "foo".to_owned(),
|
||||
quoted: false,
|
||||
},
|
||||
JsonPathElem::Bracket {
|
||||
key: Expr::Value(number("0")),
|
||||
},
|
||||
JsonPathElem::Dot {
|
||||
key: "bar".to_owned(),
|
||||
quoted: false,
|
||||
}
|
||||
]
|
||||
},
|
||||
})],
|
||||
select.projection
|
||||
);
|
||||
|
||||
// dot and bracket notation can be mixed (starting with bracket case)
|
||||
// https://docs.snowflake.com/en/user-guide/querying-semistructured#dot-notation
|
||||
let sql = r#"SELECT a[0].foo.bar"#;
|
||||
let select = snowflake().verified_only_select(sql);
|
||||
assert_eq!(
|
||||
vec![SelectItem::UnnamedExpr(Expr::JsonAccess {
|
||||
value: Box::new(Expr::Identifier(Ident::new("a"))),
|
||||
path: JsonPath {
|
||||
path: vec![
|
||||
JsonPathElem::Bracket {
|
||||
key: Expr::Value(number("0")),
|
||||
},
|
||||
JsonPathElem::Dot {
|
||||
key: "foo".to_owned(),
|
||||
quoted: false,
|
||||
},
|
||||
JsonPathElem::Dot {
|
||||
key: "bar".to_owned(),
|
||||
quoted: false,
|
||||
}
|
||||
]
|
||||
},
|
||||
})],
|
||||
select.projection
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue