bill/AQuery
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

bug fixes& code cleanups on parser

Browse Source 
initial code-gen
This commit is contained in:
bill
2022-01-16 06:34:05 -05:00
parent 90712aff7d
commit 49a3fc0a78
16 changed files with 174 additions and 634 deletions
Download Patch File Download Diff File Expand all files Collapse all files
aquery_parser/__init__.py
+88
View File
@@ -0,0 +1,88 @@
# encoding: utf-8
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
#
# Contact: Kyle Lahnakoski (kyle@lahnakoski.com)
#
from __future__ import absolute_import, division, unicode_literals
import json
from threading import Lock
from aquery_parser.sql_parser import scrub
from aquery_parser.utils import ansi_string, simple_op, normal_op
parse_locker = Lock() # ENSURE ONLY ONE PARSING AT A TIME
common_parser = None
mysql_parser = None
sqlserver_parser = None
SQL_NULL = {"null": {}}
def parse(sql, null=SQL_NULL, calls=simple_op):
"""
:param sql: String of SQL
:param null: What value to use as NULL (default is the null function `{"null":{}}`)
:return: parse tree
"""
global common_parser
with parse_locker:
if not common_parser:
common_parser = sql_parser.common_parser()
result = _parse(common_parser, sql, null, calls)
return result
def parse_mysql(sql, null=SQL_NULL, calls=simple_op):
"""
PARSE MySQL ASSUME DOUBLE QUOTED STRINGS ARE LITERALS
:param sql: String of SQL
:param null: What value to use as NULL (default is the null function `{"null":{}}`)
:return: parse tree
"""
global mysql_parser
with parse_locker:
if not mysql_parser:
mysql_parser = sql_parser.mysql_parser()
return _parse(mysql_parser, sql, null, calls)
def parse_sqlserver(sql, null=SQL_NULL, calls=simple_op):
"""
PARSE MySQL ASSUME DOUBLE QUOTED STRINGS ARE LITERALS
:param sql: String of SQL
:param null: What value to use as NULL (default is the null function `{"null":{}}`)
:return: parse tree
"""
global sqlserver_parser
with parse_locker:
if not sqlserver_parser:
sqlserver_parser = sql_parser.sqlserver_parser()
return _parse(sqlserver_parser, sql, null, calls)
parse_bigquery = parse_mysql
def _parse(parser, sql, null, calls):
utils.null_locations = []
utils.scrub_op = calls
sql = sql.rstrip().rstrip(";")
parse_result = parser.parse_string(sql, parse_all=True)
output = scrub(parse_result)
for o, n in utils.null_locations:
o[n] = null
return output
_ = json.dumps
__all__ = ["parse", "format", "parse_mysql", "parse_bigquery", "normal_op", "simple_op"]
aquery_parser/keywords.py
+401
View File
@@ -0,0 +1,401 @@
# encoding: utf-8
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
#
# Contact: Kyle Lahnakoski (kyle@lahnakoski.com)
#
# SQL CONSTANTS
from mo_parsing import *
from aquery_parser.utils import SQL_NULL, keyword
NULL = keyword("null") / (lambda: SQL_NULL)
TRUE = keyword("true") / (lambda: True)
FALSE = keyword("false") / (lambda: False)
NOCASE = keyword("nocase")
ASC = keyword("asc")
DESC = keyword("desc")
# SIMPLE KEYWORDS
AS = keyword("as").suppress()
ASSUMING = keyword("assuming")
ALL = keyword("all")
BY = keyword("by").suppress()
CAST = keyword("cast")
CONSTRAINT = keyword("constraint").suppress()
CREATE = keyword("create").suppress()
CROSS = keyword("cross")
DISTINCT = keyword("distinct")
EXCEPT = keyword("except")
FETCH = keyword("fetch").suppress()
FROM = keyword("from").suppress()
FULL = keyword("full")
FUNCTION = keyword("function").suppress()
GROUP = keyword("group").suppress()
HAVING = keyword("having").suppress()
INNER = keyword("inner")
INTERVAL = keyword("interval")
JOIN = keyword("join")
LEFT = keyword("left")
LIKE = keyword("like")
LIMIT = keyword("limit").suppress()
MINUS = keyword("minus")
OFFSET = keyword("offset").suppress()
ON = keyword("on").suppress()
ORDER = keyword("order").suppress()
OUTER = keyword("outer")
OVER = keyword("over").suppress()
PARTITION = keyword("partition").suppress()
# PERCENT = keyword("percent").suppress()
RIGHT = keyword("right")
RLIKE = keyword("rlike")
SELECT = keyword("select").suppress()
TABLE = keyword("table").suppress()
THEN = keyword("then").suppress()
TOP = keyword("top").suppress()
UNION = keyword("union")
INTERSECT = keyword("intersect")
USING = keyword("using").suppress()
WHEN = keyword("when").suppress()
WHERE = keyword("where").suppress()
WITH = keyword("with").suppress()
WITHIN = keyword("within").suppress()
PRIMARY = keyword("primary").suppress()
FOREIGN = keyword("foreign").suppress()
KEY = keyword("key").suppress()
UNIQUE = keyword("unique").suppress()
INDEX = keyword("index").suppress()
REFERENCES = keyword("references").suppress()
RECURSIVE = keyword("recursive").suppress()
VALUES = keyword("values").suppress()
WINDOW = keyword("window")
PRIMARY_KEY = Group(PRIMARY + KEY).set_parser_name("primary_key")
FOREIGN_KEY = Group(FOREIGN + KEY).set_parser_name("foreign_key")
# SIMPLE OPERATORS
CONCAT = Literal("||").set_parser_name("concat")
MUL = Literal("*").set_parser_name("mul")
DIV = Literal("/").set_parser_name("div")
MOD = Literal("%").set_parser_name("mod")
NEG = Literal("-").set_parser_name("neg")
ADD = Literal("+").set_parser_name("add")
SUB = Literal("-").set_parser_name("sub")
BINARY_NOT = Literal("~").set_parser_name("binary_not")
BINARY_AND = Literal("&").set_parser_name("binary_and")
BINARY_OR = Literal("|").set_parser_name("binary_or")
GTE = Literal(">=").set_parser_name("gte")
LTE = Literal("<=").set_parser_name("lte")
LT = Literal("<").set_parser_name("lt")
GT = Literal(">").set_parser_name("gt")
EEQ = (
# conservative equality https://github.com/klahnakoski/jx-sqlite/blob/dev/docs/Logical%20Equality.md#definitions
Literal("==") | Literal("=")
).set_parser_name("eq")
DEQ = (
# decisive equality
# https://sparkbyexamples.com/apache-hive/hive-relational-arithmetic-logical-operators/
Literal("<=>").set_parser_name("eq!")
)
IDF = (
# decisive equality
# https://prestodb.io/docs/current/functions/comparison.html#is-distinct-from-and-is-not-distinct-from
keyword("is distinct from").set_parser_name("eq!")
)
INDF = (
# decisive equality
# https://prestodb.io/docs/current/functions/comparison.html#is-distinct-from-and-is-not-distinct-from
keyword("is not distinct from").set_parser_name("ne!")
)
FASSIGN = Literal(":=").set_parser_name("fassign") # Assignment in UDFs
NEQ = (Literal("!=") | Literal("<>")).set_parser_name("neq")
LAMBDA = Literal("->").set_parser_name("lambda")
AND = keyword("and")
BETWEEN = keyword("between")
CASE = keyword("case").suppress()
COLLATE = keyword("collate")
END = keyword("end")
ELSE = keyword("else").suppress()
IN = keyword("in")
IS = keyword("is")
NOT = keyword("not")
OR = keyword("or")
LATERAL = keyword("lateral")
VIEW = keyword("view")
# COMPOUND KEYWORDS
joins = (
(
Optional(CROSS | OUTER | INNER | ((FULL | LEFT | RIGHT) + Optional(INNER | OUTER)))
+ JOIN
+ Optional(LATERAL)
)
| LATERAL + VIEW + Optional(OUTER)
) / (lambda tokens: " ".join(tokens).lower())
UNION_ALL = (UNION + ALL).set_parser_name("union_all")
WITHIN_GROUP = Group(WITHIN + GROUP).set_parser_name("within_group")
SELECT_DISTINCT = Group(SELECT + DISTINCT).set_parser_name("select distinct")
PARTITION_BY = Group(PARTITION + BY).set_parser_name("partition by")
GROUP_BY = Group(GROUP + BY).set_parser_name("group by")
ORDER_BY = Group(ORDER + BY).set_parser_name("order by")
# COMPOUND OPERATORS
AT_TIME_ZONE = Group(keyword("at") + keyword("time") + keyword("zone"))
NOT_BETWEEN = Group(NOT + BETWEEN).set_parser_name("not_between")
NOT_LIKE = Group(NOT + LIKE).set_parser_name("not_like")
NOT_RLIKE = Group(NOT + RLIKE).set_parser_name("not_rlike")
NOT_IN = Group(NOT + IN).set_parser_name("nin")
IS_NOT = Group(IS + NOT).set_parser_name("is_not")
_SIMILAR = keyword("similar")
_TO = keyword("to")
SIMILAR_TO = Group(_SIMILAR + _TO).set_parser_name("similar_to")
NOT_SIMILAR_TO = Group(NOT + _SIMILAR + _TO).set_parser_name("not_similar_to")
RESERVED = MatchFirst([
# ONY INCLUDE SINGLE WORDS
ALL,
AND,
AS,
ASC,
ASSUMING,
BETWEEN,
BY,
CASE,
COLLATE,
CONSTRAINT,
CREATE,
CROSS,
DESC,
DISTINCT,
EXCEPT,
ELSE,
END,
FALSE,
FETCH,
FOREIGN,
FROM,
FULL,
FUNCTION,
GROUP_BY,
GROUP,
HAVING,
IN,
INDEX,
INNER,
INTERSECT,
INTERVAL,
IS_NOT,
IS,
JOIN,
KEY,
LATERAL,
LEFT,
LIKE,
LIMIT,
MINUS,
NOCASE,
NOT,
NULL,
OFFSET,
ON,
OR,
ORDER,
OUTER,
OVER,
PARTITION,
PRIMARY,
REFERENCES,
RIGHT,
RLIKE,
SELECT,
THEN,
TRUE,
UNION,
UNIQUE,
USING,
WHEN,
WHERE,
WINDOW,
WITH,
WITHIN,
])
L_INLINE = Literal("<k>").suppress()
R_INLINE = Literal("</k>").suppress()
LBRACE = Literal("{").suppress()
RBRACE = Literal("}").suppress()
LB = Literal("(").suppress()
RB = Literal(")").suppress()
EQ = Char("=").suppress()
join_keywords = {
"join",
"full join",
"cross join",
"inner join",
"left join",
"right join",
"full outer join",
"right outer join",
"left outer join",
}
precedence = {
# https://www.sqlite.org/lang_expr.html
"literal": -1,
"interval": 0,
"cast": 0,
"collate": 0,
"concat": 1,
"mul": 2,
"div": 1.5,
"mod": 2,
"neg": 3,
"add": 3,
"sub": 2.5,
"binary_not": 4,
"binary_and": 4,
"binary_or": 4,
"gte": 5,
"lte": 5,
"lt": 5,
"gt": 6,
"eq": 7,
"neq": 7,
"missing": 7,
"exists": 7,
"at_time_zone": 8,
"between": 8,
"not_between": 8,
"in": 8,
"nin": 8,
"is": 8,
"like": 8,
"not_like": 8,
"rlike": 8,
"not_rlike": 8,
"similar_to": 8,
"not_similar_to": 8,
"and": 10,
"or": 11,
"lambda": 12,
"join": 18,
"list": 18,
"function": 30,
"select": 30,
"from": 30,
"window": 35,
"union": 40,
"union_all": 40,
"except": 40,
"minus": 40,
"intersect": 40,
"order": 50,
}
KNOWN_OPS = [
COLLATE,
CONCAT,
MUL | DIV | MOD,
NEG,
ADD | SUB,
BINARY_NOT,
BINARY_AND,
BINARY_OR,
GTE | LTE | LT | GT,
EEQ | NEQ | DEQ | IDF | INDF,
AT_TIME_ZONE,
(BETWEEN, AND),
(NOT_BETWEEN, AND),
IN,
NOT_IN,
IS_NOT,
IS,
LIKE,
NOT_LIKE,
RLIKE,
NOT_RLIKE,
SIMILAR_TO,
NOT_SIMILAR_TO,
NOT,
AND,
OR,
LAMBDA,
]
times = ["now", "today", "tomorrow", "eod"]
durations = {
"microseconds": "microsecond",
"microsecond": "microsecond",
"microsecs": "microsecond",
"microsec": "microsecond",
"useconds": "microsecond",
"usecond": "microsecond",
"usecs": "microsecond",
"usec": "microsecond",
"us": "microsecond",
"milliseconds": "millisecond",
"millisecond": "millisecond",
"millisecon": "millisecond",
"mseconds": "millisecond",
"msecond": "millisecond",
"millisecs": "millisecond",
"millisec": "millisecond",
"msecs": "millisecond",
"msec": "millisecond",
"ms": "millisecond",
"seconds": "second",
"second": "second",
"secs": "second",
"sec": "second",
"s": "second",
"minutes": "minute",
"minute": "minute",
"mins": "minute",
"min": "minute",
"m": "minute",
"hours": "hour",
"hour": "hour",
"hrs": "hour",
"hr": "hour",
"h": "hour",
"days": "day",
"day": "day",
"d": "day",
"dayofweek": "dow",
"dow": "dow",
"weekday": "dow",
"weeks": "week",
"week": "week",
"w": "week",
"months": "month",
"month": "month",
"mons": "month",
"mon": "month",
"quarters": "quarter",
"quarter": "quarter",
"years": "year",
"year": "year",
"decades": "decade",
"decade": "decade",
"decs": "decade",
"dec": "decade",
"centuries": "century",
"century": "century",
"cents": "century",
"cent": "century",
"c": "century",
"millennia": "millennium",
"millennium": "millennium",
"mils": "millennium",
"mil": "millennium",
"epoch": "epoch",
}
aquery_parser/sql_parser.py
+634
View File
@@ -0,0 +1,634 @@
# encoding: utf-8
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
#
# Contact: Kyle Lahnakoski (kyle@lahnakoski.com)
#
from mo_parsing.helpers import restOfLine
from mo_parsing.infix import delimited_list
from mo_parsing.whitespaces import NO_WHITESPACE, Whitespace
from aquery_parser.keywords import *
from aquery_parser.types import get_column_type, time_functions
from aquery_parser.utils import *
from aquery_parser.windows import window
def no_dashes(tokens, start, string):
if "-" in tokens[0]:
index = tokens[0].find("-")
raise ParseException(
tokens.type,
start + index + 1, # +1 TO ENSURE THIS MESSAGE HAS PRIORITY
string,
"""Ambiguity: Use backticks (``) around identifiers with dashes, or add space around subtraction operator.""",
)
digit = Char("0123456789")
simple_ident = (
Char(FIRST_IDENT_CHAR)
+ Char(IDENT_CHAR)[...] # let's not support dashes in var_names.
)
simple_ident = Regex(simple_ident.__regex__()[1])
def common_parser():
combined_ident = Combine(delimited_list(
ansi_ident | mysql_backtick_ident | simple_ident, separator=".", combine=True,
)).set_parser_name("identifier")
return parser(ansi_string, combined_ident)
def mysql_parser():
mysql_string = ansi_string | mysql_doublequote_string
mysql_ident = Combine(delimited_list(
mysql_backtick_ident | sqlserver_ident | simple_ident,
separator=".",
combine=True,
)).set_parser_name("mysql identifier")
return parser(mysql_string, mysql_ident)
def sqlserver_parser():
combined_ident = Combine(delimited_list(
ansi_ident
| mysql_backtick_ident
| sqlserver_ident
| Word(FIRST_IDENT_CHAR, IDENT_CHAR),
separator=".",
combine=True,
)).set_parser_name("identifier")
return parser(ansi_string, combined_ident, sqlserver=True)
def parser(literal_string, ident, sqlserver=False):
with Whitespace() as engine:
engine.add_ignore(Literal("--") + restOfLine)
engine.add_ignore(Literal("#") + restOfLine)
engine.add_ignore(Literal("/*") + SkipTo("*/", include=True))
var_name = ~RESERVED + ident
inline_kblock = (L_INLINE + SkipTo(R_INLINE, include=True))("k9")
# EXPRESSIONS
expr = Forward()
column_type, column_definition, column_def_references = get_column_type(
expr, var_name, literal_string
)
# CASE
case = (
CASE
+ Group(ZeroOrMore(
(WHEN + expr("when") + THEN + expr("then")) / to_when_call
))("case")
+ Optional(ELSE + expr("else"))
+ END
) / to_case_call
switch = (
CASE
+ expr("value")
+ Group(ZeroOrMore(
(WHEN + expr("when") + THEN + expr("then")) / to_when_call
))("case")
+ Optional(ELSE + expr("else"))
+ END
) / to_switch_call
cast = (
Group(CAST("op") + LB + expr("params") + AS + column_type("params") + RB)
/ to_json_call
)
trim = (
Group(
keyword("trim").suppress()
+ LB
+ Optional(
(keyword("both") | keyword("trailing") | keyword("leading"))
/ (lambda t: t[0].lower())
)("direction")
+ (
assign("from", expr)
| expr("chars") + Optional(assign("from", expr))
)
+ RB
).set_parser_name("trim")
/ to_trim_call
)
_standard_time_intervals = MatchFirst([
keyword(d) / (lambda t: durations[t[0].lower()]) for d in durations.keys()
]).set_parser_name("duration")("params")
duration = (
real_num | int_num | literal_string
)("params") + _standard_time_intervals
interval = (
INTERVAL + ("'" + delimited_list(duration) + "'" | duration)
) / to_interval_call
timestamp = (
time_functions("op")
+ (
literal_string("params")
| MatchFirst([
keyword(t) / (lambda t: t.lower()) for t in times
])("params")
)
) / to_json_call
extract = (
keyword("extract")("op")
+ LB
+ (_standard_time_intervals | expr("params"))
+ FROM
+ expr("params")
+ RB
) / to_json_call
alias = Optional((
(
AS
+ (var_name("name") + Optional(LB + delimited_list(ident("col")) + RB))
| (
var_name("name")
+ Optional(
(LB + delimited_list(ident("col")) + RB)
| (AS + delimited_list(var_name("col")))
)
)
)
/ to_alias
)("name"))
named_column = Group(Group(expr)("value") + alias)
stack = (
keyword("stack")("op")
+ LB
+ int_num("width")
+ ","
+ delimited_list(expr)("args")
+ RB
) / to_stack
# ARRAY[foo],
# ARRAY < STRING > [foo, bar], INVALID
# ARRAY < STRING > [foo, bar],
create_array = (
keyword("array")("op")
+ Optional(LT.suppress() + column_type("type") + GT.suppress())
+ (
LB + delimited_list(Group(expr))("args") + RB
| (Literal("[") + delimited_list(Group(expr))("args") + Literal("]"))
)
)
if not sqlserver:
# SQL SERVER DOES NOT SUPPORT [] FOR ARRAY CONSTRUCTION (USED FOR IDENTIFIERS)
create_array = (
Literal("[") + delimited_list(Group(expr))("args") + Literal("]")
| create_array
)
create_array = create_array / to_array
create_map = (
keyword("map")
+ Literal("[")
+ expr("keys")
+ ","
+ expr("values")
+ Literal("]")
) / to_map
create_struct = (
keyword("struct")("op")
+ Optional(
LT.suppress() + delimited_list(column_type)("types") + GT.suppress()
)
+ LB
+ delimited_list(Group((expr("value") + alias) / to_select_call))("args")
+ RB
).set_parser_name("create struct") / to_struct
distinct = (
DISTINCT("op") + delimited_list(named_column)("params")
) / to_json_call
query = Forward().set_parser_name("query")
call_function = (
ident("op")
+ LB
+ Optional(Group(query) | delimited_list(Group(expr)))("params")
+ Optional(
(keyword("respect") | keyword("ignore"))("nulls")
+ keyword("nulls").suppress()
)
+ RB
).set_parser_name("call function") / to_json_call
with NO_WHITESPACE:
def scale(tokens):
return {"mul": [tokens[0], tokens[1]]}
scale_function = ((real_num | int_num) + call_function) / scale
scale_ident = ((real_num | int_num) + ident) / scale
compound = (
NULL
| TRUE
| FALSE
| NOCASE
| interval
| timestamp
| extract
| case
| switch
| cast
| distinct
| trim
| stack
| create_array
| create_map
| create_struct
| (LB + Group(query) + RB)
| (LB + Group(delimited_list(expr)) / to_tuple_call + RB)
| literal_string.set_parser_name("string")
| hex_num.set_parser_name("hex")
| scale_function
| scale_ident
| real_num.set_parser_name("float")
| int_num.set_parser_name("int")
| call_function
| Combine(var_name + Optional(".*"))
)
sort_column = (
expr("value").set_parser_name("sort1")
+ Optional(DESC("sort") | ASC("sort"))
+ Optional(assign("nulls", keyword("first") | keyword("last")))
)
window_clause, over_clause = window(expr, var_name, sort_column)
expr << (
(
Literal("*")
| infix_notation(
compound,
[
(
Literal("[").suppress() + expr + Literal("]").suppress(),
1,
LEFT_ASSOC,
to_offset,
),
(
Literal(".").suppress() + simple_ident,
1,
LEFT_ASSOC,
to_offset,
),
(window_clause, 1, LEFT_ASSOC, to_window_mod),
(
assign("filter", LB + WHERE + expr + RB),
1,
LEFT_ASSOC,
to_window_mod,
),
]
+ [
(
o,
1 if o in unary_ops else (3 if isinstance(o, tuple) else 2),
unary_ops.get(o, LEFT_ASSOC),
to_lambda if o is LAMBDA else to_json_operator,
)
for o in KNOWN_OPS
],
)
)("value").set_parser_name("expression")
)
select_column = (
Group(
expr("value") + alias | Literal("*")("value")
).set_parser_name("column")
/ to_select_call
)
table_source = Forward()
join = (
Group(joins)("op")
+ table_source("join")
+ Optional((ON + expr("on")) | (USING + expr("using")))
| (
Group(WINDOW)("op")
+ Group(var_name("name") + AS + over_clause("value"))("join")
)
) / to_join_call
fassign = Group(var_name("var") + Suppress(FASSIGN) + expr("expr") + Suppress(";"))("assignment")
fassigns = fassign + ZeroOrMore(fassign, Whitespace(white=" \t"))
fbody = (Optional(fassigns) + expr("ret"))
udf = (
FUNCTION
+ var_name("fname")
+ LB
+ Optional(delimited_list(var_name)("params"))
+ RB
+ LBRACE
+ fbody
+ RBRACE
)
selection = (
(SELECT + DISTINCT + ON + LB)
+ delimited_list(select_column)("distinct_on")
+ RB
+ delimited_list(select_column)("select")
| SELECT + DISTINCT + delimited_list(select_column)("select_distinct")
| (
SELECT
+ Optional(
TOP
+ expr("value")
+ Optional(keyword("percent"))("percent")
+ Optional(WITH + keyword("ties"))("ties")
)("top")
/ to_top_clause
+ delimited_list(select_column)("select")
)
)
row = (LB + delimited_list(Group(expr)) + RB) / to_row
values = VALUES + delimited_list(row) / to_values
unordered_sql = Group(
values
| selection
+ Optional(
(FROM + delimited_list(table_source) + ZeroOrMore(join))("from")
+ Optional(WHERE + expr("where"))
+ Optional(GROUP_BY + delimited_list(Group(named_column))("groupby"))
+ Optional(HAVING + expr("having"))
)
).set_parser_name("unordered sql")
with NO_WHITESPACE:
def mult(tokens):
amount = tokens["bytes"]
scale = tokens["scale"].lower()
return {
"bytes": amount
* {"b": 1, "k": 1_000, "m": 1_000_000, "g": 1_000_000_000}[scale]
}
ts_bytes = (
(real_num | int_num)("bytes") + Char("bBkKmMgG")("scale")
) / mult
tablesample = assign(
"tablesample",
LB
+ (
(
keyword("bucket")("op")
+ int_num("params")
+ keyword("out of")
+ int_num("params")
+ Optional(ON + expr("on"))
)
/ to_json_call
| (real_num | int_num)("percent") + keyword("percent")
| int_num("rows") + keyword("rows")
| ts_bytes
)
+ RB,
)
assumption = Group((ASC|DESC) ("ord") + var_name("attrib"))
assumptions = (ASSUMING + Group(delimited_list(assumption))("assumptions"))
table_source << Group(
((LB + query + RB) | stack | call_function | var_name)("value")
+ Optional(assumptions)
+ Optional(flag("with ordinality"))
+ Optional(tablesample)
+ alias
).set_parser_name("table_source") / to_table
rows = Optional(keyword("row") | keyword("rows"))
limit = (
Optional(assign("offset", expr) + rows)
& Optional(
FETCH
+ Optional(keyword("first") | keyword("next"))
+ expr("fetch")
+ rows
+ Optional(keyword("only"))
)
& Optional(assign("limit", expr))
)
ordered_sql = (
(
(unordered_sql | (LB + query + RB))
+ ZeroOrMore(
Group(
(UNION | INTERSECT | EXCEPT | MINUS) + Optional(ALL | DISTINCT)
)("op")
+ (unordered_sql | (LB + query + RB))
)
)("union")
+ Optional(ORDER_BY + delimited_list(Group(sort_column))("orderby"))
+ limit
).set_parser_name("ordered sql") / to_union_call
with_expr = delimited_list(Group(
(
(var_name("name") + Optional(LB + delimited_list(ident("col")) + RB))
/ to_alias
)("name")
+ (AS + LB + (query | expr)("value") + RB)
))
query << (
Optional(assign("with recursive", with_expr) | assign("with", with_expr))
+ Group(ordered_sql)("query")
) / to_query
#####################################################################
# DML STATEMENTS
#####################################################################
# MySQL's index_type := Using + ( "BTREE" | "HASH" )
index_type = Optional(assign("using", ident("index_type")))
index_column_names = LB + delimited_list(var_name("columns")) + RB
column_def_delete = assign(
"on delete",
(keyword("cascade") | keyword("set null") | keyword("set default")),
)
table_def_foreign_key = FOREIGN_KEY + Optional(
Optional(var_name("index_name"))
+ index_column_names
+ column_def_references
+ Optional(column_def_delete)
)
index_options = ZeroOrMore(var_name)("table_constraint_options")
table_constraint_definition = Optional(CONSTRAINT + var_name("name")) + (
assign("primary key", index_type + index_column_names + index_options)
| (
Optional(flag("unique"))
+ Optional(INDEX | KEY)
+ Optional(var_name("name"))
+ index_type
+ index_column_names
+ index_options
)("index")
| assign("check", LB + expr + RB)
| table_def_foreign_key("foreign_key")
)
table_element = (
column_definition("columns") | table_constraint_definition("constraint")
)
create_table = (
keyword("create")
+ Optional(keyword("or") + flag("replace"))
+ Optional(flag("temporary"))
+ TABLE
+ Optional((keyword("if not exists") / (lambda: False))("replace"))
+ var_name("name")
+ Optional(LB + delimited_list(table_element) + RB)
+ ZeroOrMore(
assign("engine", EQ + var_name)
| assign("collate", EQ + var_name)
| assign("auto_increment", EQ + int_num)
| assign("comment", EQ + literal_string)
| assign("default character set", EQ + var_name)
| assign("default charset", EQ + var_name)
)
+ Optional(AS.suppress() + infix_notation(query, [])("query"))
)("create_table")
create_view = (
keyword("create")
+ Optional(keyword("or") + flag("replace"))
+ Optional(flag("temporary"))
+ VIEW.suppress()
+ Optional((keyword("if not exists") / (lambda: False))("replace"))
+ var_name("name")
+ AS
+ query("query")
)("create_view")
# CREATE INDEX a ON u USING btree (e);
create_index = (
keyword("create index")
+ Optional(keyword("or") + flag("replace"))(INDEX | KEY)
+ Optional((keyword("if not exists") / (lambda: False))("replace"))
+ var_name("name")
+ ON
+ var_name("table")
+ index_type
+ index_column_names
+ index_options
)("create index")
cache_options = Optional((
keyword("options").suppress()
+ LB
+ Dict(delimited_list(Group(
literal_string / (lambda tokens: tokens[0]["literal"])
+ Optional(EQ)
+ var_name
)))
+ RB
)("options"))
create_cache = (
keyword("cache").suppress()
+ Optional(flag("lazy"))
+ TABLE
+ var_name("name")
+ cache_options
+ Optional(AS + query("query"))
)("cache")
drop_table = (
keyword("drop table") + Optional(flag("if exists")) + var_name("table")
)("drop")
drop_view = (
keyword("drop view") + Optional(flag("if exists")) + var_name("view")
)("drop")
drop_index = (
keyword("drop index") + Optional(flag("if exists")) + var_name("index")
)("drop")
insert = (
keyword("insert").suppress()
+ (
flag("overwrite") + keyword("table").suppress()
| keyword("into").suppress() + Optional(keyword("table").suppress())
)
+ var_name("table")
+ Optional(LB + delimited_list(var_name)("columns") + RB)
+ Optional(flag("if exists"))
+ (values | query)("query")
) / to_insert_call
update = (
keyword("update")("op")
+ var_name("params")
+ assign("set", Dict(delimited_list(Group(var_name + EQ + expr))))
+ Optional(assign("where", expr))
) / to_json_call
delete = (
keyword("delete")("op")
+ keyword("from").suppress()
+ var_name("params")
+ Optional(assign("where", expr))
) / to_json_call
sql_stmts = delimited_list( (
query
| (insert | update | delete)
| (create_table | create_view | create_cache | create_index)
| (drop_table | drop_view | drop_index)
)("stmts"), ";")
other_stmt = (
inline_kblock
| udf
) ("stmts")
stmts = ZeroOrMore(sql_stmts|other_stmt)
return stmts.finalize()
aquery_parser/types.py
+223
View File
@@ -0,0 +1,223 @@
# encoding: utf-8
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
#
# Contact: Kyle Lahnakoski (kyle@lahnakoski.com)
#
# KNOWN TYPES
from mo_parsing import Forward, Group, Optional, MatchFirst, Literal, ZeroOrMore, export
from mo_parsing.infix import delimited_list, RIGHT_ASSOC, LEFT_ASSOC
from aquery_parser.keywords import (
RB,
LB,
NEG,
NOT,
BINARY_NOT,
NULL,
EQ,
KNOWN_OPS,
LT,
GT,
)
from aquery_parser.utils import (
keyword,
to_json_call,
int_num,
ansi_string,
ansi_ident,
assign,
flag,
)
_size = Optional(LB + int_num("params") + RB)
_sizes = Optional(LB + delimited_list(int_num("params")) + RB)
simple_types = Forward()
BIGINT = Group(keyword("bigint")("op") + Optional(_size)+Optional(flag("unsigned"))) / to_json_call
BOOL = Group(keyword("bool")("op")) / to_json_call
BOOLEAN = Group(keyword("boolean")("op")) / to_json_call
DOUBLE = Group(keyword("double")("op")) / to_json_call
FLOAT64 = Group(keyword("float64")("op")) / to_json_call
FLOAT = Group(keyword("float")("op")) / to_json_call
GEOMETRY = Group(keyword("geometry")("op")) / to_json_call
INTEGER = Group(keyword("integer")("op")) / to_json_call
INT = (keyword("int")("op") + _size) / to_json_call
INT32 = Group(keyword("int32")("op")) / to_json_call
INT64 = Group(keyword("int64")("op")) / to_json_call
REAL = Group(keyword("real")("op")) / to_json_call
TEXT = Group(keyword("text")("op")) / to_json_call
SMALLINT = Group(keyword("smallint")("op")) / to_json_call
STRING = Group(keyword("string")("op")) / to_json_call
BLOB = (keyword("blob")("op") + _size) / to_json_call
BYTES = (keyword("bytes")("op") + _size) / to_json_call
CHAR = (keyword("char")("op") + _size) / to_json_call
NCHAR = (keyword("nchar")("op") + _size) / to_json_call
VARCHAR = (keyword("varchar")("op") + _size) / to_json_call
VARCHAR2 = (keyword("varchar2")("op") + _size) / to_json_call
VARBINARY = (keyword("varbinary")("op") + _size) / to_json_call
TINYINT = (keyword("tinyint")("op") + _size) / to_json_call
UUID = Group(keyword("uuid")("op")) / to_json_call
DECIMAL = (keyword("decimal")("op") + _sizes) / to_json_call
DOUBLE_PRECISION = (
Group((keyword("double precision") / (lambda: "double_precision"))("op"))
/ to_json_call
)
NUMERIC = (keyword("numeric")("op") + _sizes) / to_json_call
NUMBER = (keyword("number")("op") + _sizes) / to_json_call
MAP_TYPE = (
keyword("map")("op") + LB + delimited_list(simple_types("params")) + RB
) / to_json_call
ARRAY_TYPE = (keyword("array")("op") + LB + simple_types("params") + RB) / to_json_call
DATE = keyword("date")
DATETIME = keyword("datetime")
DATETIME_W_TIMEZONE = keyword("datetime with time zone")
TIME = keyword("time")
TIMESTAMP = keyword("timestamp")
TIMESTAMP_W_TIMEZONE = keyword("timestamp with time zone")
TIMESTAMPTZ = keyword("timestamptz")
TIMETZ = keyword("timetz")
time_functions = DATE | DATETIME | TIME | TIMESTAMP | TIMESTAMPTZ | TIMETZ
# KNOWNN TIME TYPES
_format = Optional((ansi_string | ansi_ident)("params"))
DATE_TYPE = (DATE("op") + _format) / to_json_call
DATETIME_TYPE = (DATETIME("op") + _format) / to_json_call
DATETIME_W_TIMEZONE_TYPE = (DATETIME_W_TIMEZONE("op") + _format) / to_json_call
TIME_TYPE = (TIME("op") + _format) / to_json_call
TIMESTAMP_TYPE = (TIMESTAMP("op") + _format) / to_json_call
TIMESTAMP_W_TIMEZONE_TYPE = (TIMESTAMP_W_TIMEZONE("op") + _format) / to_json_call
TIMESTAMPTZ_TYPE = (TIMESTAMPTZ("op") + _format) / to_json_call
TIMETZ_TYPE = (TIMETZ("op") + _format) / to_json_call
simple_types << MatchFirst([
ARRAY_TYPE,
BIGINT,
BOOL,
BOOLEAN,
BLOB,
BYTES,
CHAR,
DATE_TYPE,
DATETIME_W_TIMEZONE_TYPE,
DATETIME_TYPE,
DECIMAL,
DOUBLE_PRECISION,
DOUBLE,
FLOAT64,
FLOAT,
GEOMETRY,
MAP_TYPE,
INTEGER,
INT,
INT32,
INT64,
NCHAR,
NUMBER,
NUMERIC,
REAL,
TEXT,
SMALLINT,
STRING,
TIME_TYPE,
TIMESTAMP_W_TIMEZONE_TYPE,
TIMESTAMP_TYPE,
TIMESTAMPTZ_TYPE,
TIMETZ_TYPE,
TINYINT,
UUID,
VARCHAR,
VARCHAR2,
VARBINARY,
])
CASTING = (Literal("::").suppress() + simple_types("params")).set_parser_name("cast")
KNOWN_OPS.insert(0, CASTING)
unary_ops = {
NEG: RIGHT_ASSOC,
NOT: RIGHT_ASSOC,
BINARY_NOT: RIGHT_ASSOC,
CASTING: LEFT_ASSOC,
}
def get_column_type(expr, var_name, literal_string):
column_definition = Forward()
column_type = Forward().set_parser_name("column type")
struct_type = (
keyword("struct")("op")
+ LT.suppress()
+ Group(delimited_list(column_definition))("params")
+ GT.suppress()
) / to_json_call
row_type = (
keyword("row")("op")
+ LB
+ Group(delimited_list(column_definition))("params")
+ RB
) / to_json_call
array_type = (
keyword("array")("op")
+ (
(
LT.suppress()
+ Group(delimited_list(column_type))("params")
+ GT.suppress()
)
| (LB + Group(delimited_list(column_type))("params") + RB)
)
) / to_json_call
column_type << (struct_type | row_type | array_type | simple_types)
column_def_identity = (
assign(
"generated",
(keyword("always") | keyword("by default") / (lambda: "by_default")),
)
+ keyword("as identity").suppress()
+ Optional(assign("start with", int_num))
+ Optional(assign("increment by", int_num))
)
column_def_references = assign(
"references", var_name("table") + LB + delimited_list(var_name)("columns") + RB,
)
column_options = ZeroOrMore(
((NOT + NULL) / (lambda: False))("nullable")
| (NULL / (lambda t: True))("nullable")
| flag("unique")
| flag("auto_increment")
| assign("comment", literal_string)
| assign("collate", Optional(EQ) + var_name)
| flag("primary key")
| column_def_identity("identity")
| column_def_references
| assign("check", LB + expr + RB)
| assign("default", expr)
).set_parser_name("column_options")
column_definition << Group(
var_name("name") + (column_type | var_name)("type") + column_options
).set_parser_name("column_definition")
return column_type, column_definition, column_def_references
export("aquery_parser.utils", unary_ops)
aquery_parser/utils.py
+617
View File
@@ -0,0 +1,617 @@
# encoding: utf-8
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
#
# Contact: Kyle Lahnakoski (kyle@lahnakoski.com)
#
import ast
from mo_dots import is_data, is_null, Data, from_data
from mo_future import text, number_types, binary_type, flatten
from mo_imports import expect
from mo_parsing import *
from mo_parsing.utils import is_number, listwrap
unary_ops = expect("unary_ops")
class Call(object):
__slots__ = ["op", "args", "kwargs"]
def __init__(self, op, args, kwargs):
self.op = op
self.args = args
self.kwargs = kwargs
IDENT_CHAR = Regex("[@_$0-9A-Za-zÀ-ÖØ-öø-ƿ]").expr.parser_config.include
FIRST_IDENT_CHAR = "".join(set(IDENT_CHAR) - set("0123456789"))
SQL_NULL = Call("null", [], {})
null_locations = []
def keyword(keywords):
return And([
Keyword(k, caseless=True) for k in keywords.split(" ")
]).set_parser_name(keywords) / (lambda: keywords.replace(" ", "_"))
def flag(keywords):
"""
RETURN {keywords: True}
"""
return (keyword(keywords) / (lambda: True))(keywords.replace(" ", "_"))
def assign(key: str, value: ParserElement):
return keyword(key).suppress() + value(key.replace(" ", "_"))
def simple_op(op, args, kwargs):
if args is None:
kwargs[op] = {}
else:
kwargs[op] = args
return kwargs
def normal_op(op, args, kwargs):
output = Data(op=op)
args = listwrap(args)
if args and (not isinstance(args[0], dict) or args[0]):
output.args = args
if kwargs:
output.kwargs = kwargs
return from_data(output)
scrub_op = simple_op
def scrub(result):
if result is SQL_NULL:
return SQL_NULL
elif result == None:
return None
elif isinstance(result, text):
return result
elif isinstance(result, binary_type):
return result.decode("utf8")
elif isinstance(result, number_types):
return result
elif isinstance(result, Call):
kwargs = scrub(result.kwargs)
args = scrub(result.args)
if args is SQL_NULL:
null_locations.append((kwargs, result.op))
return scrub_op(result.op, args, kwargs)
elif isinstance(result, dict) and not result:
return result
elif isinstance(result, list):
output = [rr for r in result for rr in [scrub(r)]]
if not output:
return None
elif len(output) == 1:
return output[0]
else:
for i, v in enumerate(output):
if v is SQL_NULL:
null_locations.append((output, i))
return output
else:
# ATTEMPT A DICT INTERPRETATION
try:
kv_pairs = list(result.items())
except Exception as c:
print(c)
output = {k: vv for k, v in kv_pairs for vv in [scrub(v)] if not is_null(vv)}
if isinstance(result, dict) or output:
for k, v in output.items():
if v is SQL_NULL:
null_locations.append((output, k))
return output
return scrub(list(result))
def _chunk(values, size):
acc = []
for v in values:
acc.append(v)
if len(acc) == size:
yield acc
acc = []
if acc:
yield acc
def to_lambda(tokens):
params, op, expr = list(tokens)
return Call("lambda", [expr], {"params": list(params)})
def to_json_operator(tokens):
# ARRANGE INTO {op: params} FORMAT
length = len(tokens.tokens)
if length == 2:
if tokens.tokens[1].type.parser_name == "cast":
return Call("cast", list(tokens), {})
# UNARY OPERATOR
op = tokens.tokens[0].type.parser_name
if op == "neg" and is_number(tokens[1]):
return -tokens[1]
return Call(op, [tokens[1]], {})
elif length == 5:
# TRINARY OPERATOR
return Call(
tokens.tokens[1].type.parser_name, [tokens[0], tokens[2], tokens[4]], {}
)
op = tokens[1]
if not isinstance(op, text):
op = op.type.parser_name
op = binary_ops.get(op, op)
if op == "eq":
if tokens[2] is SQL_NULL:
return Call("missing", tokens[0], {})
elif tokens[0] is SQL_NULL:
return Call("missing", tokens[2], {})
elif op == "neq":
if tokens[2] is SQL_NULL:
return Call("exists", tokens[0], {})
elif tokens[0] is SQL_NULL:
return Call("exists", tokens[2], {})
elif op == "eq!":
if tokens[2] is SQL_NULL:
return Call("missing", tokens[0], {})
elif tokens[0] is SQL_NULL:
return Call("missing", tokens[2], {})
elif op == "ne!":
if tokens[2] is SQL_NULL:
return Call("exists", tokens[0], {})
elif tokens[0] is SQL_NULL:
return Call("exists", tokens[2], {})
elif op == "is":
if tokens[2] is SQL_NULL:
return Call("missing", tokens[0], {})
else:
return Call("exists", tokens[0], {})
elif op == "is_not":
if tokens[2] is SQL_NULL:
return Call("exists", tokens[0], {})
else:
return Call("missing", tokens[0], {})
operands = [tokens[0], tokens[2]]
binary_op = Call(op, operands, {})
if op in {"add", "mul", "and", "or"}:
# ASSOCIATIVE OPERATORS
acc = []
for operand in operands:
while isinstance(operand, ParseResults) and isinstance(operand.type, Group):
# PARENTHESES CAUSE EXTRA GROUP LAYERS
operand = operand[0]
if isinstance(operand, ParseResults) and isinstance(
operand.type, Forward
):
operand = operand[0]
if isinstance(operand, Call) and operand.op == op:
acc.extend(operand.args)
elif isinstance(operand, list):
acc.append(operand)
elif isinstance(operand, dict) and operand.get(op):
acc.extend(operand.get(op))
else:
acc.append(operand)
binary_op = Call(op, acc, {})
return binary_op
def to_offset(tokens):
expr, offset = tokens.tokens
return Call("get", [expr, offset], {})
def to_window_mod(tokens):
expr, window = tokens.tokens
return Call("value", [expr], {**window})
def to_tuple_call(tokens):
# IS THIS ONE VALUE IN (), OR MANY?
tokens = list(tokens)
if len(tokens) == 1:
return [tokens[0]]
if all(isinstance(r, number_types) for r in tokens):
return [tokens]
if all(
isinstance(r, number_types) or (is_data(r) and "literal" in r.keys())
for r in tokens
):
candidate = {"literal": [r["literal"] if is_data(r) else r for r in tokens]}
return candidate
return [tokens]
binary_ops = {
"::": "cast",
"COLLATE": "collate",
"||": "concat",
"*": "mul",
"/": "div",
"%": "mod",
"+": "add",
"-": "sub",
"&": "binary_and",
"|": "binary_or",
"<": "lt",
"<=": "lte",
">": "gt",
">=": "gte",
"=": "eq",
"==": "eq",
"is distinct from": "eq!", # https://sparkbyexamples.com/apache-hive/hive-relational-arithmetic-logical-operators/
"is_distinct_from": "eq!",
"is not distinct from": "ne!",
"is_not_distinct_from": "ne!",
"<=>": "eq!", # https://sparkbyexamples.com/apache-hive/hive-relational-arithmetic-logical-operators/
"!=": "neq",
"<>": "neq",
"not in": "nin",
"in": "in",
"is_not": "neq",
"is": "eq",
"similar_to": "similar_to",
"like": "like",
"rlike": "rlike",
"not like": "not_like",
"not_like": "not_like",
"not rlike": "not_rlike",
"not_rlike": "not_rlike",
"not_simlilar_to": "not_similar_to",
"or": "or",
"and": "and",
"->": "lambda",
"union": "union",
"union_all": "union_all",
"union all": "union_all",
"except": "except",
"minus": "minus",
"intersect": "intersect",
}
is_set_op = ("union", "union_all", "except", "minus", "intersect")
def to_trim_call(tokens):
frum = tokens["from"]
if not frum:
return Call("trim", [tokens["chars"]], {"direction": tokens["direction"]})
return Call(
"trim",
[frum],
{"characters": tokens["chars"], "direction": tokens["direction"]},
)
def to_json_call(tokens):
# ARRANGE INTO {op: params} FORMAT
op = tokens["op"].lower()
op = binary_ops.get(op, op)
params = tokens["params"]
if isinstance(params, (dict, str, int, Call)):
args = [params]
else:
args = list(params)
kwargs = {k: v for k, v in tokens.items() if k not in ("op", "params")}
return ParseResults(
tokens.type,
tokens.start,
tokens.end,
[Call(op, args, kwargs)],
tokens.failures,
)
def to_interval_call(tokens):
# ARRANGE INTO {interval: [amount, type]} FORMAT
params = tokens["params"]
if not params:
params = {}
if params.length() == 2:
return Call("interval", params, {})
return Call("add", [Call("interval", p, {}) for p in _chunk(params, size=2)], {})
def to_case_call(tokens):
cases = list(tokens["case"])
elze = tokens["else"]
if elze != None:
cases.append(elze)
return Call("case", cases, {})
def to_switch_call(tokens):
# CONVERT TO CLASSIC CASE STATEMENT
value = tokens["value"]
acc = []
for c in list(tokens["case"]):
acc.append(Call("when", [Call("eq", [value] + c.args, {})], c.kwargs))
elze = tokens["else"]
if elze != None:
acc.append(elze)
return Call("case", acc, {})
def to_when_call(tokens):
tok = tokens
return Call("when", [tok["when"]], {"then": tok["then"]})
def to_join_call(tokens):
op = " ".join(tokens["op"])
if tokens["join"]["name"]:
output = {op: {
"name": tokens["join"]["name"],
"value": tokens["join"]["value"],
}}
else:
output = {op: tokens["join"]}
output["on"] = tokens["on"]
output["using"] = tokens["using"]
return output
def to_expression_call(tokens):
if set(tokens.keys()) & {"over", "within", "filter"}:
return
return ParseResults(
tokens.type,
tokens.start,
tokens.end,
listwrap(tokens["value"]),
tokens.failures,
)
def to_over(tokens):
if not tokens:
return {}
def to_alias(tokens):
cols = tokens["col"]
name = tokens["name"]
if cols:
return {name: cols}
return name
def to_top_clause(tokens):
value = tokens["value"]
if not value:
return None
value = value.value()
if tokens["ties"]:
output = {}
output["ties"] = True
if tokens["percent"]:
output["percent"] = value
else:
output["value"] = value
return output
elif tokens["percent"]:
return {"percent": value}
else:
return [value]
def to_row(tokens):
columns = list(tokens)
if len(columns) > 1:
return {"select": [{"value": v[0]} for v in columns]}
else:
return {"select": {"value": columns[0]}}
def get_literal(value):
if isinstance(value, (int, float)):
return value
elif isinstance(value, Call):
return
elif value is SQL_NULL:
return value
elif "literal" in value:
return value["literal"]
def to_values(tokens):
rows = list(tokens)
if len(rows) > 1:
values = [
[get_literal(s["value"]) for s in listwrap(row["select"])] for row in rows
]
if all(flatten(values)):
return {"from": {"literal": values}}
return {"union_all": list(tokens)}
else:
return rows
def to_stack(tokens):
width = tokens["width"]
args = listwrap(tokens["args"])
return Call("stack", args, {"width": width})
def to_array(tokens):
types = list(tokens["type"])
args = list(tokens["args"])
output = Call("create_array", args, {})
if types:
output = Call("cast", [output, Call("array", types, {})], {})
return output
def to_map(tokens):
keys = tokens["keys"]
values = tokens["values"]
return Call("create_map", [keys, values], {})
def to_struct(tokens):
types = list(tokens["types"])
args = list(d for a in tokens["args"] for d in [a if a["name"] else a["value"]])
output = Call("create_struct", args, {})
if types:
output = Call("cast", [output, Call("struct", types, {})], {})
return output
def to_select_call(tokens):
expr = tokens["value"]
if expr == "*":
return ["*"]
try:
call = expr[0][0]
if call.op == "value":
return {"name": tokens["name"], "value": call.args, **call.kwargs}
except:
pass
def to_union_call(tokens):
unions = tokens["union"]
if isinstance(unions, dict):
return unions
elif unions.type.parser_name == "unordered sql":
output = {k: v for k, v in unions.items()} # REMOVE THE Group()
else:
unions = list(unions)
sources = [unions[i] for i in range(0, len(unions), 2)]
operators = ["_".join(unions[i]) for i in range(1, len(unions), 2)]
acc = sources[0]
last_union = None
for op, so in list(zip(operators, sources[1:])):
if op == last_union and "union" in op:
acc[op] = acc[op] + [so]
else:
acc = {op: [acc, so]}
last_union = op
if not tokens["orderby"] and not tokens["offset"] and not tokens["limit"]:
return acc
else:
output = {"from": acc}
output["orderby"] = tokens["orderby"]
output["limit"] = tokens["limit"]
output["offset"] = tokens["offset"]
output["fetch"] = tokens["fetch"]
return output
def to_insert_call(tokens):
options = {
k: v for k, v in tokens.items() if k not in ["columns", "table", "query"]
}
query = tokens["query"]
columns = tokens["columns"]
try:
values = query["from"]["literal"]
if values:
if columns:
data = [dict(zip(columns, row)) for row in values]
return Call("insert", [tokens["table"]], {"values": data, **options})
else:
return Call("insert", [tokens["table"]], {"values": values, **options})
except Exception:
pass
return Call(
"insert", [tokens["table"]], {"columns": columns, "query": query, **options}
)
def to_query(tokens):
output = tokens["query"][0]
try:
output["with"] = tokens["with"]
output["with_recursive"] = tokens["with_recursive"]
return output
except Exception as cause:
return
def to_table(tokens):
output = dict(tokens)
if len(list(output.keys())) > 1:
return output
else:
return output["value"]
def unquote(tokens):
val = tokens[0]
if val.startswith("'") and val.endswith("'"):
val = "'" + val[1:-1].replace("''", "\\'") + "'"
elif val.startswith('"') and val.endswith('"'):
val = '"' + val[1:-1].replace('""', '\\"') + '"'
elif val.startswith("`") and val.endswith("`"):
val = '"' + val[1:-1].replace("``", "`").replace('"', '\\"') + '"'
elif val.startswith("[") and val.endswith("]"):
val = '"' + val[1:-1].replace("]]", "]").replace('"', '\\"') + '"'
elif val.startswith("+"):
val = val[1:]
un = ast.literal_eval(val).replace(".", "\\.")
return un
def to_string(tokens):
val = tokens[0]
val = "'" + val[1:-1].replace("''", "\\'") + "'"
return {"literal": ast.literal_eval(val)}
# NUMBERS
real_num = (
Regex(r"[+-]?(\d+\.\d*|\.\d+)([eE][+-]?\d+)?").set_parser_name("float")
/ (lambda t: float(t[0]))
)
def parse_int(tokens):
if "e" in tokens[0].lower():
return int(float(tokens[0]))
else:
return int(tokens[0])
int_num = Regex(r"[+-]?\d+([eE]\+?\d+)?").set_parser_name("int") / parse_int
hex_num = (
Regex(r"0x[0-9a-fA-F]+").set_parser_name("hex") / (lambda t: {"hex": t[0][2:]})
)
# STRINGS
ansi_string = Regex(r"\'(\'\'|[^'])*\'") / to_string
mysql_doublequote_string = Regex(r'\"(\"\"|[^"])*\"') / to_string
# BASIC IDENTIFIERS
ansi_ident = Regex(r'\"(\"\"|[^"])*\"') / unquote
mysql_backtick_ident = Regex(r"\`(\`\`|[^`])*\`") / unquote
sqlserver_ident = Regex(r"\[(\]\]|[^\]])*\]") / unquote
aquery_parser/windows.py
+107
View File
@@ -0,0 +1,107 @@
# encoding: utf-8
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
#
# Contact: Kyle Lahnakoski (kyle@lahnakoski.com)
#
from __future__ import absolute_import, division, unicode_literals
from mo_parsing.infix import delimited_list
from aquery_parser.keywords import *
from aquery_parser.utils import *
# https://docs.microsoft.com/en-us/sql/t-sql/queries/select-over-clause-transact-sql?view=sql-server-ver15
def _to_bound_call(tokens):
zero = tokens["zero"]
if zero:
return {"min": 0, "max": 0}
direction = scrub(tokens["direction"])
limit = scrub(tokens["limit"])
if direction == "preceding":
if limit == "unbounded":
return {"max": 0}
elif is_data(limit):
return {"min": {"neg": limit}, "max": 0}
else:
return {"min": -limit, "max": 0}
else: # following
if limit == "unbounded":
return {"min": 0}
elif is_data(limit):
return {"min": {"neg": limit}, "max": 0}
else:
return {"min": 0, "max": limit}
def _to_between_call(tokens):
minn = scrub(tokens["min"])
maxx = scrub(tokens["max"])
if maxx.get("max") == 0:
# following
return {
"min": minn.get("min"),
"max": maxx.get("min"),
}
elif minn.get("min") == 0:
# preceding
return {"min": minn.get("max"), "max": maxx.get("max")}
else:
return {
"min": minn.get("min"),
"max": maxx.get("max"),
}
UNBOUNDED = keyword("unbounded")
PRECEDING = keyword("preceding")
FOLLOWING = keyword("following")
CURRENT_ROW = keyword("current row")
ROWS = keyword("rows")
RANGE = keyword("range")
def window(expr, var_name, sort_column):
bound_row = (
CURRENT_ROW("zero")
| (UNBOUNDED | int_num)("limit") + (PRECEDING | FOLLOWING)("direction")
) / _to_bound_call
bound_expr = (
CURRENT_ROW("zero")
| (UNBOUNDED | expr)("limit") + (PRECEDING | FOLLOWING)("direction")
) / _to_bound_call
between_row = (
BETWEEN + bound_row("min") + AND + bound_row("max")
) / _to_between_call
between_expr = (
BETWEEN + bound_expr("min") + AND + bound_expr("max")
) / _to_between_call
row_clause = (ROWS.suppress() + (between_row | bound_row)) | (
RANGE.suppress() + (between_expr | bound_expr)
)
over_clause = (
LB
+ Optional(PARTITION_BY + delimited_list(Group(expr))("partitionby"))
+ Optional(ORDER_BY + delimited_list(Group(sort_column))("orderby"))
+ Optional(row_clause("range"))
+ RB
)
window_clause = Optional((
WITHIN_GROUP
+ LB
+ Optional(ORDER_BY + delimited_list(Group(sort_column))("orderby"))
+ RB
)("within")) + ((OVER + (over_clause | var_name) / to_over)("over"))
return window_clause, over_clause