diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 88f98d7..f35f8bf 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -23,7 +23,7 @@ jobs: - name: Build queryparser run: | - wget http://www.antlr.org/download/antlr-4.13.1-complete.jar + wget http://www.antlr.org/download/antlr-4.13.2-complete.jar make pip install -I -e .[test] pip install pytest-cov diff --git a/README.md b/README.md index b9a76c0..ba6bdc0 100644 --- a/README.md +++ b/README.md @@ -42,7 +42,7 @@ To generate the parsers you need `python3` , `java` above version The current version of `antlr-4.*-complete.jar` can be downloaded via ```bash -wget http://www.antlr.org/download/antlr-4.13.1-complete.jar +wget http://www.antlr.org/download/antlr-4.13.2-complete.jar ``` After cloning the project run @@ -63,9 +63,9 @@ to install the generated parser in your virtual environment. Additional requirements ----------------------- The queryparser assumes that the PostgreSQL database has the extension -[pg_sphere](https://github.com/kimakan/pgsphere/tree/aiprdbms16) installed. -Although the `pg_sphere` is not required for the python module, the PostgreSQL -**queries will not run** without this extension installed on the database. +[pg_sphere](https://github.com/postgrespro/pgsphere) installed. +Although `pg_sphere` is not required for the Python module, PostgreSQL +**queries will not run** without this extension installed in the database. Parsing MySQL and PostgreSQL @@ -120,7 +120,7 @@ the `ADQLQueryTranslator` class ```python from queryparser.adql import ADQLQueryTranslator -adql = "SELECT TOP 100 POINT('ICRS', ra, de) FROM db.tab;" +adql = "SELECT TOP 100 POINT('ICRS', ra, dec) FROM db.tab;" adt = ADQLQueryTranslator(adql) ``` @@ -130,7 +130,7 @@ and calling adt.to_postgresql() ``` -which returns a translated string representing a valid MySQL query if +which returns a translated string representing a valid PostgreSQL query if the ADQL query had no errors. The PostgreSQL query can then be parsed with the `PostgreSQLQueryProcessor` in the same way as shown above. @@ -149,4 +149,3 @@ then run the test suite with ```bash python -m pytest lib/ ``` - diff --git a/docs/development.md b/docs/development.md index 46fde71..4314571 100644 --- a/docs/development.md +++ b/docs/development.md @@ -9,8 +9,7 @@ processed and can be used to check, for example, for permissions before the query is sent to the server. In addition, queryparser can also work with ADQL queries which can be translated -to either MySQL or PostgreSQL language and then further processed the same -way. +to PostgreSQL and then further processed the same way. Below is a more detailed description along with instructions for anyone who is interested in further developing the package. @@ -49,14 +48,14 @@ Individual dialect functionality (MySQL in this case) with increased verbosity can be tested with ```bash -pytest /lib/queryparser/testing/test_mysql.py -v +pytest lib/queryparser/testing/test_mysql.py -v ``` Individual tests (20th MySQL test in this case, but otherwise any test that includes the string 't20') are ran with ```bash -pytest /lib/queryparser/testing/test_mysql.py -k t20 +pytest lib/queryparser/testing/test_mysql.py -k t20 ``` If the package `pytest-cov` is installed then the detailed coverage report @@ -120,7 +119,7 @@ and an ADQL translator. The processors accept any SELECT-like query and after the process_query() method has been executed, several elements of the query are extracted (all touch columns and tables, used keywords and functions). Before the processing the query is validated and invalid queries are rejected. -The ADQL translator allows translating valid ADQL queries to MySQL or PostgreSQL. +The ADQL translator allows translating valid ADQL queries to PostgreSQL. Most of the processor code is shared between MySQL and PostgreSQL and is therefore merged together inside of the `common.py`. This file consists diff --git a/generate.py b/generate.py index 46a5762..1914503 100755 --- a/generate.py +++ b/generate.py @@ -6,7 +6,7 @@ import subprocess import re -ANTLR_JAR = 'antlr-4.13.1-complete.jar' +ANTLR_JAR = 'antlr-4.13.2-complete.jar' ANTLR_DIRS = ('.', '/usr/local/lib/', '/usr/local/bin/') QUERYPARSER_SRC = 'src/queryparser/' diff --git a/pyproject.toml b/pyproject.toml index 35b4cc1..d741bdf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,10 +8,12 @@ description = "Package for parsing PostgreSQL/MySQL and translating ADQL to Post readme = "README.md" dynamic = ["version"] license = { text = "Apache-2.0" } -authors = [{ name = "Gal Matijevic", email = "gmatijevic@aip.de" }] +authors = [ + { name = "Kirill Makan", email = "kmakan@aip.de" }, + { name = "Gal Matijevic", email = "gmatijevic@aip.de" } +] maintainers = [ { name = "Kirill Makan", email = "kmakan@aip.de" }, - { name = "Simeon Reusch", email = "sreusch@aip.de" }, ] requires-python = ">=3.9" classifiers = [ @@ -24,8 +26,10 @@ classifiers = [ "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", ] -dependencies = ["antlr4-python3-runtime==4.13.1"] +dependencies = ["antlr4-python3-runtime==4.13.2"] [project.optional-dependencies] test = ["pytest~=8.2.0", "PyYAML~=6.0"] diff --git a/src/queryparser/__init__.py b/src/queryparser/__init__.py index c9880e6..32a90a3 100644 --- a/src/queryparser/__init__.py +++ b/src/queryparser/__init__.py @@ -1 +1 @@ -__version__ = '0.7.4' +__version__ = '0.8.0' diff --git a/src/queryparser/adql/ADQLLexer.g4 b/src/queryparser/adql/ADQLLexer.g4 index 97d80a0..b6fc338 100644 --- a/src/queryparser/adql/ADQLLexer.g4 +++ b/src/queryparser/adql/ADQLLexer.g4 @@ -316,14 +316,14 @@ HEX_DIGIT: ( '0x' (HEX_DIGIT_FRAGMENT)+ ) ; -CSL: SQ ('\u0020' .. '\u0026' | '\u0028' .. '\u007f')+ SQ; +CSL: SQ ( SQ SQ | '\u0020' .. '\u0026' | '\u0028' .. '\u007f' )* SQ; -ID: - (( 'A'..'Z' | 'a'..'z' | '_' | '%' | '$') ( 'A'..'Z' | 'a'..'z' | '_' | '$' | '%' | '0'..'9' )*) | - //(DQ NQC+ DQ) - (DQ ('\u0020' .. '\u0021' | '\u0023' .. '\u007f')+ DQ) -; +DELIMITED_ID: + DQ ('\u0020' .. '\u0021' | '\u0023' .. '\u007f')+ DQ; +ID: + ( 'A'..'Z' | 'a'..'z' ) + ( 'A'..'Z' | 'a'..'z' | '_' | '0'..'9' )*; //NDQC: 'A'..'Z' | 'a'..'z' | '_' | '0'..'9' ; diff --git a/src/queryparser/adql/ADQLParser.g4 b/src/queryparser/adql/ADQLParser.g4 index 8860330..6c6257a 100644 --- a/src/queryparser/adql/ADQLParser.g4 +++ b/src/queryparser/adql/ADQLParser.g4 @@ -23,7 +23,7 @@ box: ( coord_sys COMMA )? coordinates COMMA numeric_value_expression COMMA numeric_value_expression RPAREN ; -catalog_name: ID ; +catalog_name: ID | DELIMITED_ID ; // centroid: CENTROID LPAREN geometry_value_expression RPAREN ; // character_representation: nonquote_character ;// | SQ SQ ; char_function: LOWER LPAREN character_string_literal RPAREN ; @@ -52,7 +52,7 @@ coordinates: coordinate1 COMMA coordinate2 ; correlation_name: identifier ; correlation_specification: ( AS )? correlation_name ; //default_function_prefix: // this is empty in the document! -delimited_identifier: DQ ID DQ ; +delimited_identifier: DELIMITED_ID ; derived_column: value_expression ( as_clause )? ; derived_table: table_subquery ; distance: @@ -70,9 +70,9 @@ factor: ( sign )? numeric_primary ; from_clause: FROM table_reference ( COMMA table_reference )* ; general_literal: character_string_literal ; general_set_function: set_function_type LPAREN ( set_quantifier )? value_expression RPAREN ; -geometry_value_expression: box | circle | point | polygon | user_defined_function ; //| centroid | region +geometry_value_expression: value_expression_primary | box | circle | point | polygon | user_defined_function ; //| centroid | region group_by_clause: GROUP BY grouping_column_reference_list ; -grouping_column_reference: column_reference ; +grouping_column_reference: value_expression ; grouping_column_reference_list: grouping_column_reference ( COMMA grouping_column_reference )* ; having_clause: HAVING search_condition ; identifier: regular_identifier | delimited_identifier ; @@ -160,7 +160,7 @@ query_term: non_join_query_primary | query_term INTERSECT ( radius: numeric_value_expression ; //region: REGION LPAREN string_value_expression RPAREN ; regular_identifier: ID ; -schema_name: ID ; //( catalog_name DOT )? unqualified_schema_name ; +schema_name: ID | DELIMITED_ID ; //( catalog_name DOT )? unqualified_schema_name ; search_condition: boolean_term | search_condition OR boolean_term ; select_list: ( select_sublist ( COMMA select_sublist )* ) | ( ASTERISK ( COMMA select_sublist ( COMMA select_sublist )* )? ) ; select_query: SELECT ( set_quantifier )? ( set_limit )? select_list table_expression ; @@ -194,7 +194,6 @@ table_reference: table_subquery: subquery ; term: factor | term ASTERISK factor | term SOLIDUS factor | term MOD_SYM factor; trig_function: ACOS LPAREN numeric_value_expression RPAREN - | ACOS LPAREN numeric_value_expression RPAREN | ASIN LPAREN numeric_value_expression RPAREN | ATAN LPAREN numeric_value_expression RPAREN | ATAN2 LPAREN numeric_value_expression COMMA numeric_value_expression RPAREN @@ -202,7 +201,7 @@ trig_function: ACOS LPAREN numeric_value_expression RPAREN | COT LPAREN numeric_value_expression RPAREN | SIN LPAREN numeric_value_expression RPAREN | TAN LPAREN numeric_value_expression RPAREN ; -unqualified_schema_name: ID ; +unqualified_schema_name: ID | DELIMITED_ID ; unsigned_decimal: INT ; unsigned_hexadecimal: HEX_DIGIT ; unsigned_literal: unsigned_numeric_literal | general_literal ; @@ -216,7 +215,8 @@ user_defined_function: user_defined_function_name: regular_identifier ; //( default_function_prefix )? regular_identifier ; user_defined_function_param: value_expression ; value_expression: - numeric_value_expression + NULL + | numeric_value_expression | string_value_expression | boolean_value_expression | geometry_value_expression ; diff --git a/src/queryparser/adql/adqltranslator.py b/src/queryparser/adql/adqltranslator.py index f6d53f4..498bfbd 100644 --- a/src/queryparser/adql/adqltranslator.py +++ b/src/queryparser/adql/adqltranslator.py @@ -4,7 +4,6 @@ import antlr4 from antlr4.error.ErrorListener import ErrorListener -import re from .ADQLLexer import ADQLLexer from .ADQLParser import ADQLParser @@ -18,9 +17,9 @@ # between the name and left parenthesis is not allowed and needs to be # deleted. adql_function_names = ('ABS', 'ACOS', 'ASIN', 'ATAN', 'ATAN2', 'CEILING', - 'COS', 'DEGREES', 'EXP', 'FLOOR', 'LOG', 'LOG10', - 'MOD', 'PI', 'POWER', 'RADIANS', 'RAND', 'SIN', - 'SQRT', 'TAN', 'TRUNCATE') + 'COS', 'COT', 'DEGREES', 'EXP', 'FLOOR', 'LOG', 'LOG10', + 'LN', 'MOD', 'PI', 'POWER', 'RADIANS', 'RAND', + 'SIN', 'SQRT', 'TAN', 'TRUNCATE', 'TRUNC') def _removeFirstChild(ctx): @@ -78,6 +77,7 @@ def _process_regular_identifier(ctx_text, sql_output): else: return ri + def _get_ancestor_class_node(ctx, ancestor_class, depth=1): """ Returns the ancestor node at 'depth' level above the current node if the node is of type 'ancestor_class'. Otherwise, returns None @@ -113,15 +113,15 @@ class ADQLGeometryTranslationVisitor(ADQLParserVisitor): as a hash. This way we can return the hashed string instead the token so we effectively translate the rule. - :param conunits: - What should we be converting the units to. If no conversion is - necessary, just pass an empty string. + ADQL geometry coordinates and radii are specified in degrees. Native + pgSphere constructors use radians, so numeric geometry values are + converted when they cross into PostgreSQL. Geometry-valued expressions + are passed through unchanged. """ - def __init__(self, output_sql, conunits="RADIANS"): + def __init__(self, output_sql): self.contexts = {} self.output_sql = output_sql - self.conunits = conunits def _convert_values(self, ctx, cidx): """ @@ -163,6 +163,10 @@ def visitRegular_identifier(self, ctx): ri = _process_regular_identifier(ctx.getText(), self.output_sql) self.contexts[ctx] = ri + def visitDelimited_identifier(self, ctx): + ri = _process_regular_identifier(ctx.getText(), self.output_sql) + self.contexts[ctx] = ri + def visitSchema_name(self, ctx): ri = _process_regular_identifier(ctx.getText(), self.output_sql) self.contexts[ctx] = ri @@ -197,14 +201,13 @@ def visitPoint(self, ctx): coords = coords[1:] if self.output_sql == 'mysql': - ctx_text = "spoint( %s(%s), %s(%s) )" % (self.conunits, coords[0], - self.conunits, coords[1]) + ctx_text = "spoint( RADIANS(%s), RADIANS(%s) )" %\ + (coords[0], coords[1]) elif self.output_sql == 'postgresql': - ctx_text = "spoint( %s(%s), %s(%s) )" % (self.conunits, coords[0], - self.conunits, coords[1]) + ctx_text = "spoint( RADIANS(%s), RADIANS(%s) )" %\ + (coords[0], coords[1]) derived_column = _get_ancestor_class_node(ctx, ADQLParser.Derived_columnContext, depth=3) if derived_column is not None: - ctx_text = f"spoint_to_array_deg({ctx_text})" if not (any([isinstance(child, ADQLParser.As_clauseContext) for child in derived_column.children])): ctx_text = f"{ctx_text} AS adql_point" else: @@ -229,11 +232,10 @@ def visitBox(self, ctx): raise QueryError('sbox values incorrect') if self.output_sql in ('mysql', 'postgresql'): - ctx_text = "sbox( spoint(%s(%s),%s(%s)),spoint(%s(%s),%s(%s)) )" %\ - (self.conunits, '%.12f' % (pos_cent_ra - dra), - self.conunits, '%.12f' % (pos_cent_dec - ddec), - self.conunits, '%.12f' % (pos_cent_ra + dra), - self.conunits, '%.12f' % (pos_cent_dec + ddec)) + ctx_text = "sbox( spoint(RADIANS(%r),RADIANS(%r))," \ + "spoint(RADIANS(%r),RADIANS(%r)) )" %\ + (pos_cent_ra - dra, pos_cent_dec - ddec, + pos_cent_ra + dra, pos_cent_dec + ddec) else: ctx_text = '' @@ -249,9 +251,8 @@ def visitCircle(self, ctx): circle_center = ctx.children[s] if isinstance(circle_center.children[0], ADQLParser.CoordinatesContext): point_parameters = self._convert_values(circle_center, 0) - point_ctx_text = "spoint(%s(%s), %s(%s))" %\ - (self.conunits, point_parameters[0], - self.conunits, point_parameters[1]) + point_ctx_text = "spoint(RADIANS(%s), RADIANS(%s))" %\ + (point_parameters[0], point_parameters[1]) else: point_ctx = circle_center.children[0].children[0].children[0] if isinstance(point_ctx, ADQLParser.PointContext): @@ -263,12 +264,11 @@ def visitCircle(self, ctx): 'the circle center. For instance, ' + 'CIRCLE(POINT(t.ra, r.dec), 0.1)') - ctx_text = "scircle( %s, %s(%s) )" %\ - (point_ctx_text, self.conunits, radius) + ctx_text = "scircle( %s, RADIANS(%s) )" %\ + (point_ctx_text, radius) if self.output_sql == 'postgresql': derived_column = _get_ancestor_class_node(ctx, ADQLParser.Derived_columnContext, depth=3) if derived_column is not None: - ctx_text = f"scircle_to_array_deg({ctx_text})" if not (any([isinstance(child, ADQLParser.As_clauseContext) for child in derived_column.children])): ctx_text = f"{ctx_text} AS circle" _remove_children(ctx) @@ -284,19 +284,15 @@ def visitPolygon(self, ctx): if len(par) > 1: pars.append(par) - ustr = '' - if self.conunits == "RADIANS": - ustr = 'd' - if self.output_sql in ('mysql', 'postgresql'): ctx_text = "spoly('{" for p in pars: - ctx_text += '(%s%s,%s%s),' % (str(p[0]), ustr, str(p[1]), ustr) + # The standard spoly input accepts degree-suffixed values. + ctx_text += '(%sd,%sd),' % (str(p[0]), str(p[1])) ctx_text = ctx_text[:-1] + "}')" if self.output_sql == 'postgresql': derived_column = _get_ancestor_class_node(ctx, ADQLParser.Derived_columnContext, depth=3) if derived_column is not None: - ctx_text = f"spoly_to_array_deg({ctx_text})" if not (any([isinstance(child, ADQLParser.As_clauseContext) for child in derived_column.children])): ctx_text = f"{ctx_text} AS adql_polygon" else: @@ -315,13 +311,35 @@ class ADQLFunctionsTranslationVisitor(ADQLParserVisitor): the replaced geometry chunks. """ - def __init__(self, contexts, output_sql, conunits="DEGREES"): + def __init__(self, contexts, output_sql): self.contexts = contexts self.output_sql = output_sql - self.conunits = conunits + + def _get_geometry_value(self, ctx, child_index): + value = ctx.children[child_index].children[0] + return self.contexts.get(value, value.getText()) + + def visitMath_function(self, ctx): + self.visitChildren(ctx) + + if self.output_sql != 'postgresql': + return + + function_name = ctx.children[0].getText().upper() + replacement = { + 'LOG': 'LN', + 'LOG10': 'LOG', + 'TRUNCATE': 'trunc', + }.get(function_name) + + if replacement is not None: + self.contexts[ctx.children[0]] = replacement + elif function_name == 'RAND': + self.contexts[ctx] = 'random()' + _remove_children(ctx) def visitArea(self, ctx): - arg = self.contexts[ctx.children[2].children[0]] + arg = self._get_geometry_value(ctx, 2) if self.output_sql == 'mysql': ctx_text = 'sarea(%s)' % arg elif self.output_sql == 'postgresql': @@ -383,8 +401,7 @@ def visitContains_predicate(self, ctx): def visitContains(self, ctx): - arg = (self.contexts[ctx.children[2].children[0]], - self.contexts[ctx.children[4].children[0]]) + arg = (self._get_geometry_value(ctx, 2), self._get_geometry_value(ctx, 4)) if self.output_sql == 'mysql': ctx_text = 'srcontainsl(%s, %s)' % arg @@ -396,34 +413,73 @@ def visitContains(self, ctx): _remove_children(ctx) self.contexts[ctx] = ctx_text - def visitDistance(self, ctx): + def visitComparison_predicate(self, ctx): + """Rewrite DISTANCE thresholds, which require the comparison's radius. + + visitDistance handles the distance function itself; this method owns + the surrounding operator and radius needed for spoint_dwithin. + """ + if self.output_sql != 'postgresql' or ctx.getChildCount() != 3: + return self.visitChildren(ctx) + + left, operator, right = ctx.children + if operator.getText() not in ('<', '<='): + return self.visitChildren(ctx) + + distance = left + while distance.getChildCount() == 1: + distance = distance.children[0] + if not isinstance(distance, ADQLParser.DistanceContext): + return self.visitChildren(ctx) + if 'DISTANCE' in right.getText().upper(): + return self.visitChildren(ctx) + + ancestor = ctx.parentCtx + while ancestor is not None and not isinstance( + ancestor, (ADQLParser.Where_clauseContext, + ADQLParser.Join_conditionContext)): + ancestor = ancestor.parentCtx + if ancestor is None: + return self.visitChildren(ctx) + + _remove_children(ctx) + self.visitDistance(distance, right.getText()) + + def visitDistance(self, ctx, dwithin_radius=None): arg = ('', '') - if isinstance(ctx.children[2],ADQLParser.Coord_valueContext): - if isinstance(ctx.children[2].children[0].children[0], ADQLParser.PointContext): - point_ctx1 = ctx.children[2].children[0].children[0] - point_ctx2 = ctx.children[4].children[0].children[0] - arg = (self.contexts[point_ctx1], - self.contexts[point_ctx2]) - else: + if isinstance(ctx.children[2], ADQLParser.Coord_valueContext): + values = (ctx.children[2].children[0], ctx.children[4].children[0]) + if self.output_sql != 'postgresql' and not all( + isinstance(value, ADQLParser.Point_valueContext) for value in values): raise QueryError('Distance in the current implementation is ' + 'possible only between two explicitly ' + 'defined points. For instance, ' + 'DISTANCE(POINT(t.ra, r.dec), POINT(0.0, 0.0)) ' + 'or DISTANCE(t.ra, r.dec, 0.0, 0.0)') + arg = tuple(self.contexts.get( + value.children[0] if isinstance(value, ADQLParser.Point_valueContext) else value, + value.getText()) for value in values) else: - arg = (f"spoint(RADIANS({_convert_values(ctx, 2, self.output_sql)[0]}), " + - f"RADIANS({_convert_values(ctx, 4, self.output_sql)[0]}))", - f"spoint(RADIANS({_convert_values(ctx, 6, self.output_sql)[0]}), " + - f"RADIANS({_convert_values(ctx, 8, self.output_sql)[0]}))") + arg = ( + "spoint(RADIANS(%s), RADIANS(%s))" % + (_convert_values(ctx, 2, self.output_sql)[0], + _convert_values(ctx, 4, self.output_sql)[0]), + "spoint(RADIANS(%s), RADIANS(%s))" % + (_convert_values(ctx, 6, self.output_sql)[0], + _convert_values(ctx, 8, self.output_sql)[0]), + ) if self.output_sql == 'mysql': - ctx_text = '%s(sdist(%s, %s))' % ((self.conunits, ) + arg) + ctx_text = 'DEGREES(sdist(%s, %s))' % arg elif self.output_sql == 'postgresql': - ctx_text = '%s(%s <-> %s)' % ((self.conunits, ) + arg) + ctx_text = 'DEGREES(%s <-> %s)' % arg derived_column = _get_ancestor_class_node(ctx, ADQLParser.Derived_columnContext, depth=9) if derived_column is not None: if not (any([isinstance(child, ADQLParser.As_clauseContext) for child in derived_column.children])): ctx_text = f"{ctx_text} AS distance" + if dwithin_radius is not None: + ctx_text = 'spoint_dwithin(%s, %s, %s)' %\ + (arg + ('RADIANS(%s)' % dwithin_radius, )) else: ctx_text = '' @@ -458,8 +514,7 @@ def visitIntersects_predicate(self, ctx): def visitIntersects(self, ctx): - arg = (self.contexts[ctx.children[2].children[0]], - self.contexts[ctx.children[4].children[0]]) + arg = (self._get_geometry_value(ctx, 2), self._get_geometry_value(ctx, 4)) if self.output_sql == 'mysql': ctx_text = 'soverlaps(%s, %s)' % arg @@ -542,13 +597,16 @@ def visitTerminal(self, node): pass try: - nd = self.contexts[node.parentCtx] + nd = self.contexts[node] except KeyError: - nd = node.getText() - if isinstance(node.parentCtx, - ADQLParser.Character_string_literalContext): - if nd == "'": - nd = None + try: + nd = self.contexts[node.parentCtx] + except KeyError: + nd = node.getText() + if isinstance(node.parentCtx, + ADQLParser.Character_string_literalContext): + if nd == "'": + nd = None if nd is not None: if isinstance(node.parentCtx, ADQLParser.Set_function_typeContext)\ @@ -684,8 +742,4 @@ def to_postgresql(self): translated_query = self.translate(translator_visitor) - # Translate LOG10 to LOG and LOG to LN. It's not the most elegant solution but it works. - translated_query = re.sub(r'(?<=[\+\-\*/\(\s,])log\(', 'LN(', translated_query, flags=re.IGNORECASE) - translated_query = re.sub(r'(?<=[\+\-\*/\(\s,])log10\(', 'LOG(', translated_query, flags=re.IGNORECASE) - return translated_query diff --git a/src/queryparser/postgresql/PostgreSQLLexer.g4 b/src/queryparser/postgresql/PostgreSQLLexer.g4 index f307aa8..7e79480 100644 --- a/src/queryparser/postgresql/PostgreSQLLexer.g4 +++ b/src/queryparser/postgresql/PostgreSQLLexer.g4 @@ -171,6 +171,7 @@ THEN_SYM : T_ H_ E_ N_ ; TIME_SYM : T_ I_ M_ E_ ; TIMESTAMP : T_ I_ M_ E_ S_ T_ A_ M_ P_ ; TRUE_SYM : T_ R_ U_ E_ ; +TRUNC : T_ R_ U_ N_ C_ ; TRUNCATE : T_ R_ U_ N_ C_ A_ T_ E_ ; UDF_0 : U_ D_ F_ '_' '0' ; UDF_1 : U_ D_ F_ '_' '1' ; @@ -207,6 +208,7 @@ UTF8 : U_ T_ F_ '8' ; //pg_sphere SPOINT : S_ P_ O_ I_ N_ T_ ; +SPOINT_DWITHIN : S_ P_ O_ I_ N_ T_ '_' D_ W_ I_ T_ H_ I_ N_ ; SCIRCLE : S_ C_ I_ R_ C_ L_ E_ ; SLINE : S_ L_ I_ N_ E_ ; SELLIPSE : S_ E_ L_ L_ I_ P_ S_ E_ ; @@ -324,4 +326,3 @@ WS '\u2004' | '\u2005' | '\u2006' | '\u2007' | '\u2008' | '\u2009' | '\u200a' | '\u2028' | '\u2029' | '\u202f' | '\u205f' | '\u3000' )+ -> channel(HIDDEN) ; - diff --git a/src/queryparser/postgresql/PostgreSQLParser.g4 b/src/queryparser/postgresql/PostgreSQLParser.g4 index e7ce066..3410ad7 100644 --- a/src/queryparser/postgresql/PostgreSQLParser.g4 +++ b/src/queryparser/postgresql/PostgreSQLParser.g4 @@ -50,7 +50,7 @@ number_functions: ABS | ACOS | ASIN | ATAN2 | ATAN | CBRT | CEIL | CEILING | COS | COT | DEGREES | DIV | EXP | FLOOR | LN | LOG | MOD | PI | POW | POWER | RADIANS | RANDOM | ROUND | SIGN | SIN | SQUARE_DEGREES | SQRT - | STERADIANS | TAN | TRUNCATE ; + | STERADIANS | TAN | TRUNC | TRUNCATE ; other_functions: ENCODE | MD5 ; @@ -66,7 +66,7 @@ custom_functions: GAIA_HEALPIX_INDEX | PDIST | UDF_0 | UDF_1 | UDF_2 | UDF_3 | UDF_4 | UDF_5 | UDF_6 | UDF_7 | UDF_8 | UDF_9 ; pg_sphere_functions: - AREA ; + AREA | SPOINT_DWITHIN ; functionList: number_functions | char_functions | time_functions | other_functions @@ -120,13 +120,7 @@ column_name: ID; column_spec: ( ( schema_name DOT )? table_name DOT )? column_name ( slice_spec )?; displayed_column : ( table_spec DOT ASTERISK ) - | ( ( bit_expr | sbit_expr | displayed_column_arr ) ( ( LIKE_SYM | ILIKE_SYM ) TEXT_STRING )? ( alias )? ) ; - -displayed_column_arr : - ( spoint_to_array_deg | spoint_to_array - | sbox_to_array_deg | sbox_to_array - | scircle_to_array_deg | scircle_to_array - | spoly_to_array_deg | spoly_to_array); + | ( ( bit_expr | sbit_expr ) ( ( LIKE_SYM | ILIKE_SYM ) TEXT_STRING )? ( alias )? ) ; exp_factor1: exp_factor2 ( AND_SYM exp_factor2 )* ; exp_factor2: ( NOT_SYM )? exp_factor3 ; @@ -268,13 +262,5 @@ sbox: SBOX LPAREN spoint COMMA spoint RPAREN ; spoly: SPOLY TEXT_STRING| SPOLY LPAREN column_spec RPAREN | SPOLY LPAREN TEXT_STRING RPAREN; spath: SPATH TEXT_STRING | SPATH LPAREN column_spec RPAREN ; strans: STRANS LPAREN bit_expr COMMA bit_expr COMMA bit_expr COMMA TRANS RPAREN ; -spoint_to_array: SPOINT_TO_ARRAY LPAREN spoint RPAREN ; -spoint_to_array_deg: SPOINT_TO_ARRAY_DEG LPAREN spoint RPAREN ; -sbox_to_array: SBOX_TO_ARRAY LPAREN sbox RPAREN ; -sbox_to_array_deg: SBOX_TO_ARRAY_DEG LPAREN sbox RPAREN ; -scircle_to_array: SCIRCLE_TO_ARRAY LPAREN scircle RPAREN ; -scircle_to_array_deg: SCIRCLE_TO_ARRAY_DEG LPAREN scircle RPAREN ; -spoly_to_array: SPOLY_TO_ARRAY LPAREN spoly RPAREN ; -spoly_to_array_deg: SPOLY_TO_ARRAY_DEG LPAREN spoly RPAREN ; pg_sphere_object: scircle | sline | sellipse | sbox | spoly | spath | simple_expr; diff --git a/src/queryparser/testing/tests.yaml b/src/queryparser/testing/tests.yaml index 5f2733a..75bc25d 100644 --- a/src/queryparser/testing/tests.yaml +++ b/src/queryparser/testing/tests.yaml @@ -875,11 +875,11 @@ adql_mysql_tests: - - SELECT TOP 10 ra, dec FROM db.tab WHERE 1=CONTAINS(POINT('ICRS', ra, dec), BOX('ICRS', -3.0, 5.0, 4.0, 10.0)); - - SELECT `ra`, `dec` FROM `db`.`tab` WHERE 1 = srcontainsl(spoint(RADIANS(`ra`), RADIANS(`dec`)), sbox(spoint(RADIANS(-5.000000000000),RADIANS(0.000000000000)),spoint(RADIANS(-1.000000000000),RADIANS(10.000000000000)))) LIMIT 10; + - SELECT `ra`, `dec` FROM `db`.`tab` WHERE 1 = srcontainsl(spoint(RADIANS(`ra`), RADIANS(`dec`)), sbox(spoint(RADIANS(-5.0),RADIANS(0.0)),spoint(RADIANS(-1.0),RADIANS(10.0)))) LIMIT 10; - - SELECT TOP 10 ra, dec FROM db.tab WHERE 0=CONTAINS(POINT('ICRS', ra, dec), BOX('ICRS', -3.0, 5.0, 4.0, 10.0)); - - SELECT `ra`, `dec` FROM `db`.`tab` WHERE 0 = srcontainsl(spoint(RADIANS(`ra`), RADIANS(`dec`)), sbox(spoint(RADIANS(-5.000000000000),RADIANS(0.000000000000)),spoint(RADIANS(-1.000000000000),RADIANS(10.000000000000)))) LIMIT 10; + - SELECT `ra`, `dec` FROM `db`.`tab` WHERE 0 = srcontainsl(spoint(RADIANS(`ra`), RADIANS(`dec`)), sbox(spoint(RADIANS(-5.0),RADIANS(0.0)),spoint(RADIANS(-1.0),RADIANS(10.0)))) LIMIT 10; - - SELECT TOP 10 DISTANCE(POINT('ICRS', ra, dec), POINT('ICRS', 13.66, -58.3)) FROM db.tab; @@ -889,6 +889,32 @@ adql_mysql_tests: # coordsys will ensure the backward compatibility for adql2.0 but can be removed # later. adql_postgresql_tests: + - + - SELECT RAND(), TRUNCATE(parallax, 2) FROM db.tab + - SELECT random(), trunc(parallax, 2) FROM db.tab; + + - + - SELECT NULL AS null_value, a + 1 AS shifted + FROM db.tab + GROUP BY a + 1 + - SELECT NULL AS null_value, a + 1 AS shifted FROM db.tab GROUP BY a + 1; + + - + - SELECT * FROM db.tab + WHERE 1 = CONTAINS((pos), CIRCLE(0.0, 0.0, 1.0)) + - SELECT * FROM db.tab WHERE (pos) @ scircle(spoint(RADIANS(0.0), RADIANS(0.0)), RADIANS(1.0)); + + - + - SELECT * FROM db.tab + WHERE 1 = CONTAINS(stored_spoint, CIRCLE(POINT(271.528, -30.171), 0.01)) + - SELECT * FROM db.tab WHERE stored_spoint @ scircle(spoint(RADIANS(271.528), RADIANS(-30.171)), RADIANS(0.01)); + + - + - SELECT "source"."column" AS "alias" + FROM "schema"."table" AS "source" + - SELECT "source"."column" AS "alias" + FROM "schema"."table" AS "source"; + - - SELECT TOP 10 LOG(a) as a FROM db.tab - SELECT LN(a) AS a FROM db.tab LIMIT 10; @@ -915,39 +941,39 @@ adql_postgresql_tests: - - SELECT POINT('icrs', 10, 10) AS "p" FROM "db".tab - - SELECT spoint_to_array_deg(spoint(RADIANS(10.0), RADIANS(10.0))) AS "p" FROM "db".tab; + - SELECT spoint(RADIANS(10.0), RADIANS(10.0)) AS "p" FROM "db".tab; - - SELECT POINT('icrs', 10, 10) FROM "db".tab - - SELECT spoint_to_array_deg(spoint(RADIANS(10.0), RADIANS(10.0))) AS adql_point FROM "db".tab; + - SELECT spoint(RADIANS(10.0), RADIANS(10.0)) AS adql_point FROM "db".tab; - - SELECT POINT(10, 10) AS "p" FROM "db".tab - - SELECT spoint_to_array_deg(spoint(RADIANS(10.0), RADIANS(10.0))) AS "p" FROM "db".tab; + - SELECT spoint(RADIANS(10.0), RADIANS(10.0)) AS "p" FROM "db".tab; - - SELECT CIRCLE(10.0, -10.5, 2.0) FROM db.tab; - - SELECT scircle_to_array_deg(scircle(spoint(RADIANS(10.0), RADIANS(-10.5)), RADIANS(2.0))) AS circle FROM db.tab; + - SELECT scircle(spoint(RADIANS(10.0), RADIANS(-10.5)), RADIANS(2.0)) AS circle FROM db.tab; - - SELECT POLYGON('ICRS', 10.0, -10.5, 20.0, 20.5, 30.0, 30.5) FROM db.tab; - - SELECT spoly_to_array_deg(spoly('{(10.0d,-10.5d),(20.0d,20.5d),(30.0d,30.5d)}')) AS adql_polygon FROM db.tab; + - SELECT spoly('{(10.0d,-10.5d),(20.0d,20.5d),(30.0d,30.5d)}') AS adql_polygon FROM db.tab; - - SELECT CIRCLE(POINT(10.0, -10.5), 2.0) FROM db.tab; - - SELECT scircle_to_array_deg(scircle(spoint(RADIANS(10.0), RADIANS(-10.5)), RADIANS(2.0))) AS circle FROM db.tab; + - SELECT scircle(spoint(RADIANS(10.0), RADIANS(-10.5)), RADIANS(2.0)) AS circle FROM db.tab; - - SELECT TOP 10 ra, dec FROM db.tab WHERE 1=CONTAINS(POINT('ICRS', ra, dec), BOX('ICRS', -3.0, 5.0, 4.0, 10.0)); - - SELECT ra, dec FROM db.tab WHERE spoint(RADIANS(ra), RADIANS(dec)) @ sbox(spoint(RADIANS(-5.000000000000),RADIANS(0.000000000000)),spoint(RADIANS(-1.000000000000),RADIANS(10.000000000000))) LIMIT 10; + - SELECT ra, dec FROM db.tab WHERE spoint(RADIANS(ra), RADIANS(dec)) @ sbox(spoint(RADIANS(-5.0),RADIANS(0.0)),spoint(RADIANS(-1.0),RADIANS(10.0))) LIMIT 10; - - SELECT TOP 10 ra, dec FROM db.tab WHERE CONTAINS(POINT('ICRS', ra, dec), BOX('ICRS', -3.0, 5.0, 4.0, 10.0)) = 1; - - SELECT ra, dec FROM db.tab WHERE spoint(RADIANS(ra), RADIANS(dec)) @ sbox(spoint(RADIANS(-5.000000000000),RADIANS(0.000000000000)),spoint(RADIANS(-1.000000000000),RADIANS(10.000000000000))) LIMIT 10; + - SELECT ra, dec FROM db.tab WHERE spoint(RADIANS(ra), RADIANS(dec)) @ sbox(spoint(RADIANS(-5.0),RADIANS(0.0)),spoint(RADIANS(-1.0),RADIANS(10.0))) LIMIT 10; - - SELECT TOP 10 ra, dec FROM db.tab WHERE 1=CONTAINS(POINT(ra, dec), BOX(-3.0, 5.0, 4.0, 10.0)); - - SELECT ra, dec FROM db.tab WHERE spoint(RADIANS(ra), RADIANS(dec)) @ sbox(spoint(RADIANS(-5.000000000000),RADIANS(0.000000000000)),spoint(RADIANS(-1.000000000000),RADIANS(10.000000000000))) LIMIT 10; + - SELECT ra, dec FROM db.tab WHERE spoint(RADIANS(ra), RADIANS(dec)) @ sbox(spoint(RADIANS(-5.0),RADIANS(0.0)),spoint(RADIANS(-1.0),RADIANS(10.0))) LIMIT 10; - - SELECT TOP 10 ra, dec FROM db.tab WHERE 1=CONTAINS(POINT(ra, dec), CIRCLE(POINT(-3.0, 4.0), 10.0)); @@ -977,9 +1003,33 @@ adql_postgresql_tests: - SELECT TOP 10 DISTANCE(ra, dec, 13.66, -58.3) FROM db.tab; - SELECT DEGREES(spoint(RADIANS(ra), RADIANS(dec)) <-> spoint(RADIANS(13.66), RADIANS(-58.3))) AS distance FROM db.tab LIMIT 10; + - + - SELECT TOP 10 ra, dec FROM db.tab + WHERE DISTANCE(POINT(ra, dec), POINT(13.66, -58.3)) <= 2.5; + - SELECT ra, dec FROM db.tab + WHERE spoint_dwithin(spoint(RADIANS(ra), RADIANS(dec)), spoint(RADIANS(13.66), RADIANS(-58.3)), RADIANS(2.5)) LIMIT 10; + + - + - SELECT ra, dec FROM db.tab + WHERE DISTANCE(ra, dec, 13.66, -58.3) < 0.00027; + - SELECT ra, dec FROM db.tab + WHERE spoint_dwithin(spoint(RADIANS(ra), RADIANS(dec)), spoint(RADIANS(13.66), RADIANS(-58.3)), RADIANS(0.00027)); + + - + - SELECT * FROM db.tab1 t1 JOIN db.tab2 t2 + ON DISTANCE(t1.ra, t1.dec, t2.ra, t2.dec) < 0.00027; + - SELECT * FROM db.tab1 t1 JOIN db.tab2 t2 + ON spoint_dwithin(spoint(RADIANS(t1.ra), RADIANS(t1.dec)), spoint(RADIANS(t2.ra), RADIANS(t2.dec)), RADIANS(0.00027)); + + - + - SELECT * FROM db.tab + WHERE DISTANCE(pos, POINT(13.66, -58.3)) <= 2.5; + - SELECT * FROM db.tab + WHERE spoint_dwithin(pos, spoint(RADIANS(13.66), RADIANS(-58.3)), RADIANS(2.5)); + - - SELECT INTERSECTS(CIRCLE('ICRS', 0, 0, 10), BOX('ICRS', 2, -3, 4, 4)) FROM db.tab; - - SELECT scircle(spoint(RADIANS(0.0), RADIANS(0.0)), RADIANS(10.0)) && sbox(spoint(RADIANS(0.000000000000),RADIANS(-5.000000000000)),spoint(RADIANS(4.000000000000),RADIANS(-1.000000000000))) FROM db.tab; + - SELECT scircle(spoint(RADIANS(0.0), RADIANS(0.0)), RADIANS(10.0)) && sbox(spoint(RADIANS(0.0),RADIANS(-5.0)),spoint(RADIANS(4.0),RADIANS(-1.0))) FROM db.tab; - - SELECT TOP 10 LOG10(ra), LOG(dec) FROM db.tab WHERE 0=INTERSECTS(POINT('ICRS', ra, dec), POLYGON('ICRS', 10.0, -10.5, 20.0, 20.5, 30.0, 30.5));