From 723a41d17a05b8b09f793f2e1b1a1c8d07a25730 Mon Sep 17 00:00:00 2001 From: Costin Leau Date: Sat, 15 Mar 2025 20:37:13 -0700 Subject: [PATCH] ESQL: Catch parsing exception (#124958) When an invalid popMode occurs (due to an invalid query), ANTLR throws an out-of-channel exception, bypassing the existing checks. This PR extends the checks and properly reports the error back to the user Fix #119025 (cherry picked from commit dc15462ca4f97cf622a284afa8ac0e797f02f8f1) # Conflicts: # x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlParser.java # x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java --- docs/changelog/124958.yaml | 6 + .../xpack/esql/parser/EsqlParser.java | 64 +- .../esql/parser/StatementParserTests.java | 1077 ++++++++++++++++- 3 files changed, 1085 insertions(+), 62 deletions(-) create mode 100644 docs/changelog/124958.yaml diff --git a/docs/changelog/124958.yaml b/docs/changelog/124958.yaml new file mode 100644 index 0000000000000..be7f646b7dcae --- /dev/null +++ b/docs/changelog/124958.yaml @@ -0,0 +1,6 @@ +pr: 124958 +summary: Catch parsing exception +area: ES|QL +type: bug +issues: + - 119025 \ No newline at end of file diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlParser.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlParser.java index 5912f1fe58bcd..a8ee18d8b2777 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlParser.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlParser.java @@ -14,6 +14,7 @@ import org.antlr.v4.runtime.Recognizer; import org.antlr.v4.runtime.Token; import org.antlr.v4.runtime.TokenSource; +import org.antlr.v4.runtime.VocabularyImpl; import org.antlr.v4.runtime.atn.PredictionMode; import org.elasticsearch.logging.LogManager; import org.elasticsearch.logging.Logger; @@ -23,12 +24,15 @@ import org.elasticsearch.xpack.esql.telemetry.PlanTelemetry; import java.util.BitSet; +import java.util.EmptyStackException; +import java.util.Map; import java.util.function.BiFunction; import java.util.function.Function; import java.util.regex.Matcher; import java.util.regex.Pattern; import static org.elasticsearch.xpack.esql.core.util.StringUtils.isInteger; +import static org.elasticsearch.xpack.esql.parser.ParserUtils.nameOrPosition; import static org.elasticsearch.xpack.esql.parser.ParserUtils.source; public class EsqlParser { @@ -44,6 +48,45 @@ public class EsqlParser { */ public static final int MAX_LENGTH = 1_000_000; + private static void replaceSymbolWithLiteral(Map symbolReplacements, String[] literalNames, String[] symbolicNames) { + for (int i = 0, replacements = symbolReplacements.size(); i < symbolicNames.length && replacements > 0; i++) { + String symName = symbolicNames[i]; + if (symName != null) { + String replacement = symbolReplacements.get(symName); + if (replacement != null && literalNames[i] == null) { + // literals are single quoted + literalNames[i] = "'" + replacement + "'"; + replacements--; + } + } + } + } + + /** + * Add the literal name to a number of tokens that due to ANTLR internals/ATN + * have their symbolic name returns instead during error reporting. + * When reporting token errors, ANTLR uses the Vocabulary class to get the displayName + * (if set), otherwise falls back to the literal one and eventually uses the symbol name. + * Since the Vocabulary is static and not pluggable, this code modifies the underlying + * arrays by setting the literal string manually based on the token index. + * This is needed since some symbols, especially around setting up the mode, end up losing + * their literal representation. + * NB: this code is highly dependent on the ANTLR internals and thus will likely break + * during upgrades. + * NB: Can't use this for replacing DEV_ since the Vocabular is static while DEV_ replacement occurs per runtime configuration + */ + static { + Map symbolReplacements = Map.of("LP", "(", "OPENING_BRACKET", "["); + + // the vocabularies have the same content however are different instances + // for extra reliability, perform the replacement for each map + VocabularyImpl parserVocab = (VocabularyImpl) EsqlBaseParser.VOCABULARY; + replaceSymbolWithLiteral(symbolReplacements, parserVocab.getLiteralNames(), parserVocab.getSymbolicNames()); + + VocabularyImpl lexerVocab = (VocabularyImpl) EsqlBaseLexer.VOCABULARY; + replaceSymbolWithLiteral(symbolReplacements, lexerVocab.getLiteralNames(), lexerVocab.getSymbolicNames()); + } + private EsqlConfig config = new EsqlConfig(); public EsqlConfig config() { @@ -111,6 +154,9 @@ private T invokeParser( return result.apply(new AstBuilder(new ExpressionBuilder.ParsingContext(params, metrics)), tree); } catch (StackOverflowError e) { throw new ParsingException("ESQL statement is too large, causing stack overflow when generating the parsing tree: [{}]", query); + // likely thrown by an invalid popMode (such as extra closing parenthesis) + } catch (EmptyStackException ese) { + throw new ParsingException("Invalid query [{}]", query); } } @@ -141,11 +187,14 @@ public void syntaxError( String message, RecognitionException e ) { - if (recognizer instanceof EsqlBaseParser parser && parser.isDevVersion() == false) { - Matcher m = REPLACE_DEV.matcher(message); - message = m.replaceAll(StringUtils.EMPTY); - } + if (recognizer instanceof EsqlBaseParser parser) { + Matcher m; + if (parser.isDevVersion() == false) { + m = REPLACE_DEV.matcher(message); + message = m.replaceAll(StringUtils.EMPTY); + } + } throw new ParsingException(message, e, line, charPositionInLine); } }; @@ -172,7 +221,7 @@ private static class ParametrizedTokenSource extends DelegatingTokenSource { @Override public Token nextToken() { Token token = delegate.nextToken(); - if (token.getType() == EsqlBaseLexer.PARAM) { + if (token.getType() == EsqlBaseLexer.PARAM || token.getType() == EsqlBaseLexer.DOUBLE_PARAMS) { checkAnonymousParam(token); if (param > params.size()) { throw new ParsingException(source(token), "Not enough actual parameters {}", params.size()); @@ -181,8 +230,9 @@ public Token nextToken() { param++; } - if (token.getType() == EsqlBaseLexer.NAMED_OR_POSITIONAL_PARAM) { - if (isInteger(token.getText().substring(1))) { + String nameOrPosition = nameOrPosition(token); + if (nameOrPosition.isBlank() == false) { + if (isInteger(nameOrPosition)) { checkPositionalParam(token); } else { checkNamedParam(token); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java index c8ff6ef1b55ae..9d2e7be770a18 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java @@ -16,6 +16,7 @@ import org.elasticsearch.xpack.esql.core.expression.Alias; import org.elasticsearch.xpack.esql.core.expression.Attribute; import org.elasticsearch.xpack.esql.core.expression.EmptyAttribute; +import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.Expressions; import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Literal; @@ -44,12 +45,14 @@ import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThanOrEqual; import org.elasticsearch.xpack.esql.plan.IndexPattern; import org.elasticsearch.xpack.esql.plan.logical.Aggregate; +import org.elasticsearch.xpack.esql.plan.logical.Dedup; import org.elasticsearch.xpack.esql.plan.logical.Dissect; import org.elasticsearch.xpack.esql.plan.logical.Drop; import org.elasticsearch.xpack.esql.plan.logical.Enrich; import org.elasticsearch.xpack.esql.plan.logical.Eval; import org.elasticsearch.xpack.esql.plan.logical.Explain; import org.elasticsearch.xpack.esql.plan.logical.Filter; +import org.elasticsearch.xpack.esql.plan.logical.Fork; import org.elasticsearch.xpack.esql.plan.logical.Grok; import org.elasticsearch.xpack.esql.plan.logical.InlineStats; import org.elasticsearch.xpack.esql.plan.logical.Keep; @@ -61,6 +64,7 @@ import org.elasticsearch.xpack.esql.plan.logical.Project; import org.elasticsearch.xpack.esql.plan.logical.Rename; import org.elasticsearch.xpack.esql.plan.logical.Row; +import org.elasticsearch.xpack.esql.plan.logical.RrfScoreEval; import org.elasticsearch.xpack.esql.plan.logical.UnresolvedRelation; import org.elasticsearch.xpack.esql.plan.logical.join.JoinTypes; import org.elasticsearch.xpack.esql.plan.logical.join.LookupJoin; @@ -398,8 +402,11 @@ public void testStatsWithoutGroupKeyMixedAggAndFilter() { public void testInlineStatsWithGroups() { var query = "inlinestats b = min(a) by c, d.e"; if (Build.current().isSnapshot() == false) { - var e = expectThrows(ParsingException.class, () -> processingCommand(query)); - assertThat(e.getMessage(), containsString("line 1:13: mismatched input 'inlinestats' expecting {")); + expectThrows( + ParsingException.class, + containsString("line 1:13: mismatched input 'inlinestats' expecting {"), + () -> processingCommand(query) + ); return; } assertEquals( @@ -424,8 +431,11 @@ public void testInlineStatsWithGroups() { public void testInlineStatsWithoutGroups() { var query = "inlinestats min(a), c = 1"; if (Build.current().isSnapshot() == false) { - var e = expectThrows(ParsingException.class, () -> processingCommand(query)); - assertThat(e.getMessage(), containsString("line 1:13: mismatched input 'inlinestats' expecting {")); + expectThrows( + ParsingException.class, + containsString("line 1:13: mismatched input 'inlinestats' expecting {"), + () -> processingCommand(query) + ); return; } assertEquals( @@ -858,16 +868,17 @@ public void testSuggestAvailableSourceCommandsOnParsingError() { Tuple.tuple("a/*hi*/", "a"), Tuple.tuple("explain [ frm a ]", "frm") )) { - ParsingException pe = expectThrows(ParsingException.class, () -> statement(queryWithUnexpectedCmd.v1())); - assertThat( - pe.getMessage(), + expectThrows( + ParsingException.class, allOf( containsString("mismatched input '" + queryWithUnexpectedCmd.v2() + "'"), containsString("'explain'"), containsString("'from'"), containsString("'row'") - ) + ), + () -> statement(queryWithUnexpectedCmd.v1()) ); + } } @@ -882,15 +893,15 @@ public void testSuggestAvailableProcessingCommandsOnParsingError() { Tuple.tuple("from a | a/*hi*/", "a"), Tuple.tuple("explain [ from a | evl b = c ]", "evl") )) { - ParsingException pe = expectThrows(ParsingException.class, () -> statement(queryWithUnexpectedCmd.v1())); - assertThat( - pe.getMessage(), + expectThrows( + ParsingException.class, allOf( containsString("mismatched input '" + queryWithUnexpectedCmd.v2() + "'"), containsString("'eval'"), containsString("'stats'"), containsString("'where'") - ) + ), + () -> statement(queryWithUnexpectedCmd.v1()) ); } } @@ -912,6 +923,13 @@ public void testDeprecatedIsNullFunction() { "line 1:22: is_null function is not supported anymore, please use 'is null'/'is not null' predicates instead" ); } + if (EsqlCapabilities.Cap.DOUBLE_PARAMETER_MARKERS_FOR_IDENTIFIERS.isEnabled()) { + expectError( + "from test | eval x = ??fn1(f)", + List.of(paramAsConstant("fn1", "IS_NULL")), + "line 1:22: is_null function is not supported anymore, please use 'is null'/'is not null' predicates instead" + ); + } } public void testMetadataFieldOnOtherSources() { @@ -981,10 +999,10 @@ public void testGrokPattern() { assertEquals("%{WORD:foo}", grok.parser().pattern()); assertEquals(List.of(referenceAttribute("foo", KEYWORD)), grok.extractedFields()); - ParsingException pe = expectThrows(ParsingException.class, () -> statement("row a = \"foo bar\" | grok a \"%{_invalid_:x}\"")); - assertThat( - pe.getMessage(), - containsString("Invalid pattern [%{_invalid_:x}] for grok: Unable to find pattern [_invalid_] in Grok's pattern dictionary") + expectThrows( + ParsingException.class, + containsString("Invalid pattern [%{_invalid_:x}] for grok: Unable to find pattern [_invalid_] in Grok's pattern dictionary"), + () -> statement("row a = \"foo bar\" | grok a \"%{_invalid_:x}\"") ); cmd = processingCommand("grok a \"%{WORD:foo} %{WORD:foo}\""); @@ -1123,8 +1141,7 @@ public void testKeepStarMvExpand() { public void testUsageOfProject() { String query = "from test | project foo, bar"; - ParsingException e = expectThrows(ParsingException.class, "Expected syntax error for " + query, () -> statement(query)); - assertThat(e.getMessage(), containsString("mismatched input 'project' expecting")); + expectThrows(ParsingException.class, containsString("mismatched input 'project' expecting"), () -> statement(query)); } public void testInputParams() { @@ -1200,6 +1217,11 @@ public void testInputParams() { public void testMissingInputParams() { expectError("row x = ?, y = ?", List.of(paramAsConstant(null, 1)), "Not enough actual parameters 1"); + + if (EsqlCapabilities.Cap.DOUBLE_PARAMETER_MARKERS_FOR_IDENTIFIERS.isEnabled()) { + expectError("from test | eval x = ??, y = ??", List.of(paramAsConstant(null, 1)), "Not enough actual parameters 1"); + expectError("from test | eval x = ??, y = ?", List.of(paramAsConstant(null, 1)), "Not enough actual parameters 1"); + } } public void testNamedParams() { @@ -1238,15 +1260,23 @@ public void testInvalidNamedParams() { expectError("from test | where x < ?#1", List.of(paramAsConstant("#1", 5)), "token recognition error at: '#'"); - expectError( - "from test | where x < ??", - List.of(paramAsConstant("n_1", 5), paramAsConstant("n_2", 5)), - "extraneous input '?' expecting " - ); - expectError("from test | where x < ?Å", List.of(paramAsConstant("Å", 5)), "line 1:24: token recognition error at: 'Å'"); expectError("from test | eval x = ?Å", List.of(paramAsConstant("Å", 5)), "line 1:23: token recognition error at: 'Å'"); + + if (EsqlCapabilities.Cap.DOUBLE_PARAMETER_MARKERS_FOR_IDENTIFIERS.isEnabled()) { + expectError( + "from test | where x < ???", + List.of(paramAsConstant("n_1", 5), paramAsConstant("n_2", 5)), + "extraneous input '?' expecting " + ); + } else { + expectError( + "from test | where x < ??", + List.of(paramAsConstant("n_1", 5), paramAsConstant("n_2", 5)), + "extraneous input '?' expecting " + ); + } } public void testPositionalParams() { @@ -1548,33 +1578,39 @@ public void testParamInAggFunction() { } public void testParamMixed() { - expectError( - "from test | where x < ? | eval y = ?n2 + ?n3 | limit ?n4", - List.of(paramAsConstant("n1", 5), paramAsConstant("n2", -1), paramAsConstant("n3", 100), paramAsConstant("n4", 10)), - "Inconsistent parameter declaration, " - + "use one of positional, named or anonymous params but not a combination of named and anonymous" - ); - - expectError( - "from test | where x < ? | eval y = ?_n2 + ?n3 | limit ?_4", - List.of(paramAsConstant("n1", 5), paramAsConstant("_n2", -1), paramAsConstant("n3", 100), paramAsConstant("n4", 10)), - "Inconsistent parameter declaration, " - + "use one of positional, named or anonymous params but not a combination of named and anonymous" - ); - - expectError( - "from test | where x < ?1 | eval y = ?n2 + ?_n3 | limit ?n4", - List.of(paramAsConstant("n1", 5), paramAsConstant("n2", -1), paramAsConstant("_n3", 100), paramAsConstant("n4", 10)), - "Inconsistent parameter declaration, " - + "use one of positional, named or anonymous params but not a combination of named and positional" + Map, String> mixedParams = new HashMap<>( + Map.ofEntries( + Map.entry(List.of("?", "?n2", "?n3"), "named and anonymous"), + Map.entry(List.of("?", "?_n2", "?n3"), "named and anonymous"), + Map.entry(List.of("?1", "?n2", "?_n3"), "named and positional"), + Map.entry(List.of("?", "?2", "?n3"), "positional and anonymous") + ) ); - expectError( - "from test | where x < ? | eval y = ?2 + ?n3 | limit ?_n4", - List.of(paramAsConstant("n1", 5), paramAsConstant("n2", -1), paramAsConstant("n3", 100), paramAsConstant("_n4", 10)), - "Inconsistent parameter declaration, " - + "use one of positional, named or anonymous params but not a combination of positional and anonymous" - ); + if (EsqlCapabilities.Cap.DOUBLE_PARAMETER_MARKERS_FOR_IDENTIFIERS.isEnabled()) { + mixedParams.put(List.of("??", "??n2", "??n3"), "named and anonymous"); + mixedParams.put(List.of("?", "??_n2", "?n3"), "named and anonymous"); + mixedParams.put(List.of("??1", "?n2", "?_n3"), "named and positional"); + mixedParams.put(List.of("?", "??2", "?n3"), "positional and anonymous"); + } + for (Map.Entry, String> mixedParam : mixedParams.entrySet()) { + List params = mixedParam.getKey(); + String errorMessage = mixedParam.getValue(); + String query = LoggerMessageFormat.format( + null, + "from test | where x < {} | eval y = {}() + {}", + params.get(0), + params.get(1), + params.get(2) + ); + expectError( + query, + List.of(paramAsConstant("n1", "f1"), paramAsConstant("n2", "fn2"), paramAsConstant("n3", "f3")), + "Inconsistent parameter declaration, " + + "use one of positional, named or anonymous params but not a combination of " + + errorMessage + ); + } } public void testIntervalParam() { @@ -1594,6 +1630,7 @@ public void testIntervalParam() { } public void testParamForIdentifier() { + // TODO will be replaced by testDoubleParamsForIdentifier after providing an identifier with a single parameter marker is deprecated // field names can appear in eval/where/stats/sort/keep/drop/rename/dissect/grok/enrich/mvexpand // eval, where assertEquals( @@ -2015,6 +2052,9 @@ public void testMissingParam() { missingParamGroupB.contains(missingParam) ? errorMvExpandFunctionNameCommandOption : error ); } + if (EsqlCapabilities.Cap.DOUBLE_PARAMETER_MARKERS_FOR_IDENTIFIERS.isEnabled()) { + expectError("from test | " + missingParam.replace("?", "??"), List.of(paramAsConstant("f4", "f1*")), error); + } } } @@ -2043,8 +2083,7 @@ public void testQuotedName() { private void assertStringAsIndexPattern(String string, String statement) { if (Build.current().isSnapshot() == false && statement.contains("METRIC")) { - var e = expectThrows(ParsingException.class, () -> statement(statement)); - assertThat(e.getMessage(), containsString("mismatched input 'METRICS' expecting {")); + expectThrows(ParsingException.class, containsString("mismatched input 'METRICS' expecting {"), () -> statement(statement)); return; } LogicalPlan from = statement(statement); @@ -2055,8 +2094,11 @@ private void assertStringAsIndexPattern(String string, String statement) { private void assertStringAsLookupIndexPattern(String string, String statement) { if (Build.current().isSnapshot() == false) { - var e = expectThrows(ParsingException.class, () -> statement(statement)); - assertThat(e.getMessage(), containsString("line 1:14: LOOKUP_🐔 is in preview and only available in SNAPSHOT build")); + expectThrows( + ParsingException.class, + containsString("line 1:14: LOOKUP_🐔 is in preview and only available in SNAPSHOT build"), + () -> statement(statement) + ); return; } var plan = statement(statement); @@ -2123,8 +2165,11 @@ public void testInlineConvertWithNonexistentType() { public void testLookup() { String query = "ROW a = 1 | LOOKUP_🐔 t ON j"; if (Build.current().isSnapshot() == false) { - var e = expectThrows(ParsingException.class, () -> statement(query)); - assertThat(e.getMessage(), containsString("line 1:13: mismatched input 'LOOKUP_🐔' expecting {")); + expectThrows( + ParsingException.class, + containsString("line 1:13: mismatched input 'LOOKUP_🐔' expecting {"), + () -> statement(query) + ); return; } var plan = statement(query); @@ -2977,4 +3022,926 @@ public void testInvalidInsistAsterisk() { expectError("FROM text | EVAL x = 4 | INSIST_🐔 *", "INSIST doesn't support wildcards, found [*]"); expectError("FROM text | EVAL x = 4 | INSIST_🐔 foo*", "INSIST doesn't support wildcards, found [foo*]"); } + + public void testValidFork() { + assumeTrue("FORK requires corresponding capability", EsqlCapabilities.Cap.FORK.isEnabled()); + + var plan = statement(""" + FROM foo* + | FORK ( WHERE a:"baz" | LIMIT 11 ) + ( WHERE b:"bar" | SORT b ) + ( WHERE c:"bat" ) + ( SORT c ) + ( LIMIT 5 ) + """); + var fork = as(plan, Fork.class); + var subPlans = fork.subPlans(); + + // first subplan + var eval = as(subPlans.get(0), Eval.class); + assertThat(as(eval.fields().get(0), Alias.class), equalTo(alias("_fork", literalString("fork1")))); + var limit = as(eval.child(), Limit.class); + assertThat(limit.limit(), instanceOf(Literal.class)); + assertThat(((Literal) limit.limit()).value(), equalTo(11)); + var filter = as(limit.child(), Filter.class); + var match = (MatchOperator) filter.condition(); + var matchField = (UnresolvedAttribute) match.field(); + assertThat(matchField.name(), equalTo("a")); + assertThat(match.query().fold(FoldContext.small()), equalTo("baz")); + + // second subplan + eval = as(subPlans.get(1), Eval.class); + assertThat(as(eval.fields().get(0), Alias.class), equalTo(alias("_fork", literalString("fork2")))); + var orderBy = as(eval.child(), OrderBy.class); + assertThat(orderBy.order().size(), equalTo(1)); + Order order = orderBy.order().get(0); + assertThat(order.child(), instanceOf(UnresolvedAttribute.class)); + assertThat(((UnresolvedAttribute) order.child()).name(), equalTo("b")); + filter = as(orderBy.child(), Filter.class); + match = (MatchOperator) filter.condition(); + matchField = (UnresolvedAttribute) match.field(); + assertThat(matchField.name(), equalTo("b")); + assertThat(match.query().fold(FoldContext.small()), equalTo("bar")); + + // third subplan + eval = as(subPlans.get(2), Eval.class); + assertThat(as(eval.fields().get(0), Alias.class), equalTo(alias("_fork", literalString("fork3")))); + filter = as(eval.child(), Filter.class); + match = (MatchOperator) filter.condition(); + matchField = (UnresolvedAttribute) match.field(); + assertThat(matchField.name(), equalTo("c")); + assertThat(match.query().fold(FoldContext.small()), equalTo("bat")); + + // fourth subplan + eval = as(subPlans.get(3), Eval.class); + assertThat(as(eval.fields().get(0), Alias.class), equalTo(alias("_fork", literalString("fork4")))); + orderBy = as(eval.child(), OrderBy.class); + assertThat(orderBy.order().size(), equalTo(1)); + order = orderBy.order().get(0); + assertThat(order.child(), instanceOf(UnresolvedAttribute.class)); + assertThat(((UnresolvedAttribute) order.child()).name(), equalTo("c")); + + // fifth subplan + eval = as(subPlans.get(4), Eval.class); + assertThat(as(eval.fields().get(0), Alias.class), equalTo(alias("_fork", literalString("fork5")))); + limit = as(eval.child(), Limit.class); + assertThat(limit.limit(), instanceOf(Literal.class)); + assertThat(((Literal) limit.limit()).value(), equalTo(5)); + } + + public void testInvalidFork() { + assumeTrue("FORK requires corresponding capability", EsqlCapabilities.Cap.FORK.isEnabled()); + + expectError("FROM foo* | FORK (WHERE a:\"baz\")", "line 1:13: Fork requires at least two branches"); + expectError("FROM foo* | FORK (LIMIT 10)", "line 1:13: Fork requires at least two branches"); + expectError("FROM foo* | FORK (SORT a)", "line 1:13: Fork requires at least two branches"); + expectError("FROM foo* | FORK (WHERE x>1 | LIMIT 5)", "line 1:13: Fork requires at least two branches"); + expectError("FROM foo* | WHERE x>1 | FORK (WHERE a:\"baz\")", "Fork requires at least two branches"); + + expectError("FROM foo* | FORK (LIMIT 10) (EVAL x = 1)", "line 1:30: mismatched input 'EVAL' expecting {'limit', 'sort', 'where'}"); + expectError("FROM foo* | FORK (EVAL x = 1) (LIMIT 10)", "line 1:19: mismatched input 'EVAL' expecting {'limit', 'sort', 'where'}"); + expectError( + "FROM foo* | FORK (WHERE x>1 |EVAL x = 1) (WHERE x>1)", + "line 1:30: mismatched input 'EVAL' expecting {'limit', 'sort', 'where'}" + ); + expectError( + "FROM foo* | FORK (WHERE x>1 |EVAL x = 1) (WHERE x>1)", + "line 1:30: mismatched input 'EVAL' expecting {'limit', 'sort', 'where'}" + ); + expectError( + "FROM foo* | FORK (WHERE x>1 |STATS count(x) by y) (WHERE x>1)", + "line 1:30: mismatched input 'STATS' expecting {'limit', 'sort', 'where'}" + ); + expectError( + "FROM foo* | FORK ( FORK (WHERE x>1) (WHERE y>1)) (WHERE z>1)", + "line 1:20: mismatched input 'FORK' expecting {'limit', 'sort', 'where'}" + ); + expectError("FROM foo* | FORK ( x+1 ) ( WHERE y>2 )", "line 1:20: mismatched input 'x+1' expecting {'limit', 'sort', 'where'}"); + expectError("FROM foo* | FORK ( LIMIT 10 ) ( y+2 )", "line 1:33: mismatched input 'y+2' expecting {'limit', 'sort', 'where'}"); + } + + public void testFieldNamesAsCommands() throws Exception { + String[] keywords = new String[] { + "dissect", + "drop", + "enrich", + "eval", + "explain", + "from", + "grok", + "keep", + "limit", + "mv_expand", + "rename", + "sort", + "stats" }; + for (String keyword : keywords) { + var plan = statement("FROM test | STATS avg(" + keyword + ")"); + var aggregate = as(plan, Aggregate.class); + } + } + + // [ and ( are used to trigger a double mode causing their symbol name (instead of text) to be used in error reporting + // this test checks that their are properly replaced in the error message + public void testPreserveParanthesis() { + // test for ( + expectError("row a = 1 not in", "line 1:17: mismatched input '' expecting '('"); + expectError("row a = 1 | where a not in", "line 1:27: mismatched input '' expecting '('"); + expectError("row a = 1 | where a not in (1", "line 1:30: mismatched input '' expecting {',', ')'}"); + expectError("row a = 1 | where a not in [1", "line 1:28: missing '(' at '['"); + expectError("row a = 1 | where a not in 123", "line 1:28: missing '(' at '123'"); + // test for [ + expectError("explain", "line 1:8: mismatched input '' expecting '['"); + expectError("explain ]", "line 1:9: token recognition error at: ']'"); + expectError("explain [row x = 1", "line 1:19: missing ']' at ''"); + } + + static Alias alias(String name, Expression value) { + return new Alias(EMPTY, name, value); + } + + public void testValidRrf() { + assumeTrue("RRF requires corresponding capability", EsqlCapabilities.Cap.RRF.isEnabled()); + + LogicalPlan plan = statement(""" + FROM foo* METADATA _id, _index, _score + | FORK ( WHERE a:"baz" ) + ( WHERE b:"bar" ) + | RRF + """); + + var orderBy = as(plan, OrderBy.class); + assertThat(orderBy.order().size(), equalTo(3)); + + assertThat(orderBy.order().get(0).child(), instanceOf(UnresolvedAttribute.class)); + assertThat(((UnresolvedAttribute) orderBy.order().get(0).child()).name(), equalTo("_score")); + assertThat(orderBy.order().get(1).child(), instanceOf(UnresolvedAttribute.class)); + assertThat(((UnresolvedAttribute) orderBy.order().get(1).child()).name(), equalTo("_id")); + assertThat(orderBy.order().get(2).child(), instanceOf(UnresolvedAttribute.class)); + assertThat(((UnresolvedAttribute) orderBy.order().get(2).child()).name(), equalTo("_index")); + + var dedup = as(orderBy.child(), Dedup.class); + assertThat(dedup.groupings().size(), equalTo(2)); + assertThat(dedup.groupings().get(0), instanceOf(UnresolvedAttribute.class)); + assertThat(dedup.groupings().get(0).name(), equalTo("_id")); + assertThat(dedup.groupings().get(1), instanceOf(UnresolvedAttribute.class)); + assertThat(dedup.groupings().get(1).name(), equalTo("_index")); + assertThat(dedup.aggregates().size(), equalTo(1)); + assertThat(dedup.aggregates().get(0), instanceOf(Alias.class)); + + var rrfScoreEval = as(dedup.child(), RrfScoreEval.class); + assertThat(rrfScoreEval.scoreAttribute(), instanceOf(UnresolvedAttribute.class)); + assertThat(rrfScoreEval.scoreAttribute().name(), equalTo("_score")); + assertThat(rrfScoreEval.forkAttribute(), instanceOf(UnresolvedAttribute.class)); + assertThat(rrfScoreEval.forkAttribute().name(), equalTo("_fork")); + + assertThat(rrfScoreEval.child(), instanceOf(Fork.class)); + } + + public void testDoubleParamsForIdentifier() { + assumeTrue( + "double parameters markers for identifiers requires snapshot build", + EsqlCapabilities.Cap.DOUBLE_PARAMETER_MARKERS_FOR_IDENTIFIERS.isEnabled() + ); + // There are three variations of double parameters - named, positional or anonymous, e.g. ??n, ??1 or ??, covered. + // Each query is executed three times with the three variations. + + // field names can appear in eval/where/stats/sort/keep/drop/rename/dissect/grok/enrich/mvexpand + // eval, where + List> doubleParams = new ArrayList<>(3); + List namedDoubleParams = List.of("??f0", "??fn1", "??f1", "??f2", "??f3"); + List positionalDoubleParams = List.of("??1", "??2", "??3", "??4", "??5"); + List anonymousDoubleParams = List.of("??", "??", "??", "??", "??"); + doubleParams.add(namedDoubleParams); + doubleParams.add(positionalDoubleParams); + doubleParams.add(anonymousDoubleParams); + for (List params : doubleParams) { + String query = LoggerMessageFormat.format(null, """ + from test + | eval {} = {}({}) + | where {} == {} + | limit 1""", params.get(0), params.get(1), params.get(2), params.get(3), params.get(4)); + assertEquals( + new Limit( + EMPTY, + new Literal(EMPTY, 1, INTEGER), + new Filter( + EMPTY, + new Eval(EMPTY, relation("test"), List.of(new Alias(EMPTY, "x", function("toString", List.of(attribute("f1.")))))), + new Equals(EMPTY, attribute("f.2"), attribute("f3")) + ) + ), + statement( + query, + new QueryParams( + List.of( + paramAsConstant("f0", "x"), + paramAsConstant("fn1", "toString"), + paramAsConstant("f1", "f1."), + paramAsConstant("f2", "f.2"), + paramAsConstant("f3", "f3") + ) + ) + ) + ); + } + + namedDoubleParams = List.of("??f0", "??fn1", "??f1", "??f2", "??f3", "??f4", "??f5", "??f6"); + positionalDoubleParams = List.of("??1", "??2", "??3", "??4", "??5", "??6", "??7", "??8"); + anonymousDoubleParams = List.of("??", "??", "??", "??", "??", "??", "??", "??"); + doubleParams.clear(); + doubleParams.add(namedDoubleParams); + doubleParams.add(positionalDoubleParams); + doubleParams.add(anonymousDoubleParams); + for (List params : doubleParams) { + String query = LoggerMessageFormat.format( + null, + """ + from test + | eval {} = {}({}.{}) + | where {}.{} == {}.{} + | limit 1""", + params.get(0), + params.get(1), + params.get(2), + params.get(3), + params.get(4), + params.get(5), + params.get(6), + params.get(7) + ); + assertEquals( + new Limit( + EMPTY, + new Literal(EMPTY, 1, INTEGER), + new Filter( + EMPTY, + new Eval( + EMPTY, + relation("test"), + List.of(new Alias(EMPTY, "x", function("toString", List.of(attribute("f1..f.2"))))) + ), + new Equals(EMPTY, attribute("f3.*.f.4."), attribute("f.5.*.f.*.6")) + ) + ), + statement( + query, + new QueryParams( + List.of( + paramAsConstant("f0", "x"), + paramAsConstant("fn1", "toString"), + paramAsConstant("f1", "f1."), + paramAsConstant("f2", "f.2"), + paramAsConstant("f3", "f3.*"), + paramAsConstant("f4", "f.4."), + paramAsConstant("f5", "f.5.*"), + paramAsConstant("f6", "f.*.6") + ) + ) + ) + ); + } + + // stats, sort, mv_expand + namedDoubleParams = List.of("??fn2", "??f3", "??f4", "??f5", "??f6"); + positionalDoubleParams = List.of("??1", "??2", "??3", "??4", "??5"); + anonymousDoubleParams = List.of("??", "??", "??", "??", "??"); + doubleParams.clear(); + doubleParams.add(namedDoubleParams); + doubleParams.add(positionalDoubleParams); + doubleParams.add(anonymousDoubleParams); + for (List params : doubleParams) { + String query = LoggerMessageFormat.format(null, """ + from test + | stats y = {}({}) by {} + | sort {} + | mv_expand {}""", params.get(0), params.get(1), params.get(2), params.get(3), params.get(4)); + assertEquals( + new MvExpand( + EMPTY, + new OrderBy( + EMPTY, + new Aggregate( + EMPTY, + relation("test"), + Aggregate.AggregateType.STANDARD, + List.of(attribute("f.4.")), + List.of(new Alias(EMPTY, "y", function("count", List.of(attribute("f3.*")))), attribute("f.4.")) + ), + List.of(new Order(EMPTY, attribute("f.5.*"), Order.OrderDirection.ASC, Order.NullsPosition.LAST)) + ), + attribute("f.6*"), + attribute("f.6*") + ), + statement( + query, + new QueryParams( + List.of( + paramAsConstant("fn2", "count"), + paramAsConstant("f3", "f3.*"), + paramAsConstant("f4", "f.4."), + paramAsConstant("f5", "f.5.*"), + paramAsConstant("f6", "f.6*") + ) + ) + ) + ); + } + + namedDoubleParams = List.of("??fn2", "??f7", "??f8", "??f9", "??f10", "??f11", "??f12", "??f13", "??f14"); + positionalDoubleParams = List.of("??1", "??2", "??3", "??4", "??5", "??6", "??7", "??8", "??9"); + anonymousDoubleParams = List.of("??", "??", "??", "??", "??", "??", "??", "??", "??"); + doubleParams.clear(); + doubleParams.add(namedDoubleParams); + doubleParams.add(positionalDoubleParams); + doubleParams.add(anonymousDoubleParams); + for (List params : doubleParams) { + String query = LoggerMessageFormat.format( + null, + """ + from test + | stats y = {}({}.{}) by {}.{} + | sort {}.{} + | mv_expand {}.{}""", + params.get(0), + params.get(1), + params.get(2), + params.get(3), + params.get(4), + params.get(5), + params.get(6), + params.get(7), + params.get(8) + ); + assertEquals( + new MvExpand( + EMPTY, + new OrderBy( + EMPTY, + new Aggregate( + EMPTY, + relation("test"), + Aggregate.AggregateType.STANDARD, + List.of(attribute("f.9.f10.*")), + List.of(new Alias(EMPTY, "y", function("count", List.of(attribute("f.7*.f8.")))), attribute("f.9.f10.*")) + ), + List.of(new Order(EMPTY, attribute("f.11..f.12.*"), Order.OrderDirection.ASC, Order.NullsPosition.LAST)) + ), + attribute("f.*.13.f.14*"), + attribute("f.*.13.f.14*") + ), + statement( + query, + new QueryParams( + List.of( + paramAsConstant("fn2", "count"), + paramAsConstant("f7", "f.7*"), + paramAsConstant("f8", "f8."), + paramAsConstant("f9", "f.9"), + paramAsConstant("f10", "f10.*"), + paramAsConstant("f11", "f.11."), + paramAsConstant("f12", "f.12.*"), + paramAsConstant("f13", "f.*.13"), + paramAsConstant("f14", "f.14*") + ) + ) + ) + ); + } + + // keep, drop, rename, grok, dissect, lookup join + namedDoubleParams = List.of("??f1", "??f2", "??f3", "??f4", "??f5", "??f6", "??f7", "??f8", "??f9"); + positionalDoubleParams = List.of("??1", "??2", "??3", "??4", "??5", "??6", "??7", "??8", "??9"); + anonymousDoubleParams = List.of("??", "??", "??", "??", "??", "??", "??", "??", "??"); + doubleParams.clear(); + doubleParams.add(namedDoubleParams); + doubleParams.add(positionalDoubleParams); + doubleParams.add(anonymousDoubleParams); + for (List params : doubleParams) { + String query = LoggerMessageFormat.format( + null, + """ + from test + | keep {}, {} + | drop {}, {} + | dissect {} "%{bar}" + | grok {} "%{WORD:foo}" + | rename {} as {} + | lookup join idx on {} + | limit 1""", + params.get(0), + params.get(1), + params.get(2), + params.get(3), + params.get(4), + params.get(5), + params.get(6), + params.get(7), + params.get(8) + ); + LogicalPlan plan = statement( + query, + new QueryParams( + List.of( + paramAsConstant("f1", "f.1.*"), + paramAsConstant("f2", "f.2"), + paramAsConstant("f3", "f3."), + paramAsConstant("f4", "f4.*"), + paramAsConstant("f5", "f.5*"), + paramAsConstant("f6", "f.6."), + paramAsConstant("f7", "f7*."), + paramAsConstant("f8", "f.8"), + paramAsConstant("f9", "f9") + ) + ) + ); + Limit limit = as(plan, Limit.class); + LookupJoin join = as(limit.child(), LookupJoin.class); + UnresolvedRelation ur = as(join.right(), UnresolvedRelation.class); + assertEquals(ur.indexPattern().indexPattern(), "idx"); + JoinTypes.UsingJoinType joinType = as(join.config().type(), JoinTypes.UsingJoinType.class); + assertEquals(joinType.coreJoin().joinName(), "LEFT OUTER"); + assertEquals(joinType.columns(), List.of(attribute("f9"))); + Rename rename = as(join.left(), Rename.class); + assertEquals(rename.renamings(), List.of(new Alias(EMPTY, "f.8", attribute("f7*.")))); + Grok grok = as(rename.child(), Grok.class); + assertEquals(grok.input(), attribute("f.6.")); + assertEquals("%{WORD:foo}", grok.parser().pattern()); + assertEquals(List.of(referenceAttribute("foo", KEYWORD)), grok.extractedFields()); + Dissect dissect = as(grok.child(), Dissect.class); + assertEquals(dissect.input(), attribute("f.5*")); + assertEquals("%{bar}", dissect.parser().pattern()); + assertEquals("", dissect.parser().appendSeparator()); + assertEquals(List.of(referenceAttribute("bar", KEYWORD)), dissect.extractedFields()); + Drop drop = as(dissect.child(), Drop.class); + List removals = drop.removals(); + assertEquals(removals, List.of(attribute("f3."), attribute("f4.*"))); + Keep keep = as(drop.child(), Keep.class); + assertEquals(keep.projections(), List.of(attribute("f.1.*"), attribute("f.2"))); + } + + namedDoubleParams = List.of( + "??f1", + "??f2", + "??f3", + "??f4", + "??f5", + "??f6", + "??f7", + "??f8", + "??f9", + "??f10", + "??f11", + "??f12", + "??f13", + "??f14" + ); + positionalDoubleParams = List.of( + "??1", + "??2", + "??3", + "??4", + "??5", + "??6", + "??7", + "??8", + "??9", + "??10", + "??11", + "??12", + "??13", + "??14" + ); + anonymousDoubleParams = List.of("??", "??", "??", "??", "??", "??", "??", "??", "??", "??", "??", "??", "??", "??"); + doubleParams.clear(); + doubleParams.add(namedDoubleParams); + doubleParams.add(positionalDoubleParams); + doubleParams.add(anonymousDoubleParams); + for (List params : doubleParams) { + String query = LoggerMessageFormat.format( + null, + """ + from test + | keep {}.{} + | drop {}.{} + | dissect {}.{} "%{bar}" + | grok {}.{} "%{WORD:foo}" + | rename {}.{} as {}.{} + | lookup join idx on {}.{} + | limit 1""", + params.get(0), + params.get(1), + params.get(2), + params.get(3), + params.get(4), + params.get(5), + params.get(6), + params.get(7), + params.get(8), + params.get(9), + params.get(10), + params.get(11), + params.get(12), + params.get(13) + ); + LogicalPlan plan = statement( + query, + new QueryParams( + List.of( + paramAsConstant("f1", "f.1.*"), + paramAsConstant("f2", "f.2"), + paramAsConstant("f3", "f3."), + paramAsConstant("f4", "f4.*"), + paramAsConstant("f5", "f.5*"), + paramAsConstant("f6", "f.6."), + paramAsConstant("f7", "f7*."), + paramAsConstant("f8", "f.8"), + paramAsConstant("f9", "f.9*"), + paramAsConstant("f10", "f.10."), + paramAsConstant("f11", "f11*."), + paramAsConstant("f12", "f.12"), + paramAsConstant("f13", "f13"), + paramAsConstant("f14", "f14") + ) + ) + ); + Limit limit = as(plan, Limit.class); + LookupJoin join = as(limit.child(), LookupJoin.class); + UnresolvedRelation ur = as(join.right(), UnresolvedRelation.class); + assertEquals(ur.indexPattern().indexPattern(), "idx"); + JoinTypes.UsingJoinType joinType = as(join.config().type(), JoinTypes.UsingJoinType.class); + assertEquals(joinType.coreJoin().joinName(), "LEFT OUTER"); + assertEquals(joinType.columns(), List.of(attribute("f13.f14"))); + Rename rename = as(join.left(), Rename.class); + assertEquals(rename.renamings(), List.of(new Alias(EMPTY, "f11*..f.12", attribute("f.9*.f.10.")))); + Grok grok = as(rename.child(), Grok.class); + assertEquals(grok.input(), attribute("f7*..f.8")); + assertEquals("%{WORD:foo}", grok.parser().pattern()); + assertEquals(List.of(referenceAttribute("foo", KEYWORD)), grok.extractedFields()); + Dissect dissect = as(grok.child(), Dissect.class); + assertEquals(dissect.input(), attribute("f.5*.f.6.")); + assertEquals("%{bar}", dissect.parser().pattern()); + assertEquals("", dissect.parser().appendSeparator()); + assertEquals(List.of(referenceAttribute("bar", KEYWORD)), dissect.extractedFields()); + Drop drop = as(dissect.child(), Drop.class); + List removals = drop.removals(); + assertEquals(removals, List.of(attribute("f3..f4.*"))); + Keep keep = as(drop.child(), Keep.class); + assertEquals(keep.projections(), List.of(attribute("f.1.*.f.2"))); + } + + // enrich, lookup join + namedDoubleParams = List.of("??f1", "??f2", "??f3"); + positionalDoubleParams = List.of("??1", "??2", "??3"); + anonymousDoubleParams = List.of("??", "??", "??"); + doubleParams.clear(); + doubleParams.add(namedDoubleParams); + doubleParams.add(positionalDoubleParams); + doubleParams.add(anonymousDoubleParams); + for (List params : doubleParams) { + String query = LoggerMessageFormat.format( + null, + "from idx1 | ENRICH idx2 ON {} WITH {} = {}", + params.get(0), + params.get(1), + params.get(2) + ); + assertEquals( + new Enrich( + EMPTY, + relation("idx1"), + null, + new Literal(EMPTY, "idx2", KEYWORD), + attribute("f.1.*"), + null, + Map.of(), + List.of(new Alias(EMPTY, "f.2", attribute("f.3*"))) + ), + statement( + query, + new QueryParams(List.of(paramAsConstant("f1", "f.1.*"), paramAsConstant("f2", "f.2"), paramAsConstant("f3", "f.3*"))) + ) + ); + } + + namedDoubleParams = List.of("??f1", "??f2", "??f3", "??f4", "??f5", "??f6"); + positionalDoubleParams = List.of("??1", "??2", "??3", "??4", "??5", "??6"); + anonymousDoubleParams = List.of("??", "??", "??", "??", "??", "??"); + doubleParams.clear(); + doubleParams.add(namedDoubleParams); + doubleParams.add(positionalDoubleParams); + doubleParams.add(anonymousDoubleParams); + for (List params : doubleParams) { + String query = LoggerMessageFormat.format( + null, + "from idx1 | ENRICH idx2 ON {}.{} WITH {}.{} = {}.{}", + params.get(0), + params.get(1), + params.get(2), + params.get(3), + params.get(4), + params.get(5) + ); + assertEquals( + new Enrich( + EMPTY, + relation("idx1"), + null, + new Literal(EMPTY, "idx2", KEYWORD), + attribute("f.1.*.f.2"), + null, + Map.of(), + List.of(new Alias(EMPTY, "f.3*.f.4.*", attribute("f.5.f.6*"))) + ), + statement( + query, + new QueryParams( + List.of( + paramAsConstant("f1", "f.1.*"), + paramAsConstant("f2", "f.2"), + paramAsConstant("f3", "f.3*"), + paramAsConstant("f4", "f.4.*"), + paramAsConstant("f5", "f.5"), + paramAsConstant("f6", "f.6*") + ) + ) + ) + ); + } + } + + public void testMixedSingleDoubleParams() { + assumeTrue( + "double parameters markers for identifiers requires snapshot build", + EsqlCapabilities.Cap.DOUBLE_PARAMETER_MARKERS_FOR_IDENTIFIERS.isEnabled() + ); + // This is a subset of testDoubleParamsForIdentifier, with single and double parameter markers mixed in the queries + // Single parameter markers represent a constant value or pattern + // double parameter markers represent identifiers - field or function names + + // mixed constant and identifier, eval/where + List> doubleParams = new ArrayList<>(3); + List namedDoubleParams = List.of("??f0", "??fn1", "?v1", "??f2", "?v3"); + List positionalDoubleParams = List.of("??1", "??2", "?3", "??4", "?5"); + List anonymousDoubleParams = List.of("??", "??", "?", "??", "?"); + doubleParams.add(namedDoubleParams); + doubleParams.add(positionalDoubleParams); + doubleParams.add(anonymousDoubleParams); + for (List params : doubleParams) { + String query = LoggerMessageFormat.format(null, """ + from test + | eval {} = {}({}) + | where {} == {} + | limit 1""", params.get(0), params.get(1), params.get(2), params.get(3), params.get(4)); + assertEquals( + new Limit( + EMPTY, + new Literal(EMPTY, 1, INTEGER), + new Filter( + EMPTY, + new Eval( + EMPTY, + relation("test"), + List.of(new Alias(EMPTY, "x", function("toString", List.of(new Literal(EMPTY, "constant_value", KEYWORD))))) + ), + new Equals(EMPTY, attribute("f.2"), new Literal(EMPTY, 100, INTEGER)) + ) + ), + statement( + query, + new QueryParams( + List.of( + paramAsConstant("f0", "x"), + paramAsConstant("fn1", "toString"), + paramAsConstant("v1", "constant_value"), + paramAsConstant("f2", "f.2"), + paramAsConstant("v3", 100) + ) + ) + ) + ); + } + + // mixed constant and identifier, stats/sort/mv_expand + namedDoubleParams = List.of("??fn2", "?v3", "??f4", "??f5", "??f6"); + positionalDoubleParams = List.of("??1", "?2", "??3", "??4", "??5"); + anonymousDoubleParams = List.of("??", "?", "??", "??", "??"); + doubleParams.clear(); + doubleParams.add(namedDoubleParams); + doubleParams.add(positionalDoubleParams); + doubleParams.add(anonymousDoubleParams); + for (List params : doubleParams) { + String query = LoggerMessageFormat.format(null, """ + from test + | stats y = {}({}) by {} + | sort {} + | mv_expand {}""", params.get(0), params.get(1), params.get(2), params.get(3), params.get(4)); + assertEquals( + new MvExpand( + EMPTY, + new OrderBy( + EMPTY, + new Aggregate( + EMPTY, + relation("test"), + Aggregate.AggregateType.STANDARD, + List.of(attribute("f.4.")), + List.of(new Alias(EMPTY, "y", function("count", List.of(new Literal(EMPTY, "*", KEYWORD)))), attribute("f.4.")) + ), + List.of(new Order(EMPTY, attribute("f.5.*"), Order.OrderDirection.ASC, Order.NullsPosition.LAST)) + ), + attribute("f.6*"), + attribute("f.6*") + ), + statement( + query, + new QueryParams( + List.of( + paramAsConstant("fn2", "count"), + paramAsConstant("v3", "*"), + paramAsConstant("f4", "f.4."), + paramAsConstant("f5", "f.5.*"), + paramAsConstant("f6", "f.6*") + ) + ) + ) + ); + } + + // mixed field name and field name pattern + LogicalPlan plan = statement( + "from test | keep ??f1, ?f2 | drop ?f3, ??f4 | lookup join idx on ??f5", + new QueryParams( + List.of( + paramAsConstant("f1", "f*1."), + paramAsPattern("f2", "f.2*"), + paramAsPattern("f3", "f3.*"), + paramAsConstant("f4", "f.4.*"), + paramAsConstant("f5", "f5") + ) + ) + ); + + LookupJoin join = as(plan, LookupJoin.class); + UnresolvedRelation ur = as(join.right(), UnresolvedRelation.class); + assertEquals(ur.indexPattern().indexPattern(), "idx"); + JoinTypes.UsingJoinType joinType = as(join.config().type(), JoinTypes.UsingJoinType.class); + assertEquals(joinType.coreJoin().joinName(), "LEFT OUTER"); + assertEquals(joinType.columns(), List.of(attribute("f5"))); + Drop drop = as(join.left(), Drop.class); + List removals = drop.removals(); + assertEquals(removals.size(), 2); + UnresolvedNamePattern up = as(removals.get(0), UnresolvedNamePattern.class); + assertEquals(up.name(), "f3.*"); + assertEquals(up.pattern(), "f3.*"); + UnresolvedAttribute ua = as(removals.get(1), UnresolvedAttribute.class); + assertEquals(ua.name(), "f.4.*"); + Keep keep = as(drop.child(), Keep.class); + assertEquals(keep.projections().size(), 2); + ua = as(keep.projections().get(0), UnresolvedAttribute.class); + assertEquals(ua.name(), "f*1."); + up = as(keep.projections().get(1), UnresolvedNamePattern.class); + assertEquals(up.name(), "f.2*"); + assertEquals(up.pattern(), "f.2*"); + ur = as(keep.child(), UnresolvedRelation.class); + assertEquals(ur, relation("test")); + + // test random single and double params + // commands in group1 take both constants(?) and identifiers(??) + List commandWithRandomSingleOrDoubleParamsGroup1 = List.of( + "eval x = {}f1, y = {}f2, z = {}f3", + "eval x = fn({}f1), y = {}f2 + {}f3", + "where {}f1 == \"a\" and {}f2 > 1 and {}f3 in (1, 2)", + "stats x = fn({}f1) by {}f2, {}f3", + "sort {}f1, {}f2, {}f3", + "dissect {}f1 \"%{bar}\"", + "grok {}f1 \"%{WORD:foo}\"" + ); + for (String command : commandWithRandomSingleOrDoubleParamsGroup1) { + String param1 = randomBoolean() ? "?" : "??"; + String param2 = randomBoolean() ? "?" : "??"; + String param3 = randomBoolean() ? "?" : "??"; + plan = statement( + LoggerMessageFormat.format(null, "from test | " + command, param1, param2, param3), + new QueryParams(List.of(paramAsConstant("f1", "f1"), paramAsConstant("f2", "f2"), paramAsConstant("f3", "f3"))) + ); + assertNotNull(plan); + } + // commands in group2 only take identifiers(??) + List commandWithRandomSingleOrDoubleParamsGroup2 = List.of( + "eval x = {}f1(), y = {}f2(), z = {}f3()", + "where {}f1 : \"b\" and {}f2() > 0 and {}f3()", + "stats x = {}f1(), {}f2(), {}f3()", + "rename {}f1 as {}f2, {}f3 as x", + "enrich idx2 ON {}f1 WITH {}f2 = {}f3", + "keep {}f1, {}f2, {}f3", + "drop {}f1, {}f2, {}f3", + "mv_expand {}f1 | mv_expand {}f2 | mv_expand {}f3", + "lookup join idx1 on {}f1 | lookup join idx2 on {}f2 | lookup join idx3 on {}f3" + ); + + for (String command : commandWithRandomSingleOrDoubleParamsGroup2) { + String param1 = randomBoolean() ? "?" : "??"; + String param2 = randomBoolean() ? "?" : "??"; + String param3 = randomBoolean() ? "?" : "??"; + if (param1.equals("?") || param2.equals("?") || param3.equals("?")) { + expectError( + LoggerMessageFormat.format(null, "from test | " + command, param1, param2, param3), + List.of(paramAsConstant("f1", "f1"), paramAsConstant("f2", "f2"), paramAsConstant("f3", "f3")), + command.contains("join") + ? "JOIN ON clause only supports fields at the moment" + : "declared as a constant, cannot be used as an identifier" + ); + } + } + } + + public void testInvalidDoubleParamsNames() { + assumeTrue( + "double parameters markers for identifiers requires snapshot build", + EsqlCapabilities.Cap.DOUBLE_PARAMETER_MARKERS_FOR_IDENTIFIERS.isEnabled() + ); + expectError( + "from test | where x < ??n1 | eval y = ??n2", + List.of(paramAsConstant("n1", "f1"), paramAsConstant("n3", "f2")), + "line 1:39: Unknown query parameter [n2], did you mean any of [n3, n1]?" + ); + + expectError("from test | where x < ??@1", List.of(paramAsConstant("@1", "f1")), "line 1:25: extraneous input '@1' expecting "); + + expectError("from test | where x < ??#1", List.of(paramAsConstant("#1", "f1")), "line 1:25: token recognition error at: '#'"); + + expectError("from test | where x < ??Å", List.of(paramAsConstant("Å", "f1")), "line 1:25: token recognition error at: 'Å'"); + + expectError("from test | eval x = ??Å", List.of(paramAsConstant("Å", "f1")), "line 1:24: token recognition error at: 'Å'"); + } + + public void testInvalidDoubleParamsPositions() { + assumeTrue( + "double parameters markers for identifiers requires snapshot build", + EsqlCapabilities.Cap.DOUBLE_PARAMETER_MARKERS_FOR_IDENTIFIERS.isEnabled() + ); + expectError( + "from test | where x < ??0", + List.of(paramAsConstant(null, "f1")), + "line 1:23: No parameter is defined for position 0, did you mean position 1" + ); + + expectError( + "from test | where x < ??2", + List.of(paramAsConstant(null, "f1")), + "line 1:23: No parameter is defined for position 2, did you mean position 1" + ); + + expectError( + "from test | where x < ??0 and y < ??2", + List.of(paramAsConstant(null, "f1")), + "line 1:23: No parameter is defined for position 0, did you mean position 1?; " + + "line 1:35: No parameter is defined for position 2, did you mean position 1?" + ); + + expectError( + "from test | where x < ??0", + List.of(paramAsConstant(null, "f1"), paramAsConstant(null, "f2")), + "line 1:23: No parameter is defined for position 0, did you mean any position between 1 and 2?" + ); + } + + public void testInvalidDoubleParamsType() { + assumeTrue( + "double parameters markers for identifiers requires snapshot build", + EsqlCapabilities.Cap.DOUBLE_PARAMETER_MARKERS_FOR_IDENTIFIERS.isEnabled() + ); + // double parameter markers cannot be declared as identifier patterns + String error = "Query parameter [??f1][f1] declared as a pattern, cannot be used as an identifier"; + List commandWithDoubleParams = List.of( + "eval x = ??f1", + "eval x = ??f1(f1)", + "where ??f1 == \"a\"", + "stats x = count(??f1)", + "sort ??f1", + "rename ??f1 as ??f2", + "dissect ??f1 \"%{bar}\"", + "grok ??f1 \"%{WORD:foo}\"", + "enrich idx2 ON ??f1 WITH ??f2 = ??f3", + "keep ??f1", + "drop ??f1", + "mv_expand ??f1", + "lookup join idx on ??f1" + ); + for (String command : commandWithDoubleParams) { + expectError( + "from test | " + command, + List.of(paramAsPattern("f1", "f1*"), paramAsPattern("f2", "f2*"), paramAsPattern("f3", "f3*")), + error + ); + } + } + + public void testUnclosedParenthesis() { + String[] queries = { "row a = )", "row ]", "from source | eval x = [1,2,3]]" }; + for (String q : queries) { + expectError(q, "Invalid query"); + } + } }