From 1a8dadc7733075c7e2f685023960fdb5ceaf662e Mon Sep 17 00:00:00 2001 From: kanoshiou Date: Thu, 17 Apr 2025 22:09:26 +0800 Subject: [PATCH 1/6] Keep attribute --- .../java/org/elasticsearch/xpack/esql/session/EsqlSession.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java index be90e6fd3b3c0..a95e07fa70baf 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java @@ -626,7 +626,7 @@ static PreAnalysisResult fieldNames(LogicalPlan parsed, Set enrichPolicy p.forEachExpression(UnresolvedNamePattern.class, up -> { var ua = new UnresolvedAttribute(up.source(), up.name()); referencesBuilder.add(ua); - if (p instanceof Keep) { + if (p instanceof Keep || p instanceof Drop) { keepCommandRefsBuilder.add(ua); } }); From 424aff5375ec419e586a826f6ee4a1c7b1d44d86 Mon Sep 17 00:00:00 2001 From: kanoshiou Date: Thu, 17 Apr 2025 22:43:00 +0800 Subject: [PATCH 2/6] Add tests --- .../testFixtures/src/main/resources/drop.csv-spec | 15 +++++++++++++++ .../session/IndexResolverFieldNamesTests.java | 10 ++++++++++ 2 files changed, 25 insertions(+) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/drop.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/drop.csv-spec index 9886d6cce0ca2..fd9cab7f21e81 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/drop.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/drop.csv-spec @@ -172,3 +172,18 @@ Milky Way Milky Way Milky Way ; + +dropAgainWithWildcardAfterEval +from languages +| eval language_code = 12, x = 13 +| drop language_code +| drop language* +| keep x +| limit 3 +; + +x:integer +13 +13 +13 +; diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/IndexResolverFieldNamesTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/IndexResolverFieldNamesTests.java index ef54dec2a0b60..275bfcb011969 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/IndexResolverFieldNamesTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/IndexResolverFieldNamesTests.java @@ -1710,6 +1710,16 @@ public void testEnrichAndJoinMaskingEvalWh() { | keep emp_no, language_name""", Set.of("emp_no", "language_name", "languages", "language_name.*", "languages.*", "emp_no.*")); } + public void testDropAgainWithWildcardAfterEval() { + assertFieldNames(""" + from employees + | eval full_name = 12 + | drop full_name + | drop *name + | keep emp_no + """, Set.of("emp_no", "emp_no.*", "*name", "*name.*")); + } + private Set fieldNames(String query, Set enrichPolicyMatchFields) { var preAnalysisResult = new EsqlSession.PreAnalysisResult(null); return EsqlSession.fieldNames(parser.createStatement(query), enrichPolicyMatchFields, preAnalysisResult).fieldNames(); From 8eca69d81fda235441b3c489233aad5b1b6f9ca5 Mon Sep 17 00:00:00 2001 From: kanoshiou Date: Thu, 17 Apr 2025 23:03:06 +0800 Subject: [PATCH 3/6] Update docs/changelog/127009.yaml --- docs/changelog/127009.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 docs/changelog/127009.yaml diff --git a/docs/changelog/127009.yaml b/docs/changelog/127009.yaml new file mode 100644 index 0000000000000..adbec139cae28 --- /dev/null +++ b/docs/changelog/127009.yaml @@ -0,0 +1,6 @@ +pr: 127009 +summary: "ESQL: Keep `DROP` attributes when resolving field names" +area: ES|QL +type: bug +issues: + - 126418 From 093933c05b910eec71cd0893c8452f6a90dd218a Mon Sep 17 00:00:00 2001 From: kanoshiou Date: Fri, 18 Apr 2025 00:29:40 +0800 Subject: [PATCH 4/6] Remove 126026 from `ALLOWED_ERRORS` --- .../xpack/esql/qa/rest/generative/GenerativeRestTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java index 8e37c7aaadfbc..73c6437051752 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java @@ -54,7 +54,6 @@ public abstract class GenerativeRestTest extends ESRestTestCase { "token recognition error at: '``", // https://github.com/elastic/elasticsearch/issues/125870 "Unknown column \\[.*\\]", // https://github.com/elastic/elasticsearch/issues/126026 "optimized incorrectly due to missing references", // https://github.com/elastic/elasticsearch/issues/116781 - "No matches found for pattern", // https://github.com/elastic/elasticsearch/issues/126418 "The incoming YAML document exceeds the limit:" // still to investigate, but it seems to be specific to the test framework ); From 7e8c5caebfce76cb6b66ddb321611090f2cb4ceb Mon Sep 17 00:00:00 2001 From: kanoshiou Date: Fri, 18 Apr 2025 00:35:44 +0800 Subject: [PATCH 5/6] Fix bwc --- .../esql/qa/testFixtures/src/main/resources/drop.csv-spec | 1 + .../elasticsearch/xpack/esql/action/EsqlCapabilities.java | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/drop.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/drop.csv-spec index fd9cab7f21e81..51597abb062d4 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/drop.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/drop.csv-spec @@ -174,6 +174,7 @@ Milky Way ; dropAgainWithWildcardAfterEval +required_capability:drop_again_with_wildcard_after_eval from languages | eval language_code = 12, x = 13 | drop language_code diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index 7dd679e681798..ad0ef458a5686 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -1017,7 +1017,13 @@ public enum Cap { * During resolution (pre-analysis) we have to consider that joins or enriches can override EVALuated values * https://github.com/elastic/elasticsearch/issues/126419 */ - FIX_JOIN_MASKING_EVAL; + FIX_JOIN_MASKING_EVAL, + + /** + * Support for keeping `DROP` attributes when resolving field names. + * see ES|QL: no matches for pattern #126418 + */ + DROP_AGAIN_WITH_WILDCARD_AFTER_EVAL; private final boolean enabled; From 754daf31c9e18700820859c7a576fe5ccf0d5250 Mon Sep 17 00:00:00 2001 From: kanoshiou Date: Fri, 18 Apr 2025 09:53:00 +0800 Subject: [PATCH 6/6] Reformat code --- .../esql/qa/testFixtures/src/main/resources/drop.csv-spec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/drop.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/drop.csv-spec index 51597abb062d4..eec8e073e3eec 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/drop.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/drop.csv-spec @@ -174,7 +174,7 @@ Milky Way ; dropAgainWithWildcardAfterEval -required_capability:drop_again_with_wildcard_after_eval +required_capability: drop_again_with_wildcard_after_eval from languages | eval language_code = 12, x = 13 | drop language_code