diff --git a/build-tools/src/integTest/groovy/org/elasticsearch/gradle/LoggedExecFuncTest.groovy b/build-tools/src/integTest/groovy/org/elasticsearch/gradle/LoggedExecFuncTest.groovy index 302fb2bcc2257..5a92f61c70d8c 100644 --- a/build-tools/src/integTest/groovy/org/elasticsearch/gradle/LoggedExecFuncTest.groovy +++ b/build-tools/src/integTest/groovy/org/elasticsearch/gradle/LoggedExecFuncTest.groovy @@ -35,7 +35,7 @@ class LoggedExecFuncTest extends AbstractGradleFuncTest { import org.elasticsearch.gradle.LoggedExec tasks.register('loggedExec', LoggedExec) { commandLine 'ls', '-lh' - spoolOutput = $spooling + getSpoolOutput().set($spooling) } """ when: @@ -54,7 +54,7 @@ class LoggedExecFuncTest extends AbstractGradleFuncTest { import org.elasticsearch.gradle.LoggedExec tasks.register('loggedExec', LoggedExec) { commandLine 'ls', 'wtf' - spoolOutput = $spooling + getSpoolOutput().set($spooling) } """ when: @@ -97,7 +97,7 @@ class LoggedExecFuncTest extends AbstractGradleFuncTest { tasks.register('loggedExec', LoggedExec) { commandLine 'echo', 'HELLO' getCaptureOutput().set(true) - spoolOutput = true + getSpoolOutput().set(true) } """ when: diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/LoggedExec.java b/build-tools/src/main/java/org/elasticsearch/gradle/LoggedExec.java index 9740a0c2f5425..acb526cf9a3bb 100644 --- a/build-tools/src/main/java/org/elasticsearch/gradle/LoggedExec.java +++ b/build-tools/src/main/java/org/elasticsearch/gradle/LoggedExec.java @@ -56,7 +56,6 @@ public abstract class LoggedExec extends DefaultTask implements FileSystemOperat protected FileSystemOperations fileSystemOperations; private ProjectLayout projectLayout; private ExecOperations execOperations; - private boolean spoolOutput; @Input @Optional @@ -84,6 +83,9 @@ public abstract class LoggedExec extends DefaultTask implements FileSystemOperat @Input abstract public Property getWorkingDir(); + @Internal + abstract public Property getSpoolOutput(); + private String output; @Inject @@ -95,14 +97,16 @@ public LoggedExec(ProjectLayout projectLayout, ExecOperations execOperations, Fi // For now mimic default behaviour of Gradle Exec task here getEnvironment().putAll(System.getenv()); getCaptureOutput().convention(false); + getSpoolOutput().convention(false); } @TaskAction public void run() { + boolean spoolOutput = getSpoolOutput().get(); if (spoolOutput && getCaptureOutput().get()) { throw new GradleException("Capturing output is not supported when spoolOutput is true."); } - if (getCaptureOutput().getOrElse(false) && getIndentingConsoleOutput().isPresent()) { + if (getCaptureOutput().get() && getIndentingConsoleOutput().isPresent()) { throw new GradleException("Capturing output is not supported when indentingConsoleOutput is configured."); } Consumer outputLogger; @@ -156,7 +160,9 @@ public void run() { if (getLogger().isInfoEnabled() == false) { if (exitValue != 0) { try { - getLogger().error("Output for " + getExecutable().get() + ":"); + if (getIndentingConsoleOutput().isPresent() == false) { + getLogger().error("Output for " + getExecutable().get() + ":"); + } outputLogger.accept(getLogger()); } catch (Exception e) { throw new GradleException("Failed to read exec output", e); @@ -173,10 +179,6 @@ private String byteStreamToString(OutputStream out) { return ((ByteArrayOutputStream) out).toString(StandardCharsets.UTF_8); } - public void setSpoolOutput(boolean spoolOutput) { - this.spoolOutput = spoolOutput; - } - public static ExecResult exec(ExecOperations execOperations, Action action) { return genericExec(execOperations::exec, action); } diff --git a/docs/changelog/88257.yaml b/docs/changelog/88257.yaml new file mode 100644 index 0000000000000..feb5df85feafb --- /dev/null +++ b/docs/changelog/88257.yaml @@ -0,0 +1,6 @@ +pr: 88257 +summary: INFO logging of snapshot restore and completion +area: Snapshot/Restore +type: enhancement +issues: + - 86610 diff --git a/docs/changelog/88276.yaml b/docs/changelog/88276.yaml new file mode 100644 index 0000000000000..942d83375b361 --- /dev/null +++ b/docs/changelog/88276.yaml @@ -0,0 +1,5 @@ +pr: 88276 +summary: Updatable API keys - logging audit trail event +area: Audit +type: enhancement +issues: [] diff --git a/docs/changelog/88333.yaml b/docs/changelog/88333.yaml new file mode 100644 index 0000000000000..f72dbe19ff1b7 --- /dev/null +++ b/docs/changelog/88333.yaml @@ -0,0 +1,5 @@ +pr: 88333 +summary: "Script: Metadata for update context" +area: Infra/Scripting +type: enhancement +issues: [] diff --git a/docs/changelog/88346.yaml b/docs/changelog/88346.yaml new file mode 100644 index 0000000000000..ca2537f28a5a9 --- /dev/null +++ b/docs/changelog/88346.yaml @@ -0,0 +1,5 @@ +pr: 88346 +summary: Updatable API keys - noop check +area: Security +type: enhancement +issues: [] diff --git a/docs/changelog/88399.yaml b/docs/changelog/88399.yaml new file mode 100644 index 0000000000000..f38fc092ae629 --- /dev/null +++ b/docs/changelog/88399.yaml @@ -0,0 +1,6 @@ +pr: 88399 +summary: Improve error when sorting on incompatible types +area: Search +type: enhancement +issues: + - 73146 diff --git a/docs/changelog/88456.yaml b/docs/changelog/88456.yaml new file mode 100644 index 0000000000000..bb3a5d1182365 --- /dev/null +++ b/docs/changelog/88456.yaml @@ -0,0 +1,5 @@ +pr: 88456 +summary: Audit API key ID when create or grant API keys +area: Audit +type: enhancement +issues: [] diff --git a/docs/reference/aggregations/metrics/avg-aggregation.asciidoc b/docs/reference/aggregations/metrics/avg-aggregation.asciidoc index 05d112a13ac03..4883eec3e617a 100644 --- a/docs/reference/aggregations/metrics/avg-aggregation.asciidoc +++ b/docs/reference/aggregations/metrics/avg-aggregation.asciidoc @@ -4,7 +4,7 @@ Avg ++++ -A `single-value` metrics aggregation that computes the average of numeric values that are extracted from the aggregated documents. These values can be extracted either from specific numeric fields in the documents. +A `single-value` metrics aggregation that computes the average of numeric values that are extracted from the aggregated documents. These values can be extracted either from specific numeric or <> fields in the documents. Assuming the data consists of documents representing exams grades (between 0 and 100) of students we can average their scores with: diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.update.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.update.txt index 1aacbb4d8cf40..ee2846331ee0f 100644 --- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.update.txt +++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.update.txt @@ -18,3 +18,17 @@ class org.elasticsearch.painless.api.Json { String dump(def) String dump(def, boolean) } + +class org.elasticsearch.script.Metadata { + String getIndex() + String getId() + String getRouting() + long getVersion() + String getOp() + void setOp(String) + ZonedDateTime getTimestamp() +} + +class org.elasticsearch.script.UpdateScript { + Metadata metadata() +} diff --git a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/15_update.yml b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/15_update.yml index a23a27a2e6578..e00f82af7cc76 100644 --- a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/15_update.yml +++ b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/15_update.yml @@ -123,3 +123,47 @@ - match: { error.root_cause.0.type: "illegal_argument_exception" } - match: { error.type: "illegal_argument_exception" } - match: { error.reason: "Iterable object is self-referencing itself" } + +--- +"Script Update Metadata": + - skip: + version: " - 8.3.99" + reason: "update metadata introduced in 8.4.0" + + - do: + update: + index: test_1 + id: "2" + body: + script: + source: "ctx._source.bar = metadata().id + '-extra'" + lang: "painless" + upsert: {} + scripted_upsert: true + + - do: + get: + index: test_1 + id: "2" + + - match: { _source.bar: 2-extra } + - match: { found: true } + + - do: + update: + index: test_1 + id: "2" + body: + script: + source: "metadata().op = 'delete'" + lang: "painless" + upsert: {} + scripted_upsert: true + + - do: + catch: missing + get: + index: test_1 + id: "2" + + - match: { found: false } diff --git a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/25_script_upsert.yml b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/25_script_upsert.yml index 559a54d28a19e..865bed8de24e9 100644 --- a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/25_script_upsert.yml +++ b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/25_script_upsert.yml @@ -93,3 +93,97 @@ id: "4" - match: { _source.within_one_minute: true } + +--- +"Script Upsert Metadata": + - skip: + version: " - 8.3.99" + reason: "update metadata introduced in 8.4.0" + + - do: + catch: /routing is unavailable for insert/ + update: + index: test_1 + id: "1" + body: + script: + source: "ctx._source.foo = metadata().routing" + lang: "painless" + upsert: {} + scripted_upsert: true + + - do: + update: + index: test_1 + id: "2" + body: + script: + source: "ctx._source.foo = metadata().index + '_1'; ctx._source.bar = 'nothing'" + lang: "painless" + upsert: {} + scripted_upsert: true + + - do: + get: + index: test_1 + id: "2" + + - match: { _source.foo: test_1_1 } + - match: { _source.bar: nothing } + + - do: + update: + index: test_1 + id: "3" + body: + script: + source: "metadata().op = 'noop'; ctx._source.bar = 'skipped?'" + lang: "painless" + upsert: {} + scripted_upsert: true + + - do: + catch: missing + get: + index: test_1 + id: "3" + + - match: { found: false } + + - do: + update: + index: test_1 + id: "3" + body: + script: + source: "metadata().op = 'create'; ctx._source.bar = 'skipped?'" + lang: "painless" + upsert: {} + scripted_upsert: true + + - do: + get: + index: test_1 + id: "3" + + - match: { found: true } + - match: { _source.bar: "skipped?" } + + # update + - do: + update: + index: test_1 + id: "2" + body: + script: + source: "ctx._source.bar = metadata().op + '-extra'" + lang: "painless" + upsert: {} + scripted_upsert: true + + - do: + get: + index: test_1 + id: "2" + + - match: { _source.bar: index-extra } diff --git a/modules/legacy-geo/src/internalClusterTest/java/org/elasticsearch/legacygeo/search/GeoBoundingBoxQueryLegacyGeoShapeIT.java b/modules/legacy-geo/src/internalClusterTest/java/org/elasticsearch/legacygeo/search/GeoBoundingBoxQueryLegacyGeoShapeIT.java index ee12e28c35616..b3c888af3327a 100644 --- a/modules/legacy-geo/src/internalClusterTest/java/org/elasticsearch/legacygeo/search/GeoBoundingBoxQueryLegacyGeoShapeIT.java +++ b/modules/legacy-geo/src/internalClusterTest/java/org/elasticsearch/legacygeo/search/GeoBoundingBoxQueryLegacyGeoShapeIT.java @@ -22,11 +22,6 @@ public class GeoBoundingBoxQueryLegacyGeoShapeIT extends GeoBoundingBoxQueryIntegTestCase { - @Override - protected boolean addMockGeoShapeFieldMapper() { - return false; - } - @Override protected Collection> nodePlugins() { return Collections.singleton(TestLegacyGeoShapeFieldMapperPlugin.class); diff --git a/modules/legacy-geo/src/internalClusterTest/java/org/elasticsearch/legacygeo/search/LegacyGeoShapeIT.java b/modules/legacy-geo/src/internalClusterTest/java/org/elasticsearch/legacygeo/search/LegacyGeoShapeIT.java index 263d3840f7333..a92aba3cbee66 100644 --- a/modules/legacy-geo/src/internalClusterTest/java/org/elasticsearch/legacygeo/search/LegacyGeoShapeIT.java +++ b/modules/legacy-geo/src/internalClusterTest/java/org/elasticsearch/legacygeo/search/LegacyGeoShapeIT.java @@ -28,11 +28,6 @@ public class LegacyGeoShapeIT extends GeoShapeIntegTestCase { - @Override - protected boolean addMockGeoShapeFieldMapper() { - return false; - } - @Override protected Collection> nodePlugins() { return Collections.singleton(TestLegacyGeoShapeFieldMapperPlugin.class); diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/RankFeatureQueryBuilderTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/RankFeatureQueryBuilderTests.java index 4bd2d0714e93d..0e496990efc40 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/RankFeatureQueryBuilderTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/RankFeatureQueryBuilderTests.java @@ -19,7 +19,6 @@ import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.AbstractQueryTestCase; -import org.elasticsearch.test.TestGeoShapeFieldMapperPlugin; import java.io.IOException; import java.util.ArrayList; @@ -54,7 +53,7 @@ protected void initializeAdditionalMappings(MapperService mapperService) throws @Override protected Collection> getPlugins() { - return Arrays.asList(MapperExtrasPlugin.class, TestGeoShapeFieldMapperPlugin.class); + return Arrays.asList(MapperExtrasPlugin.class); } @Override diff --git a/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ChildrenTests.java b/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ChildrenTests.java index 96660aea34d23..9d12d1bc3d72f 100644 --- a/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ChildrenTests.java +++ b/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ChildrenTests.java @@ -11,7 +11,6 @@ import org.elasticsearch.join.ParentJoinPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.search.aggregations.BaseAggregationTestCase; -import org.elasticsearch.test.TestGeoShapeFieldMapperPlugin; import java.util.Arrays; import java.util.Collection; @@ -20,7 +19,7 @@ public class ChildrenTests extends BaseAggregationTestCase> getPlugins() { - return Arrays.asList(ParentJoinPlugin.class, TestGeoShapeFieldMapperPlugin.class); + return Arrays.asList(ParentJoinPlugin.class); } @Override diff --git a/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ParentTests.java b/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ParentTests.java index c424dab398803..75fb02eab2dbb 100644 --- a/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ParentTests.java +++ b/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ParentTests.java @@ -11,7 +11,6 @@ import org.elasticsearch.join.ParentJoinPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.search.aggregations.BaseAggregationTestCase; -import org.elasticsearch.test.TestGeoShapeFieldMapperPlugin; import java.util.Arrays; import java.util.Collection; @@ -20,7 +19,7 @@ public class ParentTests extends BaseAggregationTestCase> getPlugins() { - return Arrays.asList(ParentJoinPlugin.class, TestGeoShapeFieldMapperPlugin.class); + return Arrays.asList(ParentJoinPlugin.class); } @Override diff --git a/modules/parent-join/src/test/java/org/elasticsearch/join/query/HasChildQueryBuilderTests.java b/modules/parent-join/src/test/java/org/elasticsearch/join/query/HasChildQueryBuilderTests.java index a32164322ce63..6f857674b6fe5 100644 --- a/modules/parent-join/src/test/java/org/elasticsearch/join/query/HasChildQueryBuilderTests.java +++ b/modules/parent-join/src/test/java/org/elasticsearch/join/query/HasChildQueryBuilderTests.java @@ -45,7 +45,6 @@ import org.elasticsearch.search.sort.FieldSortBuilder; import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.test.AbstractQueryTestCase; -import org.elasticsearch.test.TestGeoShapeFieldMapperPlugin; import org.elasticsearch.test.VersionUtils; import org.elasticsearch.xcontent.XContentBuilder; @@ -77,7 +76,7 @@ public class HasChildQueryBuilderTests extends AbstractQueryTestCase> getPlugins() { - return Arrays.asList(ParentJoinPlugin.class, TestGeoShapeFieldMapperPlugin.class); + return Arrays.asList(ParentJoinPlugin.class); } @Override diff --git a/modules/parent-join/src/test/java/org/elasticsearch/join/query/HasParentQueryBuilderTests.java b/modules/parent-join/src/test/java/org/elasticsearch/join/query/HasParentQueryBuilderTests.java index 88176cd0f39fc..5801919aa4b24 100644 --- a/modules/parent-join/src/test/java/org/elasticsearch/join/query/HasParentQueryBuilderTests.java +++ b/modules/parent-join/src/test/java/org/elasticsearch/join/query/HasParentQueryBuilderTests.java @@ -31,7 +31,6 @@ import org.elasticsearch.search.sort.FieldSortBuilder; import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.test.AbstractQueryTestCase; -import org.elasticsearch.test.TestGeoShapeFieldMapperPlugin; import org.elasticsearch.test.VersionUtils; import org.elasticsearch.xcontent.XContentBuilder; @@ -60,7 +59,7 @@ public class HasParentQueryBuilderTests extends AbstractQueryTestCase> getPlugins() { - return Arrays.asList(ParentJoinPlugin.class, TestGeoShapeFieldMapperPlugin.class); + return Arrays.asList(ParentJoinPlugin.class); } @Override diff --git a/modules/parent-join/src/test/java/org/elasticsearch/join/query/ParentIdQueryBuilderTests.java b/modules/parent-join/src/test/java/org/elasticsearch/join/query/ParentIdQueryBuilderTests.java index 0340ac991db21..19a268bc73391 100644 --- a/modules/parent-join/src/test/java/org/elasticsearch/join/query/ParentIdQueryBuilderTests.java +++ b/modules/parent-join/src/test/java/org/elasticsearch/join/query/ParentIdQueryBuilderTests.java @@ -23,7 +23,6 @@ import org.elasticsearch.join.ParentJoinPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.AbstractQueryTestCase; -import org.elasticsearch.test.TestGeoShapeFieldMapperPlugin; import org.elasticsearch.xcontent.XContentBuilder; import org.hamcrest.Matchers; @@ -48,7 +47,7 @@ public class ParentIdQueryBuilderTests extends AbstractQueryTestCase> getPlugins() { - return Arrays.asList(ParentJoinPlugin.class, TestGeoShapeFieldMapperPlugin.class); + return Arrays.asList(ParentJoinPlugin.class); } @Override diff --git a/modules/percolator/src/internalClusterTest/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java b/modules/percolator/src/internalClusterTest/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java index 6f64e697e907e..315ff6492a23c 100644 --- a/modules/percolator/src/internalClusterTest/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java +++ b/modules/percolator/src/internalClusterTest/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java @@ -62,11 +62,6 @@ public class PercolatorQuerySearchIT extends ESIntegTestCase { - @Override - protected boolean addMockGeoShapeFieldMapper() { - return false; - } - @Override protected Collection> nodePlugins() { return Arrays.asList(PercolatorPlugin.class, TestGeoShapeFieldMapperPlugin.class); diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryBuilderTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryBuilderTests.java index 9003e749f0d9c..b0df61bad4c53 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryBuilderTests.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryBuilderTests.java @@ -29,7 +29,6 @@ import org.elasticsearch.ingest.RandomDocumentPicks; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.AbstractQueryTestCase; -import org.elasticsearch.test.TestGeoShapeFieldMapperPlugin; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentFactory; import org.elasticsearch.xcontent.XContentParser; @@ -77,7 +76,7 @@ public class PercolateQueryBuilderTests extends AbstractQueryTestCase> getPlugins() { - return Arrays.asList(PercolatorPlugin.class, TestGeoShapeFieldMapperPlugin.class); + return Arrays.asList(PercolatorPlugin.class); } @Override diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/AbstractAsyncBulkByScrollActionScriptTestCase.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/AbstractAsyncBulkByScrollActionScriptTestCase.java index 3a6a7fcb6c42c..02d5aaef31098 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/AbstractAsyncBulkByScrollActionScriptTestCase.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/AbstractAsyncBulkByScrollActionScriptTestCase.java @@ -53,7 +53,7 @@ protected T applyScript(Consumer> (params, ctx) -> new UpdateScript(Collections.emptyMap(), ctx) { @Override public void execute() { - scriptBody.accept(getCtx()); + scriptBody.accept(ctx); } } ); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzerIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzerIT.java index 21677e420dd23..e7ba86ed5bfcd 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzerIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzerIT.java @@ -25,6 +25,7 @@ import org.elasticsearch.plugins.EnginePlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.TestGeoShapeFieldMapperPlugin; import org.elasticsearch.test.transport.MockTransportService; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xcontent.XContentBuilder; @@ -57,6 +58,7 @@ protected Collection> nodePlugins() { List> plugins = new ArrayList<>(super.nodePlugins()); plugins.add(EngineTestPlugin.class); plugins.add(MockTransportService.TestPlugin.class); + plugins.add(TestGeoShapeFieldMapperPlugin.class); return plugins; } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/ClusterStateDiffIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/ClusterStateDiffIT.java index 5b995482f6fa7..b669cbf11bffc 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/ClusterStateDiffIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/ClusterStateDiffIT.java @@ -727,6 +727,7 @@ public ClusterState.Custom randomCreate(String name) { UUIDs.randomBase64UUID(), new Snapshot(randomName("repo"), new SnapshotId(randomName("snap"), UUIDs.randomBase64UUID())), RestoreInProgress.State.fromValue((byte) randomIntBetween(0, 3)), + randomBoolean(), emptyList(), ImmutableOpenMap.of() ) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/geo/GeoBoundingBoxQueryGeoPointIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/geo/GeoBoundingBoxQueryGeoPointIT.java index 5d4b24d12b63f..f9909f9de5b02 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/geo/GeoBoundingBoxQueryGeoPointIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/geo/GeoBoundingBoxQueryGeoPointIT.java @@ -9,14 +9,24 @@ package org.elasticsearch.search.geo; import org.elasticsearch.Version; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.test.TestGeoShapeFieldMapperPlugin; import org.elasticsearch.test.VersionUtils; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentFactory; import java.io.IOException; +import java.util.Collection; +import java.util.Collections; public class GeoBoundingBoxQueryGeoPointIT extends GeoBoundingBoxQueryIntegTestCase { + @SuppressWarnings("deprecation") + @Override + protected Collection> nodePlugins() { + return Collections.singleton(TestGeoShapeFieldMapperPlugin.class); + } + @Override public XContentBuilder getMapping() throws IOException { XContentBuilder xContentBuilder = XContentFactory.jsonBuilder() diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/geo/GeoBoundingBoxQueryGeoShapeIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/geo/GeoBoundingBoxQueryGeoShapeIT.java index a81801c53c34b..403b890ea3dc6 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/geo/GeoBoundingBoxQueryGeoShapeIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/geo/GeoBoundingBoxQueryGeoShapeIT.java @@ -9,14 +9,24 @@ package org.elasticsearch.search.geo; import org.elasticsearch.Version; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.test.TestGeoShapeFieldMapperPlugin; import org.elasticsearch.test.VersionUtils; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentFactory; import java.io.IOException; +import java.util.Collection; +import java.util.Collections; public class GeoBoundingBoxQueryGeoShapeIT extends GeoBoundingBoxQueryIntegTestCase { + @SuppressWarnings("deprecation") + @Override + protected Collection> nodePlugins() { + return Collections.singleton(TestGeoShapeFieldMapperPlugin.class); + } + @Override public XContentBuilder getMapping() throws IOException { XContentBuilder xContentBuilder = XContentFactory.jsonBuilder() diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/geo/GeoShapeIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/geo/GeoShapeIT.java index 5394232ace8fa..a12de847a62bd 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/geo/GeoShapeIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/geo/GeoShapeIT.java @@ -9,13 +9,23 @@ package org.elasticsearch.search.geo; import org.elasticsearch.Version; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.test.TestGeoShapeFieldMapperPlugin; import org.elasticsearch.test.VersionUtils; import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; +import java.util.Collection; +import java.util.Collections; public class GeoShapeIT extends GeoShapeIntegTestCase { + @SuppressWarnings("deprecation") + @Override + protected Collection> nodePlugins() { + return Collections.singleton(TestGeoShapeFieldMapperPlugin.class); + } + @Override protected void getGeoShapeMapping(XContentBuilder b) throws IOException { b.field("type", "geo_shape"); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/query/QueryStringIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/query/QueryStringIT.java index 2090c4909870a..743cf268caf78 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/query/QueryStringIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/query/QueryStringIT.java @@ -13,14 +13,17 @@ import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.query.Operator; +import org.elasticsearch.plugins.Plugin; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.TestGeoShapeFieldMapperPlugin; import org.elasticsearch.xcontent.XContentType; import org.junit.Before; import java.io.IOException; import java.util.ArrayList; +import java.util.Collection; import java.util.HashSet; import java.util.List; import java.util.Set; @@ -35,6 +38,12 @@ public class QueryStringIT extends ESIntegTestCase { + @SuppressWarnings("deprecation") + @Override + protected Collection> nodePlugins() { + return List.of(TestGeoShapeFieldMapperPlugin.class); + } + @Before public void setup() throws Exception { String indexBody = copyToStringFromClasspath("/org/elasticsearch/search/query/all-query-index.json"); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/query/SimpleQueryStringIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/query/SimpleQueryStringIT.java index 387123189217c..7a4941d8454df 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/query/SimpleQueryStringIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/query/SimpleQueryStringIT.java @@ -28,13 +28,13 @@ import org.elasticsearch.search.SearchHits; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.TestGeoShapeFieldMapperPlugin; import org.elasticsearch.xcontent.XContentFactory; import org.elasticsearch.xcontent.XContentType; import java.io.IOException; import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Set; @@ -65,7 +65,7 @@ public class SimpleQueryStringIT extends ESIntegTestCase { @Override protected Collection> nodePlugins() { - return Collections.singletonList(MockAnalysisPlugin.class); + return List.of(MockAnalysisPlugin.class, TestGeoShapeFieldMapperPlugin.class); } public void testSimpleQueryString() throws ExecutionException, InterruptedException { diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/sort/FieldSortIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/sort/FieldSortIT.java index 1593aed61df44..4e4997134fdcc 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/sort/FieldSortIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/sort/FieldSortIT.java @@ -2122,4 +2122,43 @@ public void testLongSortOptimizationCorrectResults() { } } + public void testSortMixedFieldTypes() { + assertAcked(prepareCreate("index_long").setMapping("foo", "type=long").get()); + assertAcked(prepareCreate("index_integer").setMapping("foo", "type=integer").get()); + assertAcked(prepareCreate("index_double").setMapping("foo", "type=double").get()); + assertAcked(prepareCreate("index_keyword").setMapping("foo", "type=keyword").get()); + + client().prepareIndex("index_long").setId("1").setSource("foo", "123").get(); + client().prepareIndex("index_integer").setId("1").setSource("foo", "123").get(); + client().prepareIndex("index_double").setId("1").setSource("foo", "123").get(); + client().prepareIndex("index_keyword").setId("1").setSource("foo", "123").get(); + refresh(); + + { // mixing long and integer types is ok, as we convert integer sort to long sort + SearchResponse searchResponse = client().prepareSearch("index_long", "index_integer") + .addSort(new FieldSortBuilder("foo")) + .setSize(10) + .get(); + assertSearchResponse(searchResponse); + } + + String errMsg = "Can't sort on field [foo]; the field has incompatible sort types"; + + { // mixing long and double types is not allowed + SearchPhaseExecutionException exc = expectThrows( + SearchPhaseExecutionException.class, + () -> client().prepareSearch("index_long", "index_double").addSort(new FieldSortBuilder("foo")).setSize(10).get() + ); + assertThat(exc.getCause().toString(), containsString(errMsg)); + } + + { // mixing long and keyword types is not allowed + SearchPhaseExecutionException exc = expectThrows( + SearchPhaseExecutionException.class, + () -> client().prepareSearch("index_long", "index_keyword").addSort(new FieldSortBuilder("foo")).setSize(10).get() + ); + assertThat(exc.getCause().toString(), containsString(errMsg)); + } + } + } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/RestoreSnapshotIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/RestoreSnapshotIT.java index c9c0c4a6cd60c..23c43a16308bd 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/RestoreSnapshotIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/RestoreSnapshotIT.java @@ -31,6 +31,7 @@ import org.elasticsearch.repositories.fs.FsRepository; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.test.MockLogAppender; +import org.elasticsearch.test.junit.annotations.TestLogging; import org.elasticsearch.xcontent.XContentFactory; import java.nio.file.Path; @@ -158,6 +159,67 @@ public void testParallelRestoreOperationsFromSingleSnapshot() throws Exception { assertThat(client.prepareGet(restoredIndexName2, sameSourceIndex ? docId : docId2).get().isExists(), equalTo(true)); } + @TestLogging( + reason = "testing the logging of the start and completion of a snapshot restore", + value = "org.elasticsearch.snapshots.RestoreService:INFO" + ) + public void testRestoreLogging() throws IllegalAccessException { + final MockLogAppender mockLogAppender = new MockLogAppender(); + try { + String indexName = "testindex"; + String repoName = "test-restore-snapshot-repo"; + String snapshotName = "test-restore-snapshot"; + Path absolutePath = randomRepoPath().toAbsolutePath(); + logger.info("Path [{}]", absolutePath); + String restoredIndexName = indexName + "-restored"; + String expectedValue = "expected"; + + mockLogAppender.start(); + Loggers.addAppender(LogManager.getLogger(RestoreService.class), mockLogAppender); + + mockLogAppender.addExpectation( + new MockLogAppender.PatternSeenEventExpectation( + "not seen start of snapshot restore", + "org.elasticsearch.snapshots.RestoreService", + Level.INFO, + "started restore of snapshot \\[.*" + snapshotName + ".*\\] for indices \\[.*" + indexName + ".*\\]" + ) + ); + + mockLogAppender.addExpectation( + new MockLogAppender.PatternSeenEventExpectation( + "not seen completion of snapshot restore", + "org.elasticsearch.snapshots.RestoreService", + Level.INFO, + "completed restore of snapshot \\[.*" + snapshotName + ".*\\]" + ) + ); + + Client client = client(); + // Write a document + String docId = Integer.toString(randomInt()); + indexDoc(indexName, docId, "value", expectedValue); + createRepository(repoName, "fs", absolutePath); + createSnapshot(repoName, snapshotName, Collections.singletonList(indexName)); + + RestoreSnapshotResponse restoreSnapshotResponse = client.admin() + .cluster() + .prepareRestoreSnapshot(repoName, snapshotName) + .setWaitForCompletion(false) + .setRenamePattern(indexName) + .setRenameReplacement(restoredIndexName) + .get(); + + assertThat(restoreSnapshotResponse.status(), equalTo(RestStatus.ACCEPTED)); + ensureGreen(restoredIndexName); + assertThat(client.prepareGet(restoredIndexName, docId).get().isExists(), equalTo(true)); + mockLogAppender.assertAllExpectationsMatched(); + } finally { + Loggers.removeAppender(LogManager.getLogger(RestoreService.class), mockLogAppender); + mockLogAppender.stop(); + } + } + public void testRestoreIncreasesPrimaryTerms() { final String indexName = randomAlphaOfLengthBetween(5, 10).toLowerCase(Locale.ROOT); createIndex(indexName, indexSettingsNoReplicas(2).build()); diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java index 8eb5f6d48ecc2..c383b2d610e5f 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java @@ -8,6 +8,7 @@ package org.elasticsearch.action.admin.cluster.snapshots.restore; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.master.MasterNodeRequest; @@ -48,6 +49,8 @@ public class RestoreSnapshotRequest extends MasterNodeRequest results, int topN, int from) { final TopFieldGroups[] shardTopDocs = results.toArray(new TopFieldGroups[numShards]); mergedTopDocs = TopFieldGroups.merge(sort, from, topN, shardTopDocs, false); } else if (topDocs instanceof TopFieldDocs firstTopDocs) { + checkSameSortTypes(results, firstTopDocs.fields); final Sort sort = new Sort(firstTopDocs.fields); final TopFieldDocs[] shardTopDocs = results.toArray(new TopFieldDocs[numShards]); mergedTopDocs = TopDocs.merge(sort, from, topN, shardTopDocs); @@ -207,6 +210,56 @@ static TopDocs mergeTopDocs(Collection results, int topN, int from) { return mergedTopDocs; } + private static void checkSameSortTypes(Collection results, SortField[] firstSortFields) { + if (results.size() < 2) return; + + SortField.Type[] firstTypes = new SortField.Type[firstSortFields.length]; + boolean isFirstResult = true; + for (TopDocs topDocs : results) { + SortField[] curSortFields = ((TopFieldDocs) topDocs).fields; + if (isFirstResult) { + for (int i = 0; i < curSortFields.length; i++) { + firstTypes[i] = getType(firstSortFields[i]); + if (firstTypes[i] == SortField.Type.CUSTOM) { + // for custom types that we can't resolve, we can't do the check + return; + } + } + isFirstResult = false; + } else { + for (int i = 0; i < curSortFields.length; i++) { + SortField.Type curType = getType(curSortFields[i]); + if (curType != firstTypes[i]) { + if (curType == SortField.Type.CUSTOM) { + // for custom types that we can't resolve, we can't do the check + return; + } + throw new IllegalArgumentException( + "Can't sort on field [" + + curSortFields[i].getField() + + "]; the field has incompatible sort types: [" + + firstTypes[i] + + "] and [" + + curType + + "] across shards!" + ); + } + } + } + } + } + + private static SortField.Type getType(SortField sortField) { + if (sortField instanceof SortedNumericSortField) { + return ((SortedNumericSortField) sortField).getNumericType(); + } + if (sortField instanceof SortedSetSortField) { + return SortField.Type.STRING; + } else { + return sortField.getType(); + } + } + static void setShardIndex(TopDocs topDocs, int shardIndex) { assert topDocs.scoreDocs.length == 0 || topDocs.scoreDocs[0].shardIndex == -1 : "shardIndex is already set"; for (ScoreDoc doc : topDocs.scoreDocs) { diff --git a/server/src/main/java/org/elasticsearch/action/support/ListenableActionFuture.java b/server/src/main/java/org/elasticsearch/action/support/ListenableActionFuture.java index 7da2324bb873b..bbac7cba72b43 100644 --- a/server/src/main/java/org/elasticsearch/action/support/ListenableActionFuture.java +++ b/server/src/main/java/org/elasticsearch/action/support/ListenableActionFuture.java @@ -14,7 +14,7 @@ import java.util.List; /** - * A {@code Future} and {@link ActionListener} against which which other {@link ActionListener}s can be registered later, to support + * A {@code Future} and {@link ActionListener} against which other {@link ActionListener}s can be registered later, to support * fanning-out a result to a dynamic collection of listeners. */ public class ListenableActionFuture extends AdapterActionFuture { diff --git a/server/src/main/java/org/elasticsearch/action/update/UpdateHelper.java b/server/src/main/java/org/elasticsearch/action/update/UpdateHelper.java index 06b850815681e..83bceeaceedf2 100644 --- a/server/src/main/java/org/elasticsearch/action/update/UpdateHelper.java +++ b/server/src/main/java/org/elasticsearch/action/update/UpdateHelper.java @@ -18,7 +18,6 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.common.util.Maps; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.Tuple; @@ -33,6 +32,8 @@ import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptService; import org.elasticsearch.script.UpdateScript; +import org.elasticsearch.script.UpdateCtxMap; +import org.elasticsearch.script.UpsertCtxMap; import org.elasticsearch.search.lookup.SourceLookup; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentType; @@ -87,25 +88,17 @@ protected Result prepare(ShardId shardId, UpdateRequest request, final GetResult * Execute a scripted upsert, where there is an existing upsert document and a script to be executed. The script is executed and a new * Tuple of operation and updated {@code _source} is returned. */ - Tuple> executeScriptedUpsert(Map upsertDoc, Script script, LongSupplier nowInMillis) { - Map ctx = Maps.newMapWithExpectedSize(3); - // Tell the script that this is a create and not an update - ctx.put(ContextFields.OP, UpdateOpType.CREATE.toString()); - ctx.put(ContextFields.SOURCE, upsertDoc); - ctx.put(ContextFields.NOW, nowInMillis.getAsLong()); - ctx = executeScript(script, ctx); - - UpdateOpType operation = UpdateOpType.lenientFromString((String) ctx.get(ContextFields.OP), logger, script.getIdOrCode()); - @SuppressWarnings("unchecked") - Map newSource = (Map) ctx.get(ContextFields.SOURCE); - + Tuple> executeScriptedUpsert(Script script, UpsertCtxMap srcAndMeta) { + // Tell the script that this is a create and not an update (insert from upsert) + srcAndMeta = executeScript(script, srcAndMeta); + UpdateOpType operation = UpdateOpType.lenientFromString(srcAndMeta.getOp(), logger, script.getIdOrCode()); if (operation != UpdateOpType.CREATE && operation != UpdateOpType.NONE) { // Only valid options for an upsert script are "create" (the default) or "none", meaning abort upsert logger.warn("Invalid upsert operation [{}] for script [{}], doing nothing...", operation, script.getIdOrCode()); operation = UpdateOpType.NONE; } - return new Tuple<>(operation, newSource); + return new Tuple<>(operation, srcAndMeta.getSource()); } /** @@ -120,11 +113,14 @@ Result prepareUpsert(ShardId shardId, UpdateRequest request, final GetResult get if (request.scriptedUpsert() && request.script() != null) { // Run the script to perform the create logic IndexRequest upsert = request.upsertRequest(); - Tuple> upsertResult = executeScriptedUpsert( - upsert.sourceAsMap(), - request.script, - nowInMillis + UpsertCtxMap srcAndMeta = new UpsertCtxMap( + getResult.getIndex(), + getResult.getId(), + UpdateOpType.CREATE.toString(), + nowInMillis.getAsLong(), + upsert.sourceAsMap() ); + Tuple> upsertResult = executeScriptedUpsert(request.script, srcAndMeta); switch (upsertResult.v1()) { case CREATE -> indexRequest = Requests.indexRequest(request.index()).source(upsertResult.v2()); case NONE -> { @@ -237,24 +233,22 @@ Result prepareUpdateScriptRequest(ShardId shardId, UpdateRequest request, GetRes final String routing = calculateRouting(getResult, currentRequest); final Tuple> sourceAndContent = XContentHelper.convertToMap(getResult.internalSourceRef(), true); final XContentType updateSourceContentType = sourceAndContent.v1(); - final Map sourceAsMap = sourceAndContent.v2(); - - Map ctx = Maps.newMapWithExpectedSize(16); - ctx.put(ContextFields.OP, UpdateOpType.INDEX.toString()); // The default operation is "index" - ctx.put(ContextFields.INDEX, getResult.getIndex()); - ctx.put(ContextFields.TYPE, MapperService.SINGLE_MAPPING_NAME); - ctx.put(ContextFields.ID, getResult.getId()); - ctx.put(ContextFields.VERSION, getResult.getVersion()); - ctx.put(ContextFields.ROUTING, routing); - ctx.put(ContextFields.SOURCE, sourceAsMap); - ctx.put(ContextFields.NOW, nowInMillis.getAsLong()); - - ctx = executeScript(request.script, ctx); - UpdateOpType operation = UpdateOpType.lenientFromString((String) ctx.get(ContextFields.OP), logger, request.script.getIdOrCode()); - - @SuppressWarnings("unchecked") - final Map updatedSourceAsMap = (Map) ctx.get(ContextFields.SOURCE); + UpdateCtxMap srcAndMeta = executeScript( + request.script, + new UpdateCtxMap( + getResult.getIndex(), + getResult.getId(), + getResult.getVersion(), + routing, + MapperService.SINGLE_MAPPING_NAME, + UpdateOpType.INDEX.toString(), // The default operation is "index" + nowInMillis.getAsLong(), + sourceAndContent.v2() + ) + ); + UpdateOpType operation = UpdateOpType.lenientFromString(srcAndMeta.getOp(), logger, request.script.getIdOrCode()); + final Map updatedSourceAsMap = srcAndMeta.getSource(); switch (operation) { case INDEX -> { @@ -307,17 +301,17 @@ Result prepareUpdateScriptRequest(ShardId shardId, UpdateRequest request, GetRes } } - private Map executeScript(Script script, Map ctx) { + private T executeScript(Script script, T ctxMap) { try { if (scriptService != null) { UpdateScript.Factory factory = scriptService.compile(script, UpdateScript.CONTEXT); - UpdateScript executableScript = factory.newInstance(script.getParams(), ctx); + UpdateScript executableScript = factory.newInstance(script.getParams(), ctxMap); executableScript.execute(); } } catch (Exception e) { throw new IllegalArgumentException("failed to execute script", e); } - return ctx; + return ctxMap; } /** @@ -429,6 +423,7 @@ public static UpdateOpType lenientFromString(String operation, Logger logger, St return UpdateOpType.INDEX; case "delete": return UpdateOpType.DELETE; + case "noop": case "none": return UpdateOpType.NONE; default: diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterInfo.java b/server/src/main/java/org/elasticsearch/cluster/ClusterInfo.java index af931e6d34996..7cc4f1bdca65b 100644 --- a/server/src/main/java/org/elasticsearch/cluster/ClusterInfo.java +++ b/server/src/main/java/org/elasticsearch/cluster/ClusterInfo.java @@ -68,12 +68,12 @@ public ClusterInfo( Map routingToDataPath, Map reservedSpace ) { - this.leastAvailableSpaceUsage = leastAvailableSpaceUsage; - this.shardSizes = shardSizes; - this.shardDataSetSizes = shardDataSetSizes; - this.mostAvailableSpaceUsage = mostAvailableSpaceUsage; - this.routingToDataPath = routingToDataPath; - this.reservedSpace = reservedSpace; + this.leastAvailableSpaceUsage = Map.copyOf(leastAvailableSpaceUsage); + this.shardSizes = Map.copyOf(shardSizes); + this.shardDataSetSizes = Map.copyOf(shardDataSetSizes); + this.mostAvailableSpaceUsage = Map.copyOf(mostAvailableSpaceUsage); + this.routingToDataPath = Map.copyOf(routingToDataPath); + this.reservedSpace = Map.copyOf(reservedSpace); } public ClusterInfo(StreamInput in) throws IOException { diff --git a/server/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java b/server/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java index 0bed0fd0f7b6f..4db01fc3f796c 100644 --- a/server/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java +++ b/server/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java @@ -39,7 +39,6 @@ import org.elasticsearch.threadpool.ThreadPool; import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -188,8 +187,8 @@ public void onResponse(NodesStatsResponse nodesStatsResponse) { leastAvailableUsagesBuilder, mostAvailableUsagesBuilder ); - leastAvailableSpaceUsages = Collections.unmodifiableMap(leastAvailableUsagesBuilder); - mostAvailableSpaceUsages = Collections.unmodifiableMap(mostAvailableUsagesBuilder); + leastAvailableSpaceUsages = Map.copyOf(leastAvailableUsagesBuilder); + mostAvailableSpaceUsages = Map.copyOf(mostAvailableUsagesBuilder); } @Override @@ -262,10 +261,10 @@ public void onResponse(IndicesStatsResponse indicesStatsResponse) { reservedSpaceBuilders.forEach((nodeAndPath, builder) -> rsrvdSpace.put(nodeAndPath, builder.build())); indicesStatsSummary = new IndicesStatsSummary( - Collections.unmodifiableMap(shardSizeByIdentifierBuilder), - Collections.unmodifiableMap(shardDataSetSizeBuilder), - Collections.unmodifiableMap(dataPathByShardRoutingBuilder), - Collections.unmodifiableMap(rsrvdSpace) + Map.copyOf(shardSizeByIdentifierBuilder), + Map.copyOf(shardDataSetSizeBuilder), + Map.copyOf(dataPathByShardRoutingBuilder), + Map.copyOf(rsrvdSpace) ); } diff --git a/server/src/main/java/org/elasticsearch/cluster/RestoreInProgress.java b/server/src/main/java/org/elasticsearch/cluster/RestoreInProgress.java index df34d0babf9d8..3b19fee0210bd 100644 --- a/server/src/main/java/org/elasticsearch/cluster/RestoreInProgress.java +++ b/server/src/main/java/org/elasticsearch/cluster/RestoreInProgress.java @@ -9,6 +9,7 @@ package org.elasticsearch.cluster; import org.elasticsearch.Version; +import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotRequest; import org.elasticsearch.cluster.ClusterState.Custom; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -104,19 +105,36 @@ public RestoreInProgress build() { /** * Restore metadata */ - public record Entry(String uuid, Snapshot snapshot, State state, List indices, Map shards) { + public record Entry( + String uuid, + Snapshot snapshot, + State state, + boolean quiet, + List indices, + Map shards + ) { /** * Creates new restore metadata * * @param uuid uuid of the restore * @param snapshot snapshot * @param state current state of the restore process + * @param quiet {@code true} if logging of the start and completion of the snapshot restore should be at {@code DEBUG} log + * level, else it should be at {@code INFO} log level * @param indices list of indices being restored * @param shards map of shards being restored to their current restore status */ - public Entry(String uuid, Snapshot snapshot, State state, List indices, Map shards) { + public Entry( + String uuid, + Snapshot snapshot, + State state, + boolean quiet, + List indices, + Map shards + ) { this.snapshot = Objects.requireNonNull(snapshot); this.state = Objects.requireNonNull(state); + this.quiet = Objects.requireNonNull(quiet); this.indices = Objects.requireNonNull(indices); if (shards == null) { this.shards = Map.of(); @@ -342,10 +360,24 @@ public RestoreInProgress(StreamInput in) throws IOException { uuid = in.readString(); Snapshot snapshot = new Snapshot(in); State state = State.fromValue(in.readByte()); + boolean quiet; + if (in.getVersion().onOrAfter(RestoreSnapshotRequest.VERSION_SUPPORTING_QUIET_PARAMETER)) { + quiet = in.readBoolean(); + } else { + // Backwards compatibility: previously there was no logging of the start or completion of a snapshot restore + quiet = true; + } List indices = in.readImmutableList(StreamInput::readString); entriesBuilder.put( uuid, - new Entry(uuid, snapshot, state, indices, in.readImmutableMap(ShardId::new, ShardRestoreStatus::readShardRestoreStatus)) + new Entry( + uuid, + snapshot, + state, + quiet, + indices, + in.readImmutableMap(ShardId::new, ShardRestoreStatus::readShardRestoreStatus) + ) ); } this.entries = Collections.unmodifiableMap(entriesBuilder); @@ -357,6 +389,9 @@ public void writeTo(StreamOutput out) throws IOException { o.writeString(entry.uuid); entry.snapshot().writeTo(o); o.writeByte(entry.state().value()); + if (out.getVersion().onOrAfter(RestoreSnapshotRequest.VERSION_SUPPORTING_QUIET_PARAMETER)) { + o.writeBoolean(entry.quiet()); + } o.writeStringCollection(entry.indices); o.writeMap(entry.shards); }); diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AllocationDecider.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AllocationDecider.java index af36ca86ed44c..1b5cf0805a821 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AllocationDecider.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AllocationDecider.java @@ -42,7 +42,7 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, Routing * Returns a {@link Decision} whether the given shard routing can be remain * on the given node. The default is {@link Decision#ALWAYS}. */ - public Decision canRemain(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { + public Decision canRemain(IndexMetadata indexMetadata, ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { return Decision.ALWAYS; } diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AllocationDeciders.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AllocationDeciders.java index 02546ccbd9b90..ff0634e5ee2f6 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AllocationDeciders.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AllocationDeciders.java @@ -86,10 +86,11 @@ public Decision canRemain(ShardRouting shardRouting, RoutingNode node, RoutingAl } return Decision.NO; } + final IndexMetadata indexMetadata = allocation.metadata().getIndexSafe(shardRouting.index()); if (allocation.debugDecision()) { Decision.Multi ret = new Decision.Multi(); for (AllocationDecider allocationDecider : allocations) { - Decision decision = allocationDecider.canRemain(shardRouting, node, allocation); + Decision decision = allocationDecider.canRemain(indexMetadata, shardRouting, node, allocation); // short track if a NO is returned. if (decision.type() == Decision.Type.NO) { maybeTraceLogNoDecision(shardRouting, node, allocationDecider); @@ -103,7 +104,7 @@ public Decision canRemain(ShardRouting shardRouting, RoutingNode node, RoutingAl // tighter loop if debug information is not collected: don't collect yes decisions + break out right away on NO Decision ret = Decision.YES; for (AllocationDecider allocationDecider : allocations) { - switch (allocationDecider.canRemain(shardRouting, node, allocation).type()) { + switch (allocationDecider.canRemain(indexMetadata, shardRouting, node, allocation).type()) { case NO -> { maybeTraceLogNoDecision(shardRouting, node, allocationDecider); return Decision.NO; diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AwarenessAllocationDecider.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AwarenessAllocationDecider.java index 5f981123e2c53..73873e4a3d693 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AwarenessAllocationDecider.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AwarenessAllocationDecider.java @@ -123,7 +123,7 @@ private void setAwarenessAttributes(List awarenessAttributes) { @Override public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { - return underCapacity(shardRouting, node, allocation, true); + return underCapacity(allocation.metadata().getIndexSafe(shardRouting.index()), shardRouting, node, allocation, true); } @Override @@ -135,8 +135,8 @@ public Decision canForceAllocateDuringReplace(ShardRouting shardRouting, Routing } @Override - public Decision canRemain(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { - return underCapacity(shardRouting, node, allocation, false); + public Decision canRemain(IndexMetadata indexMetadata, ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { + return underCapacity(indexMetadata, shardRouting, node, allocation, false); } private static final Decision YES_NOT_ENABLED = Decision.single( @@ -155,13 +155,18 @@ public Decision canRemain(ShardRouting shardRouting, RoutingNode node, RoutingAl private static final Decision YES_ALL_MET = Decision.single(Decision.Type.YES, NAME, "node meets all awareness attribute requirements"); - private Decision underCapacity(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation, boolean moveToNode) { + private Decision underCapacity( + IndexMetadata indexMetadata, + ShardRouting shardRouting, + RoutingNode node, + RoutingAllocation allocation, + boolean moveToNode + ) { if (awarenessAttributes.isEmpty()) { return YES_NOT_ENABLED; } final boolean debug = allocation.debugDecision(); - final IndexMetadata indexMetadata = allocation.metadata().getIndexSafe(shardRouting.index()); if (indexMetadata.getAutoExpandReplicas().expandToAllNodes()) { return YES_AUTO_EXPAND_ALL; diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDecider.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDecider.java index 6917b9d9b7b18..69aae52a20ca4 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDecider.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDecider.java @@ -461,7 +461,7 @@ public Decision canForceAllocateDuringReplace(ShardRouting shardRouting, Routing ); @Override - public Decision canRemain(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { + public Decision canRemain(IndexMetadata indexMetadata, ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { if (shardRouting.currentNodeId().equals(node.nodeId()) == false) { throw new IllegalArgumentException("Shard [" + shardRouting + "] is not allocated on node: [" + node.nodeId() + "]"); } @@ -472,7 +472,7 @@ public Decision canRemain(ShardRouting shardRouting, RoutingNode node, RoutingAl return decision; } - if (allocation.metadata().index(shardRouting.index()).ignoreDiskWatermarks()) { + if (indexMetadata.ignoreDiskWatermarks()) { return YES_DISK_WATERMARKS_IGNORED; } diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/FilterAllocationDecider.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/FilterAllocationDecider.java index fda55b419ff01..e32fc14bc617b 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/FilterAllocationDecider.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/FilterAllocationDecider.java @@ -90,11 +90,11 @@ public FilterAllocationDecider(Settings settings, ClusterSettings clusterSetting @Override public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { + IndexMetadata indexMetadata = allocation.metadata().getIndexSafe(shardRouting.index()); if (shardRouting.unassigned() && shardRouting.recoverySource().getType() == RecoverySource.Type.LOCAL_SHARDS) { // only for unassigned - we filter allocation right after the index creation (for shard shrinking) to ensure // that once it has been allocated post API the replicas can be allocated elsewhere without user interaction // this is a setting that can only be set within the system! - IndexMetadata indexMetadata = allocation.metadata().getIndexSafe(shardRouting.index()); DiscoveryNodeFilters initialRecoveryFilters = DiscoveryNodeFilters.trimTier(indexMetadata.getInitialRecoveryFilters()); if (initialRecoveryFilters != null && initialRecoveryFilters.match(node.node()) == false) { String explanation = @@ -102,7 +102,7 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, Routing return allocation.decision(Decision.NO, NAME, explanation, initialRecoveryFilters); } } - return shouldFilter(shardRouting, node.node(), allocation); + return shouldFilter(indexMetadata, node.node(), allocation); } @Override @@ -111,8 +111,8 @@ public Decision canAllocate(IndexMetadata indexMetadata, RoutingNode node, Routi } @Override - public Decision canRemain(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { - return shouldFilter(shardRouting, node.node(), allocation); + public Decision canRemain(IndexMetadata indexMetadata, ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { + return shouldFilter(indexMetadata, node.node(), allocation); } @Override @@ -126,16 +126,6 @@ public Decision shouldAutoExpandToNode(IndexMetadata indexMetadata, DiscoveryNod return allocation.decision(Decision.YES, NAME, "node passes include/exclude/require filters"); } - private Decision shouldFilter(ShardRouting shardRouting, DiscoveryNode node, RoutingAllocation allocation) { - Decision decision = shouldClusterFilter(node, allocation); - if (decision != null) return decision; - - decision = shouldIndexFilter(allocation.metadata().getIndexSafe(shardRouting.index()), node, allocation); - if (decision != null) return decision; - - return allocation.decision(Decision.YES, NAME, "node passes include/exclude/require filters"); - } - private Decision shouldFilter(IndexMetadata indexMd, DiscoveryNode node, RoutingAllocation allocation) { Decision decision = shouldClusterFilter(node, allocation); if (decision != null) return decision; diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/NodeReplacementAllocationDecider.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/NodeReplacementAllocationDecider.java index dbaecd67cf429..f97e8ce9b5cea 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/NodeReplacementAllocationDecider.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/NodeReplacementAllocationDecider.java @@ -15,7 +15,6 @@ import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.allocation.RoutingAllocation; -import java.util.Map; import java.util.Optional; public class NodeReplacementAllocationDecider extends AllocationDecider { @@ -74,7 +73,7 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, Routing } @Override - public Decision canRemain(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { + public Decision canRemain(IndexMetadata indexMetadata, ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { if (replacementOngoing(allocation) == false) { return NO_REPLACEMENTS; } else if (isReplacementSource(allocation, node.nodeId())) { @@ -185,8 +184,8 @@ private static boolean isReplacementSource(RoutingAllocation allocation, String if (nodeId == null || replacementOngoing(allocation) == false) { return false; } - final Map nodeShutdowns = allocation.nodeShutdowns(); - return nodeShutdowns.containsKey(nodeId) && nodeShutdowns.get(nodeId).getType().equals(SingleNodeShutdownMetadata.Type.REPLACE); + final SingleNodeShutdownMetadata shutdown = allocation.nodeShutdowns().get(nodeId); + return shutdown != null && shutdown.getType().equals(SingleNodeShutdownMetadata.Type.REPLACE); } /** diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/NodeShutdownAllocationDecider.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/NodeShutdownAllocationDecider.java index ff6cd0796e98e..0c6a481ce03eb 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/NodeShutdownAllocationDecider.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/NodeShutdownAllocationDecider.java @@ -63,7 +63,7 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, Routing * determine if shards can remain on their current node. */ @Override - public Decision canRemain(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { + public Decision canRemain(IndexMetadata indexMetadata, ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { return this.canAllocate(shardRouting, node, allocation); } diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ShardsLimitAllocationDecider.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ShardsLimitAllocationDecider.java index f9868bf312224..e53688654e64b 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ShardsLimitAllocationDecider.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ShardsLimitAllocationDecider.java @@ -81,22 +81,28 @@ private void setClusterShardLimit(int clusterShardLimit) { @Override public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { - return doDecide(shardRouting, node, allocation, (count, limit) -> count >= limit); + return doDecide( + allocation.metadata().getIndexSafe(shardRouting.index()), + shardRouting, + node, + allocation, + (count, limit) -> count >= limit + ); } @Override - public Decision canRemain(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { - return doDecide(shardRouting, node, allocation, (count, limit) -> count > limit); + public Decision canRemain(IndexMetadata indexMetadata, ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { + return doDecide(indexMetadata, shardRouting, node, allocation, (count, limit) -> count > limit); } private Decision doDecide( + IndexMetadata indexMd, ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation, BiPredicate decider ) { - IndexMetadata indexMd = allocation.metadata().getIndexSafe(shardRouting.index()); final int indexShardLimit = indexMd.getShardsPerNodeLimit(); // Capture the limit here in case it changes during this method's // execution diff --git a/server/src/main/java/org/elasticsearch/index/IndexService.java b/server/src/main/java/org/elasticsearch/index/IndexService.java index e651b6b7a4da9..02b806406575c 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexService.java +++ b/server/src/main/java/org/elasticsearch/index/IndexService.java @@ -96,7 +96,6 @@ import java.util.function.Supplier; import static java.util.Collections.emptyMap; -import static java.util.Collections.unmodifiableMap; import static org.elasticsearch.core.Strings.format; public class IndexService extends AbstractIndexComponent implements IndicesClusterStateService.AllocatedIndex { @@ -118,7 +117,7 @@ public class IndexService extends AbstractIndexComponent implements IndicesClust private final SimilarityService similarityService; private final EngineFactory engineFactory; private final IndexWarmer warmer; - private volatile Map shards = emptyMap(); + private volatile Map shards = Map.of(); private final AtomicBoolean closed = new AtomicBoolean(false); private final AtomicBoolean deleted = new AtomicBoolean(false); private final IndexSettings indexSettings; @@ -523,16 +522,13 @@ public synchronized IndexShard createShard( @Override public synchronized void removeShard(int shardId, String reason) { - final ShardId sId = new ShardId(index(), shardId); - final IndexShard indexShard; - if (shards.containsKey(shardId) == false) { + final IndexShard indexShard = shards.get(shardId); + if (indexShard == null) { return; } logger.debug("[{}] closing... (reason: [{}])", shardId, reason); - HashMap newShards = new HashMap<>(shards); - indexShard = newShards.remove(shardId); - shards = unmodifiableMap(newShards); - closeShard(reason, sId, indexShard, indexShard.store(), indexShard.getIndexEventListener()); + shards = Maps.copyMapWithRemovedEntry(shards, shardId); + closeShard(reason, indexShard.shardId(), indexShard, indexShard.store(), indexShard.getIndexEventListener()); logger.debug("[{}] closed (reason: [{}])", shardId, reason); } diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestCtxMap.java b/server/src/main/java/org/elasticsearch/ingest/IngestCtxMap.java new file mode 100644 index 0000000000000..dc5bf06697c51 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/ingest/IngestCtxMap.java @@ -0,0 +1,145 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.ingest; + +import org.elasticsearch.common.util.Maps; +import org.elasticsearch.index.VersionType; +import org.elasticsearch.script.CtxMap; +import org.elasticsearch.script.Metadata; + +import java.time.ZonedDateTime; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * Map containing ingest source and metadata. + * + * The Metadata values in {@link IngestDocument.Metadata} are validated when put in the map. + * _index, _id and _routing must be a String or null + * _version_type must be a lower case VersionType or null + * _version must be representable as a long without loss of precision or null + * _dyanmic_templates must be a map + * _if_seq_no must be a long or null + * _if_primary_term must be a long or null + * + * The map is expected to be used by processors, server code should the typed getter and setters where possible. + */ +class IngestCtxMap extends CtxMap { + + /** + * Create an IngestCtxMap with the given metadata, source and default validators + */ + IngestCtxMap( + String index, + String id, + long version, + String routing, + VersionType versionType, + ZonedDateTime timestamp, + Map source + ) { + super(new HashMap<>(source), new IngestMetadata(index, id, version, routing, versionType, timestamp)); + } + + /** + * Create IngestCtxMap from a source and metadata + * + * @param source the source document map + * @param metadata the metadata map + */ + IngestCtxMap(Map source, Metadata metadata) { + super(source, metadata); + } + + /** + * Fetch the timestamp from the ingestMetadata, if it exists + * @return the timestamp for the document or null + */ + public static ZonedDateTime getTimestamp(Map ingestMetadata) { + if (ingestMetadata == null) { + return null; + } + Object ts = ingestMetadata.get(IngestDocument.TIMESTAMP); + if (ts instanceof ZonedDateTime timestamp) { + return timestamp; + } else if (ts instanceof String str) { + return ZonedDateTime.parse(str); + } + return null; + } + + static class IngestMetadata extends Metadata { + private static final FieldProperty UPDATABLE_STRING = new FieldProperty<>(String.class, true, true, null); + static final Map> PROPERTIES = Map.of( + INDEX, + UPDATABLE_STRING, + ID, + UPDATABLE_STRING, + ROUTING, + UPDATABLE_STRING, + VERSION_TYPE, + new FieldProperty<>(String.class, true, true, (k, v) -> { + try { + VersionType.fromString(v); + return; + } catch (IllegalArgumentException ignored) {} + throw new IllegalArgumentException( + k + + " must be a null or one of [" + + Arrays.stream(VersionType.values()).map(vt -> VersionType.toString(vt)).collect(Collectors.joining(", ")) + + "] but was [" + + v + + "] with type [" + + v.getClass().getName() + + "]" + ); + }), + VERSION, + new FieldProperty<>(Number.class, false, true, FieldProperty.LONGABLE_NUMBER), + TYPE, + new FieldProperty<>(String.class, false, false, null), + IF_SEQ_NO, + new FieldProperty<>(Number.class, true, true, FieldProperty.LONGABLE_NUMBER), + IF_PRIMARY_TERM, + new FieldProperty<>(Number.class, true, true, FieldProperty.LONGABLE_NUMBER), + DYNAMIC_TEMPLATES, + new FieldProperty<>(Map.class, true, true, null) + ); + + protected final ZonedDateTime timestamp; + + IngestMetadata(String index, String id, long version, String routing, VersionType versionType, ZonedDateTime timestamp) { + this(metadataMap(index, id, version, routing, versionType), timestamp); + } + + IngestMetadata(Map metadata, ZonedDateTime timestamp) { + super(metadata, PROPERTIES); + this.timestamp = timestamp; + } + + /** + * Create the backing metadata map with the standard contents assuming default validators. + */ + protected static Map metadataMap(String index, String id, long version, String routing, VersionType versionType) { + Map metadata = Maps.newHashMapWithExpectedSize(IngestDocument.Metadata.values().length); + metadata.put(IngestDocument.Metadata.INDEX.getFieldName(), index); + metadata.put(IngestDocument.Metadata.ID.getFieldName(), id); + metadata.put(IngestDocument.Metadata.VERSION.getFieldName(), version); + if (routing != null) { + metadata.put(IngestDocument.Metadata.ROUTING.getFieldName(), routing); + } + if (versionType != null) { + metadata.put(IngestDocument.Metadata.VERSION_TYPE.getFieldName(), VersionType.toString(versionType)); + } + return metadata; + } + } +} diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java b/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java index a77d5d57b3170..81ecfa39bccd0 100644 --- a/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java +++ b/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java @@ -49,7 +49,7 @@ public final class IngestDocument { static final String TIMESTAMP = "timestamp"; - private final IngestSourceAndMetadata sourceAndMetadata; + private final IngestCtxMap sourceAndMetadata; private final Map ingestMetadata; // Contains all pipelines that have been executed for this document @@ -58,15 +58,7 @@ public final class IngestDocument { private boolean doNoSelfReferencesCheck = false; public IngestDocument(String index, String id, long version, String routing, VersionType versionType, Map source) { - this.sourceAndMetadata = new IngestSourceAndMetadata( - index, - id, - version, - routing, - versionType, - ZonedDateTime.now(ZoneOffset.UTC), - source - ); + this.sourceAndMetadata = new IngestCtxMap(index, id, version, routing, versionType, ZonedDateTime.now(ZoneOffset.UTC), source); this.ingestMetadata = new HashMap<>(); this.ingestMetadata.put(TIMESTAMP, sourceAndMetadata.getMetadata().getTimestamp()); } @@ -76,7 +68,7 @@ public IngestDocument(String index, String id, long version, String routing, Ver */ public IngestDocument(IngestDocument other) { this( - new IngestSourceAndMetadata(deepCopyMap(other.sourceAndMetadata.getSource()), other.sourceAndMetadata.getMetadata().clone()), + new IngestCtxMap(deepCopyMap(other.sourceAndMetadata.getSource()), other.sourceAndMetadata.getMetadata().clone()), deepCopyMap(other.ingestMetadata) ); } @@ -85,10 +77,13 @@ public IngestDocument(IngestDocument other) { * Constructor to create an IngestDocument from its constituent maps. The maps are shallow copied. */ public IngestDocument(Map sourceAndMetadata, Map ingestMetadata) { - Tuple, Map> sm = IngestSourceAndMetadata.splitSourceAndMetadata(sourceAndMetadata); - this.sourceAndMetadata = new IngestSourceAndMetadata( + Tuple, Map> sm = IngestCtxMap.splitSourceAndMetadata( + sourceAndMetadata, + Arrays.stream(IngestDocument.Metadata.values()).map(IngestDocument.Metadata::getFieldName).collect(Collectors.toSet()) + ); + this.sourceAndMetadata = new IngestCtxMap( sm.v1(), - new org.elasticsearch.script.Metadata(sm.v2(), IngestSourceAndMetadata.getTimestamp(ingestMetadata)) + new IngestCtxMap.IngestMetadata(sm.v2(), IngestCtxMap.getTimestamp(ingestMetadata)) ); this.ingestMetadata = new HashMap<>(ingestMetadata); this.ingestMetadata.computeIfPresent(TIMESTAMP, (k, v) -> { @@ -102,7 +97,7 @@ public IngestDocument(Map sourceAndMetadata, Map /** * Constructor to create an IngestDocument from its constituent maps */ - IngestDocument(IngestSourceAndMetadata sourceAndMetadata, Map ingestMetadata) { + IngestDocument(IngestCtxMap sourceAndMetadata, Map ingestMetadata) { this.sourceAndMetadata = sourceAndMetadata; this.ingestMetadata = ingestMetadata; } @@ -724,9 +719,9 @@ public Map getSourceAndMetadata() { } /** - * Get source and metadata map as {@link IngestSourceAndMetadata} + * Get source and metadata map as {@link IngestCtxMap} */ - public IngestSourceAndMetadata getIngestSourceAndMetadata() { + public IngestCtxMap getIngestSourceAndMetadata() { return sourceAndMetadata; } @@ -763,7 +758,7 @@ public static Object deepCopy(Object value) { for (Map.Entry entry : mapValue.entrySet()) { copy.put(entry.getKey(), deepCopy(entry.getValue())); } - // TODO(stu): should this check for IngestSourceAndMetadata in addition to Map? + // TODO(stu): should this check for IngestCtxMap in addition to Map? return copy; } else if (value instanceof List listValue) { List copy = new ArrayList<>(listValue.size()); diff --git a/server/src/main/java/org/elasticsearch/plugins/PluginsService.java b/server/src/main/java/org/elasticsearch/plugins/PluginsService.java index 058885de79f81..7e2e13d5343f5 100644 --- a/server/src/main/java/org/elasticsearch/plugins/PluginsService.java +++ b/server/src/main/java/org/elasticsearch/plugins/PluginsService.java @@ -309,7 +309,7 @@ public List loadServiceProviders(Class service) { List result = new ArrayList<>(); for (LoadedPlugin pluginTuple : plugins()) { - ServiceLoader.load(service, pluginTuple.loader()).iterator().forEachRemaining(c -> result.add(c)); + result.addAll(createExtensions(service, pluginTuple.instance)); } return Collections.unmodifiableList(result); @@ -348,11 +348,18 @@ static T createExtension(Class extensionClass, Class extensi throw new IllegalStateException("no public " + extensionConstructorMessage(extensionClass, extensionPointType)); } - if (constructors.length > 1) { + Constructor constructor = constructors[0]; + // Using modules and SPI requires that we declare the default no-arg constructor apart from our custom + // one arg constructor with a plugin. + if (constructors.length == 2) { + // we prefer the one arg constructor in this case + if (constructors[1].getParameterCount() > 0) { + constructor = constructors[1]; + } + } else if (constructors.length > 1) { throw new IllegalStateException("no unique public " + extensionConstructorMessage(extensionClass, extensionPointType)); } - final Constructor constructor = constructors[0]; if (constructor.getParameterCount() > 1) { throw new IllegalStateException(extensionSignatureMessage(extensionClass, extensionPointType, plugin)); } diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestSourceAndMetadata.java b/server/src/main/java/org/elasticsearch/script/CtxMap.java similarity index 76% rename from server/src/main/java/org/elasticsearch/ingest/IngestSourceAndMetadata.java rename to server/src/main/java/org/elasticsearch/script/CtxMap.java index 592c135d46501..d7916e91fc450 100644 --- a/server/src/main/java/org/elasticsearch/ingest/IngestSourceAndMetadata.java +++ b/server/src/main/java/org/elasticsearch/script/CtxMap.java @@ -6,14 +6,11 @@ * Side Public License, v 1. */ -package org.elasticsearch.ingest; +package org.elasticsearch.script; import org.elasticsearch.common.util.Maps; import org.elasticsearch.core.Tuple; -import org.elasticsearch.index.VersionType; -import org.elasticsearch.script.Metadata; -import java.time.ZonedDateTime; import java.util.AbstractCollection; import java.util.AbstractMap; import java.util.AbstractSet; @@ -27,45 +24,21 @@ import java.util.stream.Collectors; /** - * Map containing ingest source and metadata. - * - * The Metadata values in {@link IngestDocument.Metadata} are validated when put in the map. - * _index, _id and _routing must be a String or null - * _version_type must be a lower case VersionType or null - * _version must be representable as a long without loss of precision or null - * _dyanmic_templates must be a map - * _if_seq_no must be a long or null - * _if_primary_term must be a long or null - * - * The map is expected to be used by processors, server code should the typed getter and setters where possible. + * A scripting ctx map with metadata for write ingest contexts. Delegates all metadata updates to metadata and + * all other updates to source. Implements the {@link Map} interface for backwards compatibility while performing + * validation via {@link Metadata}. */ -class IngestSourceAndMetadata extends AbstractMap { - +public class CtxMap extends AbstractMap { protected final Map source; protected final Metadata metadata; /** - * Create an IngestSourceAndMetadata with the given metadata, source and default validators - */ - IngestSourceAndMetadata( - String index, - String id, - long version, - String routing, - VersionType versionType, - ZonedDateTime timestamp, - Map source - ) { - this(new HashMap<>(source), new Metadata(index, id, version, routing, versionType, timestamp)); - } - - /** - * Create IngestSourceAndMetadata from a source and metadata + * Create CtxMap from a source and metadata * * @param source the source document map * @param metadata the metadata map */ - IngestSourceAndMetadata(Map source, Metadata metadata) { + protected CtxMap(Map source, Metadata metadata) { this.source = source != null ? source : new HashMap<>(); this.metadata = metadata; Set badKeys = metadata.metadataKeys(this.source.keySet()); @@ -81,38 +54,25 @@ class IngestSourceAndMetadata extends AbstractMap { /** * Returns a new metadata map and the existing source map with metadata removed. */ - public static Tuple, Map> splitSourceAndMetadata(Map sourceAndMetadata) { - if (sourceAndMetadata instanceof IngestSourceAndMetadata ingestSourceAndMetadata) { - return new Tuple<>(new HashMap<>(ingestSourceAndMetadata.source), new HashMap<>(ingestSourceAndMetadata.metadata.getMap())); + public static Tuple, Map> splitSourceAndMetadata( + Map sourceAndMetadata, + Set metadataKeys + ) { + if (sourceAndMetadata instanceof CtxMap ctxMap) { + return new Tuple<>(new HashMap<>(ctxMap.source), new HashMap<>(ctxMap.metadata.getMap())); } - Map metadata = Maps.newHashMapWithExpectedSize(IngestDocument.Metadata.values().length); + + Map metadata = Maps.newHashMapWithExpectedSize(metadataKeys.size()); Map source = new HashMap<>(sourceAndMetadata); - for (IngestDocument.Metadata ingestDocumentMetadata : IngestDocument.Metadata.values()) { - String metadataName = ingestDocumentMetadata.getFieldName(); - if (sourceAndMetadata.containsKey(metadataName)) { - metadata.put(metadataName, source.remove(metadataName)); + + for (String metadataKey : metadataKeys) { + if (sourceAndMetadata.containsKey(metadataKey)) { + metadata.put(metadataKey, source.remove(metadataKey)); } } return new Tuple<>(source, metadata); } - /** - * Fetch the timestamp from the ingestMetadata, if it exists - * @return the timestamp for the document or null - */ - public static ZonedDateTime getTimestamp(Map ingestMetadata) { - if (ingestMetadata == null) { - return null; - } - Object ts = ingestMetadata.get(IngestDocument.TIMESTAMP); - if (ts instanceof ZonedDateTime timestamp) { - return timestamp; - } else if (ts instanceof String str) { - return ZonedDateTime.parse(str); - } - return null; - } - /** * get the source map, if externally modified then the guarantees of this class are not enforced */ @@ -326,10 +286,10 @@ public Object setValue(Object value) { @Override public boolean equals(Object o) { if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; + if ((o instanceof CtxMap) == false) return false; if (super.equals(o) == false) return false; - IngestSourceAndMetadata that = (IngestSourceAndMetadata) o; - return Objects.equals(source, that.source) && Objects.equals(metadata, that.metadata); + CtxMap ctxMap = (CtxMap) o; + return source.equals(ctxMap.source) && metadata.equals(ctxMap.metadata); } @Override diff --git a/server/src/main/java/org/elasticsearch/script/Metadata.java b/server/src/main/java/org/elasticsearch/script/Metadata.java index 1cb9ce6e9a30f..27ef71906d7d4 100644 --- a/server/src/main/java/org/elasticsearch/script/Metadata.java +++ b/server/src/main/java/org/elasticsearch/script/Metadata.java @@ -8,13 +8,8 @@ package org.elasticsearch.script; -import org.elasticsearch.common.util.Maps; -import org.elasticsearch.index.VersionType; -import org.elasticsearch.ingest.IngestDocument; - import java.time.ZonedDateTime; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -22,6 +17,7 @@ import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.function.BiConsumer; import java.util.stream.Collectors; /** @@ -52,80 +48,31 @@ public class Metadata { protected static final String IF_PRIMARY_TERM = "_if_primary_term"; protected static final String DYNAMIC_TEMPLATES = "_dynamic_templates"; - protected static final Map VALIDATORS = Map.of( - INDEX, - Metadata::stringValidator, - ID, - Metadata::stringValidator, - ROUTING, - Metadata::stringValidator, - VERSION_TYPE, - Metadata::versionTypeValidator, - VERSION, - Metadata::notNullLongValidator, - TYPE, - Metadata::stringValidator, - IF_SEQ_NO, - Metadata::longValidator, - IF_PRIMARY_TERM, - Metadata::longValidator, - DYNAMIC_TEMPLATES, - Metadata::mapValidator - ); - protected final Map map; - protected final Map validators; - - // timestamp is new to ingest metadata, so it doesn't need to be backed by the map for back compat - protected final ZonedDateTime timestamp; - - public Metadata(String index, String id, long version, String routing, VersionType versionType, ZonedDateTime timestamp) { - this(metadataMap(index, id, version, routing, versionType), timestamp, VALIDATORS); - } + protected final Map> properties; + protected static final FieldProperty BAD_KEY = new FieldProperty<>(null, false, false, null); - public Metadata(Map map, ZonedDateTime timestamp) { - this(map, timestamp, VALIDATORS); - } - - Metadata(Map map, ZonedDateTime timestamp, Map validators) { + public Metadata(Map map, Map> properties) { this.map = map; - this.timestamp = timestamp; - this.validators = validators; + this.properties = properties; validateMetadata(); } - /** - * Create the backing metadata map with the standard contents assuming default validators. - */ - protected static Map metadataMap(String index, String id, long version, String routing, VersionType versionType) { - Map metadata = Maps.newHashMapWithExpectedSize(IngestDocument.Metadata.values().length); - metadata.put(IngestDocument.Metadata.INDEX.getFieldName(), index); - metadata.put(IngestDocument.Metadata.ID.getFieldName(), id); - metadata.put(IngestDocument.Metadata.VERSION.getFieldName(), version); - if (routing != null) { - metadata.put(IngestDocument.Metadata.ROUTING.getFieldName(), routing); - } - if (versionType != null) { - metadata.put(IngestDocument.Metadata.VERSION_TYPE.getFieldName(), VersionType.toString(versionType)); - } - return metadata; - } - /** * Check that all metadata map contains only valid metadata and no extraneous keys and source map contains no metadata */ protected void validateMetadata() { int numMetadata = 0; - for (Map.Entry entry : validators.entrySet()) { + for (Map.Entry> entry : properties.entrySet()) { String key = entry.getKey(); if (map.containsKey(key)) { numMetadata++; } - entry.getValue().accept(MapOperation.INIT, key, map.get(key)); + entry.getValue().check(MapOperation.INIT, key, map.get(key)); } if (numMetadata < map.size()) { Set keys = new HashSet<>(map.keySet()); - keys.removeAll(validators.keySet()); + keys.removeAll(properties.keySet()); throw new IllegalArgumentException( "Unexpected metadata keys [" + keys.stream().sorted().map(k -> k + ":" + map.get(k)).collect(Collectors.joining(", ")) + "]" ); @@ -174,7 +121,7 @@ public void setVersion(long version) { } public ZonedDateTime getTimestamp() { - return timestamp; + throw new UnsupportedOperationException("unimplemented"); } // These are not available to scripts @@ -220,7 +167,7 @@ public Number getNumber(String key) { * this call. */ public boolean isMetadata(String key) { - return validators.containsKey(key); + return properties.containsKey(key); } /** @@ -228,8 +175,7 @@ public boolean isMetadata(String key) { * @throws IllegalArgumentException if {@link #isMetadata(String)} is false or the key cannot be updated to the value. */ public Object put(String key, Object value) { - Validator v = validators.getOrDefault(key, this::badKey); - v.accept(MapOperation.UPDATE, key, value); + properties.getOrDefault(key, Metadata.BAD_KEY).check(MapOperation.UPDATE, key, value); return map.put(key, value); } @@ -259,8 +205,7 @@ public Object get(String key) { * @throws IllegalArgumentException if {@link #isMetadata(String)} is false or the key cannot be removed. */ public Object remove(String key) { - Validator v = validators.getOrDefault(key, this::badKey); - v.accept(MapOperation.REMOVE, key, null); + properties.getOrDefault(key, Metadata.BAD_KEY).check(MapOperation.REMOVE, key, null); return map.remove(key); } @@ -280,7 +225,7 @@ public int size() { @Override public Metadata clone() { - return new Metadata(new HashMap<>(map), timestamp, new HashMap<>(validators)); + return new Metadata(new HashMap<>(map), new HashMap<>(properties)); } /** @@ -295,7 +240,7 @@ public Map getMap() { */ public Set metadataKeys(Set other) { Set keys = null; - for (String key : validators.keySet()) { + for (String key : properties.keySet()) { if (other.contains(key)) { if (keys == null) { keys = new HashSet<>(); @@ -306,99 +251,6 @@ public Set metadataKeys(Set other) { return keys != null ? keys : Collections.emptySet(); } - /** - * Allow a String or null. - * @throws IllegalArgumentException if {@param value} is neither a {@link String} nor null - */ - protected static void stringValidator(MapOperation op, String key, Object value) { - if (op == MapOperation.REMOVE || value == null || value instanceof String) { - return; - } - throw new IllegalArgumentException( - key + " must be null or a String but was [" + value + "] with type [" + value.getClass().getName() + "]" - ); - } - - /** - * Allow Numbers that can be represented as longs without loss of precision or null - * @throws IllegalArgumentException if the value cannot be represented as a long - */ - public static void longValidator(MapOperation op, String key, Object value) { - if (op == MapOperation.REMOVE || value == null) { - return; - } - if (value instanceof Number number) { - long version = number.longValue(); - // did we round? - if (number.doubleValue() == version) { - return; - } - } - throw new IllegalArgumentException( - key + " may only be set to an int or a long but was [" + value + "] with type [" + value.getClass().getName() + "]" - ); - } - - /** - * Same as {@link #longValidator(MapOperation, String, Object)} but {@param value} cannot be null. - * @throws IllegalArgumentException if value is null or cannot be represented as a long. - */ - protected static void notNullLongValidator(MapOperation op, String key, Object value) { - if (op == MapOperation.REMOVE || value == null) { - throw new IllegalArgumentException(key + " cannot be removed or set to null"); - } - longValidator(op, key, value); - } - - /** - * Allow maps. - * @throws IllegalArgumentException if {@param value} is not a {@link Map} - */ - public static void mapValidator(MapOperation op, String key, Object value) { - if (op == MapOperation.REMOVE || value == null || value instanceof Map) { - return; - } - throw new IllegalArgumentException( - key + " must be a null or a Map but was [" + value + "] with type [" + value.getClass().getName() + "]" - ); - } - - /** - * Allow lower case Strings that map to VersionType values, or null. - * @throws IllegalArgumentException if {@param value} cannot be converted via {@link VersionType#fromString(String)} - */ - protected static void versionTypeValidator(MapOperation op, String key, Object value) { - if (op == MapOperation.REMOVE || value == null) { - return; - } - if (value instanceof String versionType) { - try { - VersionType.fromString(versionType); - return; - } catch (IllegalArgumentException ignored) {} - } - throw new IllegalArgumentException( - key - + " must be a null or one of [" - + Arrays.stream(VersionType.values()).map(vt -> VersionType.toString(vt)).collect(Collectors.joining(", ")) - + "] but was [" - + value - + "] with type [" - + value.getClass().getName() - + "]" - ); - } - - private void badKey(MapOperation op, String key, Object value) { - throw new IllegalArgumentException( - "unexpected metadata key [" - + key - + "], expected one of [" - + validators.keySet().stream().sorted().collect(Collectors.joining(", ")) - + "]" - ); - } - /** * The operation being performed on the value in the map. * INIT: Initial value - the metadata value as passed into this class @@ -411,26 +263,72 @@ public enum MapOperation { REMOVE } - /** - * A "TriConsumer" that tests if the {@link MapOperation}, the metadata key and value are valid. - * - * throws IllegalArgumentException if the given triple is invalid - */ - @FunctionalInterface - public interface Validator { - void accept(MapOperation op, String key, Object value); - } + public record FieldProperty (Class type, boolean nullable, boolean writable, BiConsumer extendedValidation) { - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - Metadata metadata = (Metadata) o; - return Objects.equals(map, metadata.map) && Objects.equals(timestamp, metadata.timestamp); - } + public static BiConsumer LONGABLE_NUMBER = (k, v) -> { + long version = v.longValue(); + // did we round? + if (v.doubleValue() == version) { + return; + } + throw new IllegalArgumentException( + k + " may only be set to an int or a long but was [" + v + "] with type [" + v.getClass().getName() + "]" + ); + }; + + public static FieldProperty ALLOW_ALL = new FieldProperty<>(null, true, true, null); + + @SuppressWarnings("fallthrough") + public void check(MapOperation op, String key, Object value) { + switch (op) { + case UPDATE: + if (writable == false) { + throw new IllegalArgumentException(key + " cannot be updated"); + } + // fall through + + case INIT: + if (value == null) { + if (nullable == false) { + throw new IllegalArgumentException(key + " cannot be null"); + } + } else { + checkType(key, value); + } + break; + + case REMOVE: + if (writable == false || nullable == false) { + throw new IllegalArgumentException(key + " cannot be removed"); + } + break; + + default: + throw new IllegalArgumentException("unexpected op [" + op + "] for key [" + key + "] and value [" + value + "]"); - @Override - public int hashCode() { - return Objects.hash(map, timestamp); + } + } + + @SuppressWarnings("unchecked") + private void checkType(String key, Object value) { + if (type == null) { + return; + } + if (type.isAssignableFrom(value.getClass()) == false) { + throw new IllegalArgumentException( + key + + " [" + + value + + "] is wrong type, expected assignable to [" + + type.getName() + + "], not [" + + value.getClass().getName() + + "]" + ); + } + if (extendedValidation != null) { + extendedValidation.accept(key, (T) value); + } + } } } diff --git a/server/src/main/java/org/elasticsearch/script/UpdateCtxMap.java b/server/src/main/java/org/elasticsearch/script/UpdateCtxMap.java new file mode 100644 index 0000000000000..a8da53d4eb247 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/script/UpdateCtxMap.java @@ -0,0 +1,144 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.script; + +import org.elasticsearch.common.util.Maps; + +import java.util.Map; +import java.util.Set; +import java.util.function.BiConsumer; +import java.util.stream.Collectors; + +/** + * Source and metadata for update (as opposed to insert via upsert) in the Update context. + */ +public class UpdateCtxMap extends CtxMap { + protected static final String OP = "op"; + protected static final String SOURCE = "_source"; + + // AbstractAsyncBulkByScrollAction.OpType uses 'noop' rather than 'none', so unify on 'noop' but allow 'none' in + // the ctx map + protected static final String LEGACY_NOOP_STRING = "none"; + + public UpdateCtxMap( + String index, + String id, + long version, + String routing, + String type, + String op, + long timestamp, + Map source + ) { + super(wrapSource(source), new UpdateMetadata(index, id, version, routing, type, op, timestamp)); + } + + protected UpdateCtxMap(Map source, Metadata metadata) { + super(wrapSource(source), metadata); + } + + protected static Map wrapSource(Map source) { + Map wrapper = Maps.newHashMapWithExpectedSize(1); + wrapper.put(SOURCE, source); + return wrapper; + } + + public String getOp() { + String op = metadata.getString(OP); + if (LEGACY_NOOP_STRING.equals(op)) { + // TODO(stu): UpdateHelper.UpdateOpType.lenientFromString allows anything into the map + return "noop"; + } + return op; + } + + public void setOp(String op) { + if (LEGACY_NOOP_STRING.equals(op)) { + throw new IllegalArgumentException(LEGACY_NOOP_STRING + " is deprecated, use 'noop' instead"); + } + metadata.put(OP, op); + } + + @Override + @SuppressWarnings("unchecked") + public Map getSource() { + Map wrapped = super.getSource(); + Object rawSource = wrapped.get(SOURCE); + if (rawSource instanceof Map map) { + return (Map) map; + } + throw new IllegalArgumentException( + "Expected source to be a map, instead was [" + rawSource + "] with type [" + rawSource.getClass().getCanonicalName() + "]" + ); + } + + public static class UpdateMetadata extends Metadata { + protected static final String TIMESTAMP = "_now"; + protected static final FieldProperty SET_ONCE_STRING = new FieldProperty<>(String.class, true, false, null); + protected static final FieldProperty SET_ONCE_LONG = new FieldProperty<>( + Number.class, + false, + false, + FieldProperty.LONGABLE_NUMBER + ); + + protected static final Set VALID_UPDATE_OPS = Set.of("noop", "index", "delete", LEGACY_NOOP_STRING); + + static final Map> PROPERTIES = Map.of( + INDEX, + SET_ONCE_STRING, + ID, + SET_ONCE_STRING, + VERSION, + SET_ONCE_LONG, + OP, + new FieldProperty<>(String.class, true, true, setValidator(VALID_UPDATE_OPS)), + TIMESTAMP, + SET_ONCE_LONG + ); + + protected static BiConsumer setValidator(Set valid) { + return (k, v) -> { + if (valid.contains(v) == false) { + throw new IllegalArgumentException( + "[" + k + "] must be one of " + valid.stream().sorted().collect(Collectors.joining(", ")) + ", not [" + v + "]" + ); + } + }; + } + + public UpdateMetadata(String index, String id, long version, String routing, String type, String op, long timestamp) { + this(metadataMap(index, id, version, routing, type, op, timestamp), PROPERTIES); + } + + protected UpdateMetadata(Map metadata, Map> properties) { + super(metadata, properties); + } + + protected static Map metadataMap( + String index, + String id, + long version, + String routing, + String type, + String op, + long timestamp + ) { + Map metadata = Maps.newHashMapWithExpectedSize(PROPERTIES.size()); + metadata.put(INDEX, index); + metadata.put(ID, id); + metadata.put(VERSION, version); + metadata.put(ROUTING, routing); + metadata.put(TYPE, type); + metadata.put(OP, op); + metadata.put(TIMESTAMP, timestamp); + return metadata; + } + } +} diff --git a/server/src/main/java/org/elasticsearch/script/UpdateScript.java b/server/src/main/java/org/elasticsearch/script/UpdateScript.java index 578e2fa7f29b1..4784b6f6067c9 100644 --- a/server/src/main/java/org/elasticsearch/script/UpdateScript.java +++ b/server/src/main/java/org/elasticsearch/script/UpdateScript.java @@ -12,7 +12,7 @@ import java.util.Map; /** - * An update script. + * A script used in the update API */ public abstract class UpdateScript { @@ -24,12 +24,11 @@ public abstract class UpdateScript { /** The generic runtime parameters for the script. */ private final Map params; - /** The update context for the script. */ - private final Map ctx; + private final UpdateCtxMap ctxMap; - public UpdateScript(Map params, Map ctx) { + public UpdateScript(Map params, UpdateCtxMap ctxMap) { this.params = params; - this.ctx = ctx; + this.ctxMap = ctxMap; } /** Return the parameters for this script. */ @@ -39,12 +38,25 @@ public Map getParams() { /** Return the update context for this script. */ public Map getCtx() { - return ctx; + return ctxMap; + } + + /** Return the update metadata for this script */ + public Metadata metadata() { + return ctxMap.getMetadata(); + } + + public void setOp(String op) { + ctxMap.setOp(op); + } + + public String getOp() { + return ctxMap.getOp(); } public abstract void execute(); public interface Factory { - UpdateScript newInstance(Map params, Map ctx); + UpdateScript newInstance(Map params, UpdateCtxMap ctxMap); } } diff --git a/server/src/main/java/org/elasticsearch/script/UpsertCtxMap.java b/server/src/main/java/org/elasticsearch/script/UpsertCtxMap.java new file mode 100644 index 0000000000000..4950d2965ad20 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/script/UpsertCtxMap.java @@ -0,0 +1,61 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.script; + +import org.elasticsearch.common.util.Maps; + +import java.util.Map; +import java.util.Set; + +/** + * Metadata for insert via upsert in the Update context + */ +public class UpsertCtxMap extends UpdateCtxMap { + public UpsertCtxMap(String index, String id, String op, long timestamp, Map source) { + super(source, new UpsertMetadata(index, id, op, timestamp)); + } + + static class UpsertMetadata extends UpdateMetadata { + protected static final Set VALID_UPSERT_OPS = Set.of("noop", "create", LEGACY_NOOP_STRING); + + static final Map> PROPERTIES = Map.of( + INDEX, + SET_ONCE_STRING, + ID, + SET_ONCE_STRING, + OP, + new FieldProperty<>(String.class, true, true, setValidator(VALID_UPSERT_OPS)), + TIMESTAMP, + SET_ONCE_LONG + ); + + public UpsertMetadata(String index, String id, String op, long timestamp) { + super(metadataMap(index, id, op, timestamp), PROPERTIES); + } + + protected static Map metadataMap(String index, String id, String op, long timestamp) { + Map metadata = Maps.newHashMapWithExpectedSize(PROPERTIES.size()); + metadata.put(INDEX, index); + metadata.put(ID, id); + metadata.put(OP, op); + metadata.put(TIMESTAMP, timestamp); + return metadata; + } + + @Override + public String getRouting() { + throw new IllegalArgumentException("routing is unavailable for insert"); + } + + @Override + public long getVersion() { + throw new IllegalArgumentException("version is unavailable for insert"); + } + } +} diff --git a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java index c1307e8706833..512fe1766133f 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java @@ -238,7 +238,6 @@ public void restoreSnapshot( final BiConsumer updater ) { try { - // Try and fill in any missing repository UUIDs in case they're needed during the restore final StepListener repositoryUuidRefreshListener = new StepListener<>(); refreshRepositoryUuids(refreshRepositoryUuidOnRestore, repositoriesService, repositoryUuidRefreshListener); @@ -737,6 +736,7 @@ public static RestoreInProgress updateRestoreStateWithDeletedIndices(RestoreInPr entry.uuid(), entry.snapshot(), overallState(RestoreInProgress.State.STARTED, shards), + entry.quiet(), entry.indices(), shards ) @@ -873,7 +873,9 @@ public RestoreInProgress applyChanges(final RestoreInProgress oldRestore) { Map shards = Map.copyOf(shardsBuilder); RestoreInProgress.State newState = overallState(RestoreInProgress.State.STARTED, shards); - builder.add(new RestoreInProgress.Entry(entry.uuid(), entry.snapshot(), newState, entry.indices(), shards)); + builder.add( + new RestoreInProgress.Entry(entry.uuid(), entry.snapshot(), newState, entry.quiet(), entry.indices(), shards) + ); } else { builder.add(entry); } @@ -1051,6 +1053,7 @@ public ClusterState execute(ClusterState currentState) { boolean changed = false; for (RestoreInProgress.Entry entry : currentState.custom(RestoreInProgress.TYPE, RestoreInProgress.EMPTY)) { if (entry.state().completed()) { + logger.log(entry.quiet() ? Level.DEBUG : Level.INFO, "completed restore of snapshot [{}]", entry.snapshot()); changed = true; } else { restoreInProgressBuilder.add(entry); @@ -1307,7 +1310,7 @@ public ClusterState execute(ClusterState currentState) { // that will be opened by the restore if (currentIndexMetadata == null) { // Index doesn't exist - create it and start recovery - // Make sure that the index we are about to create has a validate name + // Make sure that the index we are about to create has a valid name ensureValidIndexName(currentState, snapshotIndexMetadata, renamedIndexName); shardLimitValidator.validateShardLimit(snapshotIndexMetadata.getSettings(), currentState); @@ -1373,6 +1376,7 @@ && isSystemIndex(snapshotIndexMetadata) == false) { restoreUUID, snapshot, overallState(RestoreInProgress.State.INIT, shards), + request.quiet(), List.copyOf(indicesToRestore.keySet()), Map.copyOf(shards) ) @@ -1569,6 +1573,12 @@ public void onFailure(Exception e) { @Override public void clusterStateProcessed(ClusterState oldState, ClusterState newState) { + logger.log( + request.quiet() ? Level.DEBUG : Level.INFO, + "started restore of snapshot [{}] for indices {}", + snapshot, + snapshotInfo.indices() + ); listener.onResponse(new RestoreCompletionResponse(restoreUUID, snapshot, restoreInfo)); } } diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java index c2ab4e808f1dc..2850dbacf73aa 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java @@ -337,7 +337,10 @@ private void snapshot( ActionListener listener ) { try { - final IndexShard indexShard = indicesService.indexServiceSafe(shardId.getIndex()).getShardOrNull(shardId.id()); + if (snapshotStatus.isAborted()) { + throw new AbortedSnapshotException(); + } + final IndexShard indexShard = indicesService.indexServiceSafe(shardId.getIndex()).getShard(shardId.id()); if (indexShard.routingEntry().primary() == false) { throw new IndexShardSnapshotFailedException(shardId, "snapshot should be performed only on primary"); } diff --git a/server/src/main/java/org/elasticsearch/snapshots/package-info.java b/server/src/main/java/org/elasticsearch/snapshots/package-info.java index 2df9a96656d01..3972a9ac02d5f 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/package-info.java +++ b/server/src/main/java/org/elasticsearch/snapshots/package-info.java @@ -43,9 +43,9 @@ * {@code UpdateIndexShardSnapshotStatusRequest}. * *
  • If as a result of the received status update requests, all shards in the cluster state are in a completed state, i.e are marked as - * either {@code SUCCESS}, {@code FAILED} or {@code MISSING}, the {@code SnapshotShardsService} will update the state of the {@code Entry} + * either {@code SUCCESS}, {@code FAILED} or {@code MISSING}, the {@code SnapshotsService} will update the state of the {@code Entry} * itself and mark it as {@code SUCCESS}. At the same time {@link org.elasticsearch.snapshots.SnapshotsService#endSnapshot} is executed, - * writing the metadata necessary to finalize the snapshot in the repository to the repository.
    • + * writing to the repository the metadata necessary to finalize the snapshot in the repository. * *
  • After writing the final metadata to the repository, a cluster state update to remove the snapshot from the cluster state is * submitted and the removal of the snapshot's {@code SnapshotsInProgress.Entry} from the cluster state completes the snapshot process. diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestTests.java index ae730ed4d4a64..922e7e03c7600 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestTests.java @@ -64,6 +64,7 @@ private RestoreSnapshotRequest randomState(RestoreSnapshotRequest instance) { } instance.partial(randomBoolean()); instance.includeAliases(randomBoolean()); + instance.quiet(randomBoolean()); if (randomBoolean()) { Map indexSettings = new HashMap<>(); @@ -127,6 +128,7 @@ protected RestoreSnapshotRequest mutateInstance(RestoreSnapshotRequest instance) public void testSource() throws IOException { RestoreSnapshotRequest original = createTestInstance(); original.snapshotUuid(null); // cannot be set via the REST API + original.quiet(false); // cannot be set via the REST API XContentBuilder builder = original.toXContent(XContentFactory.jsonBuilder(), new ToXContent.MapParams(Collections.emptyMap())); XContentParser parser = XContentType.JSON.xContent() .createParser(NamedXContentRegistry.EMPTY, null, BytesReference.bytes(builder).streamInput()); diff --git a/server/src/test/java/org/elasticsearch/action/update/UpdateRequestTests.java b/server/src/test/java/org/elasticsearch/action/update/UpdateRequestTests.java index 5ee90a2f38684..d05b9c48e5766 100644 --- a/server/src/test/java/org/elasticsearch/action/update/UpdateRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/update/UpdateRequestTests.java @@ -91,19 +91,19 @@ public void setUp() throws Exception { ctx.put("_timestamp", ctx.get("_now")); return null; }); - scripts.put("ctx.op = delete", vars -> { + scripts.put("ctx.op = 'delete'", vars -> { @SuppressWarnings("unchecked") final Map ctx = (Map) vars.get("ctx"); ctx.put("op", "delete"); return null; }); - scripts.put("ctx.op = bad", vars -> { + scripts.put("ctx.op = 'bad'", vars -> { @SuppressWarnings("unchecked") final Map ctx = (Map) vars.get("ctx"); ctx.put("op", "bad"); return null; }); - scripts.put("ctx.op = none", vars -> { + scripts.put("ctx.op = 'none'", vars -> { @SuppressWarnings("unchecked") final Map ctx = (Map) vars.get("ctx"); ctx.put("op", "none"); @@ -381,7 +381,7 @@ public void testIndexTimeout() { public void testDeleteTimeout() { final GetResult getResult = new GetResult("test", "1", 0, 1, 0, true, new BytesArray("{\"f\":\"v\"}"), null, null); - final UpdateRequest updateRequest = new UpdateRequest("test", "1").script(mockInlineScript("ctx.op = delete")) + final UpdateRequest updateRequest = new UpdateRequest("test", "1").script(mockInlineScript("ctx.op = 'delete'")) .timeout(randomTimeValue()); runTimeoutTest(getResult, updateRequest); } @@ -598,7 +598,7 @@ public void testUpdateScript() throws Exception { assertThat(result.updatedSourceAsMap().get("body").toString(), equalTo("foo")); // Now where the script changes the op to "delete" - request = new UpdateRequest("test", "1").script(mockInlineScript("ctx.op = delete")); + request = new UpdateRequest("test", "1").script(mockInlineScript("ctx.op = 'delete'")); result = updateHelper.prepareUpdateScriptRequest(shardId, request, getResult, ESTestCase::randomNonNegativeLong); @@ -608,9 +608,9 @@ public void testUpdateScript() throws Exception { // We treat everything else as a No-op boolean goodNoop = randomBoolean(); if (goodNoop) { - request = new UpdateRequest("test", "1").script(mockInlineScript("ctx.op = none")); + request = new UpdateRequest("test", "1").script(mockInlineScript("ctx.op = 'none'")); } else { - request = new UpdateRequest("test", "1").script(mockInlineScript("ctx.op = bad")); + request = new UpdateRequest("test", "1").script(mockInlineScript("ctx.op = 'bad'")); } result = updateHelper.prepareUpdateScriptRequest(shardId, request, getResult, ESTestCase::randomNonNegativeLong); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexStateServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexStateServiceTests.java index 43cad5e994326..21f41912e9514 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexStateServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexStateServiceTests.java @@ -343,6 +343,7 @@ private static ClusterState addRestoredIndex(final String index, final int numSh "_uuid", snapshot, RestoreInProgress.State.INIT, + false, Collections.singletonList(index), shardsBuilder ); diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitorTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitorTests.java index 08e68db23e8e2..1e3be3f54c205 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitorTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitorTests.java @@ -964,7 +964,7 @@ private static ClusterInfo clusterInfo( Map diskUsages, Map reservedSpace ) { - return new ClusterInfo(diskUsages, null, null, null, null, reservedSpace); + return new ClusterInfo(diskUsages, Map.of(), Map.of(), Map.of(), Map.of(), reservedSpace); } private static DiscoveryNode newFrozenOnlyNode(String nodeId) { diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/RandomAllocationDeciderTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/RandomAllocationDeciderTests.java index 59419dd4aa3d0..76b1aba8a39cf 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/RandomAllocationDeciderTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/RandomAllocationDeciderTests.java @@ -224,7 +224,7 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, Routing } @Override - public Decision canRemain(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { + public Decision canRemain(IndexMetadata indexMetadata, ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { return getRandomDecision(); } diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ThrottlingAllocationTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ThrottlingAllocationTests.java index e616b81bebc8e..4077c878d3c4e 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ThrottlingAllocationTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ThrottlingAllocationTests.java @@ -413,6 +413,7 @@ private ClusterState createRecoveryStateAndInitializeAllocations( restoreUUID, snapshot, RestoreInProgress.State.INIT, + false, new ArrayList<>(snapshotIndices), restoreShards ); diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/AllocationDecidersTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/AllocationDecidersTests.java index 7d62cd4da927b..c8a06783ebba6 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/AllocationDecidersTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/AllocationDecidersTests.java @@ -12,6 +12,7 @@ import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.routing.RecoverySource; import org.elasticsearch.cluster.routing.RoutingNode; @@ -54,7 +55,12 @@ public Decision canRebalance(ShardRouting shardRouting, RoutingAllocation alloca } @Override - public Decision canRemain(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { + public Decision canRemain( + IndexMetadata indexMetadata, + ShardRouting shardRouting, + RoutingNode node, + RoutingAllocation allocation + ) { return Decision.YES; } @@ -79,18 +85,25 @@ public Decision canRebalance(RoutingAllocation allocation) { } })); - ClusterState clusterState = ClusterState.builder(new ClusterName("test")).build(); + IndexMetadata idx = IndexMetadata.builder("idx").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(0).build(); + IndexMetadata testIdx = IndexMetadata.builder("test") + .settings(settings(Version.CURRENT)) + .numberOfShards(1) + .numberOfReplicas(0) + .build(); + ClusterState clusterState = ClusterState.builder(new ClusterName("test")) + .metadata(Metadata.builder().put(idx, false).put(testIdx, false).build()) + .build(); final RoutingAllocation allocation = new RoutingAllocation(deciders, clusterState, null, null, 0L); allocation.setDebugMode(mode); final UnassignedInfo unassignedInfo = new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "_message"); final ShardRouting shardRouting = ShardRouting.newUnassigned( - new ShardId("test", "testUUID", 0), + new ShardId(testIdx.getIndex(), 0), true, RecoverySource.ExistingStoreRecoverySource.INSTANCE, unassignedInfo ); - IndexMetadata idx = IndexMetadata.builder("idx").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(0).build(); RoutingNode routingNode = RoutingNodesHelper.routingNode("testNode", null); verify(deciders.canAllocate(shardRouting, routingNode, allocation), matcher); @@ -130,7 +143,12 @@ public Decision canRebalance(ShardRouting shardRouting, RoutingAllocation alloca } @Override - public Decision canRemain(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { + public Decision canRemain( + IndexMetadata indexMetadata, + ShardRouting shardRouting, + RoutingNode node, + RoutingAllocation allocation + ) { return decisionOne; } @@ -171,7 +189,12 @@ public Decision canRebalance(ShardRouting shardRouting, RoutingAllocation alloca } @Override - public Decision canRemain(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { + public Decision canRemain( + IndexMetadata indexMetadata, + ShardRouting shardRouting, + RoutingNode node, + RoutingAllocation allocation + ) { return decision(allocation); } @@ -208,20 +231,28 @@ private Decision decision(RoutingAllocation allocation) { } })); + IndexMetadata testIdx = IndexMetadata.builder("test") + .settings(settings(Version.CURRENT)) + .numberOfShards(1) + .numberOfReplicas(0) + .build(); + // no debug should just short-circuit to no, no matter what kind of no type return the first decider returns final ShardRouting shardRouting = ShardRouting.newUnassigned( - new ShardId("test", "testUUID", 0), + new ShardId(testIdx.getIndex(), 0), true, RecoverySource.ExistingStoreRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "_message") ); final RoutingNode routingNode = RoutingNodesHelper.routingNode("testNode", null); - final ClusterState clusterState = ClusterState.builder(new ClusterName("test")).build(); final IndexMetadata indexMetadata = IndexMetadata.builder("idx") .settings(settings(Version.CURRENT)) .numberOfShards(1) .numberOfReplicas(0) .build(); + final ClusterState clusterState = ClusterState.builder(new ClusterName("test")) + .metadata(Metadata.builder().put(testIdx, false).put(indexMetadata, false).build()) + .build(); final RoutingAllocation allocation = new RoutingAllocation(allocationDeciders, clusterState, null, null, 0L); assertSame(Decision.NO, allocationDeciders.canAllocate(shardRouting, routingNode, allocation)); diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java index b1244ff4f17e5..8793f6f9c63e5 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java @@ -919,10 +919,15 @@ public void testCanRemainWithShardRelocatingAway() { System.nanoTime() ); routingAllocation.debugDecision(true); - Decision decision = diskThresholdDecider.canRemain(firstRouting, firstRoutingNode, routingAllocation); + Decision decision = diskThresholdDecider.canRemain( + routingAllocation.metadata().getIndexSafe(firstRouting.index()), + firstRouting, + firstRoutingNode, + routingAllocation + ); assertThat(decision.type(), equalTo(Decision.Type.NO)); assertThat( - ((Decision.Single) decision).getExplanation(), + decision.getExplanation(), containsString( "the shard cannot remain on this node because it is above the high watermark cluster setting " + "[cluster.routing.allocation.disk.watermark.high=70%] and there is less than the required [30.0%] free disk on node, " @@ -951,7 +956,12 @@ public void testCanRemainWithShardRelocatingAway() { System.nanoTime() ); routingAllocation.debugDecision(true); - decision = diskThresholdDecider.canRemain(firstRouting, firstRoutingNode, routingAllocation); + decision = diskThresholdDecider.canRemain( + routingAllocation.metadata().getIndexSafe(firstRouting.index()), + firstRouting, + firstRoutingNode, + routingAllocation + ); assertThat(decision.type(), equalTo(Decision.Type.YES)); assertEquals( "there is enough disk on this node for the shard to remain, free: [60b]", @@ -1109,7 +1119,12 @@ public void testWatermarksEnabledForSingleDataNode() { System.nanoTime() ); routingAllocation.debugDecision(true); - Decision decision = diskThresholdDecider.canRemain(startedShard, clusterState.getRoutingNodes().node("data"), routingAllocation); + Decision decision = diskThresholdDecider.canRemain( + routingAllocation.metadata().getIndexSafe(startedShard.index()), + startedShard, + clusterState.getRoutingNodes().node("data"), + routingAllocation + ); assertThat(decision.type(), equalTo(Decision.Type.NO)); assertThat( decision.getExplanation(), @@ -1197,7 +1212,7 @@ public void testDiskThresholdWithSnapshotShardSizes() { Map shards = Map.of(shardId, new RestoreInProgress.ShardRestoreStatus("node1")); final RestoreInProgress.Builder restores = new RestoreInProgress.Builder().add( - new RestoreInProgress.Entry("_restore_uuid", snapshot, RestoreInProgress.State.INIT, List.of("test"), shards) + new RestoreInProgress.Entry("_restore_uuid", snapshot, RestoreInProgress.State.INIT, false, List.of("test"), shards) ); ClusterState clusterState = ClusterState.builder(new ClusterName(getTestName())) @@ -1308,7 +1323,7 @@ static class DevNullClusterInfo extends ClusterInfo { Map shardSizes, Map reservedSpace ) { - super(leastAvailableSpaceUsage, mostAvailableSpaceUsage, shardSizes, null, null, reservedSpace); + super(leastAvailableSpaceUsage, mostAvailableSpaceUsage, shardSizes, Map.of(), Map.of(), reservedSpace); } @Override diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java index 37d94aba96d1a..dd380fb98e725 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java @@ -107,7 +107,7 @@ public void testCanAllocateUsesMaxAvailableSpace() { leastAvailableUsages, mostAvailableUsage, Map.of("[test][0][p]", 10L), // 10 bytes, - null, + Map.of(), Map.of(), Map.of() ); @@ -185,7 +185,7 @@ public void testCannotAllocateDueToLackOfDiskResources() { leastAvailableUsages, mostAvailableUsage, Map.of("[test][0][p]", shardSize), - null, + Map.of(), Map.of(), Map.of() ); @@ -307,7 +307,7 @@ public void testCanRemainUsesLeastAvailableSpace() { leastAvailableUsages, mostAvailableUsage, shardSizes, - null, + Map.of(), shardRoutingMap, Map.of() ); @@ -319,13 +319,13 @@ public void testCanRemainUsesLeastAvailableSpace() { System.nanoTime() ); allocation.debugDecision(true); - Decision decision = decider.canRemain(test_0, RoutingNodesHelper.routingNode("node_0", node_0), allocation); + Decision decision = decider.canRemain(indexMetadata, test_0, RoutingNodesHelper.routingNode("node_0", node_0), allocation); assertEquals(Decision.Type.YES, decision.type()); assertThat( ((Decision.Single) decision).getExplanation(), containsString("there is enough disk on this node for the shard to remain, free: [10b]") ); - decision = decider.canRemain(test_1, RoutingNodesHelper.routingNode("node_1", node_1), allocation); + decision = decider.canRemain(indexMetadata, test_1, RoutingNodesHelper.routingNode("node_1", node_1), allocation); assertEquals(Decision.Type.NO, decision.type()); assertThat( ((Decision.Single) decision).getExplanation(), @@ -336,26 +336,26 @@ public void testCanRemainUsesLeastAvailableSpace() { ) ); try { - decider.canRemain(test_0, RoutingNodesHelper.routingNode("node_1", node_1), allocation); + decider.canRemain(indexMetadata, test_0, RoutingNodesHelper.routingNode("node_1", node_1), allocation); fail("not allocated on this node"); } catch (IllegalArgumentException ex) { // not allocated on that node } try { - decider.canRemain(test_1, RoutingNodesHelper.routingNode("node_0", node_0), allocation); + decider.canRemain(indexMetadata, test_1, RoutingNodesHelper.routingNode("node_0", node_0), allocation); fail("not allocated on this node"); } catch (IllegalArgumentException ex) { // not allocated on that node } - decision = decider.canRemain(test_2, RoutingNodesHelper.routingNode("node_1", node_1), allocation); + decision = decider.canRemain(indexMetadata, test_2, RoutingNodesHelper.routingNode("node_1", node_1), allocation); assertEquals("can stay since allocated on a different path with enough space", Decision.Type.YES, decision.type()); assertThat( ((Decision.Single) decision).getExplanation(), containsString("this shard is not allocated on the most utilized disk and can remain") ); - decision = decider.canRemain(test_2, RoutingNodesHelper.routingNode("node_1", node_1), allocation); + decision = decider.canRemain(indexMetadata, test_2, RoutingNodesHelper.routingNode("node_1", node_1), allocation); assertEquals("can stay since we don't have information about this shard", Decision.Type.YES, decision.type()); assertThat( ((Decision.Single) decision).getExplanation(), @@ -745,7 +745,7 @@ public void testDecidesYesIfWatermarksIgnored() { allFullUsages, allFullUsages, Map.of("[test][0][p]", 10L), - null, + Map.of(), Map.of(), Map.of() ); @@ -762,7 +762,12 @@ public void testDecidesYesIfWatermarksIgnored() { assertThat(decision.type(), equalTo(Decision.Type.YES)); assertThat(decision.getExplanation(), containsString("disk watermarks are ignored on this index")); - decision = decider.canRemain(test_0.initialize(node_0.getId(), null, 0L).moveToStarted(), routingNode, allocation); + decision = decider.canRemain( + metadata.getIndexSafe(test_0.index()), + test_0.initialize(node_0.getId(), null, 0L).moveToStarted(), + routingNode, + allocation + ); assertThat(decision.type(), equalTo(Decision.Type.YES)); assertThat(decision.getExplanation(), containsString("disk watermarks are ignored on this index")); } @@ -815,7 +820,7 @@ public void testCannotForceAllocateOver100PercentUsage() { // bigger than available space final long shardSize = randomIntBetween(1, 10); shardSizes.put("[test][0][p]", shardSize); - ClusterInfo clusterInfo = new ClusterInfo(leastAvailableUsages, mostAvailableUsage, shardSizes, null, Map.of(), Map.of()); + ClusterInfo clusterInfo = new ClusterInfo(leastAvailableUsages, mostAvailableUsage, shardSizes, Map.of(), Map.of(), Map.of()); RoutingAllocation allocation = new RoutingAllocation( new AllocationDeciders(Collections.singleton(decider)), clusterState, diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/NodeReplacementAllocationDeciderTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/NodeReplacementAllocationDeciderTests.java index 173fb7619e5ec..824af0be3e0cc 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/NodeReplacementAllocationDeciderTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/NodeReplacementAllocationDeciderTests.java @@ -95,7 +95,7 @@ public void testNoReplacements() { assertThat(decision.type(), equalTo(Decision.Type.YES)); assertThat(decision.getExplanation(), equalTo(NodeReplacementAllocationDecider.NO_REPLACEMENTS.getExplanation())); - decision = decider.canRemain(shard, routingNode, allocation); + decision = decider.canRemain(null, shard, routingNode, allocation); assertThat(decision.type(), equalTo(Decision.Type.YES)); assertThat(decision.getExplanation(), equalTo(NodeReplacementAllocationDecider.NO_REPLACEMENTS.getExplanation())); } @@ -149,7 +149,7 @@ public void testCannotRemainOnReplacedNode() { RoutingNode routingNode = RoutingNodesHelper.routingNode(NODE_A.getId(), NODE_A, shard); allocation.debugDecision(true); - Decision decision = decider.canRemain(shard, routingNode, allocation); + Decision decision = decider.canRemain(indexMetadata, shard, routingNode, allocation); assertThat(decision.type(), equalTo(Decision.Type.NO)); assertThat( decision.getExplanation(), @@ -158,13 +158,13 @@ public void testCannotRemainOnReplacedNode() { routingNode = RoutingNodesHelper.routingNode(NODE_B.getId(), NODE_B, shard); - decision = decider.canRemain(shard, routingNode, allocation); + decision = decider.canRemain(indexMetadata, shard, routingNode, allocation); assertThat(decision.type(), equalTo(Decision.Type.YES)); assertThat(decision.getExplanation(), equalTo("node [" + NODE_B.getId() + "] is not being replaced")); routingNode = RoutingNodesHelper.routingNode(NODE_C.getId(), NODE_C, shard); - decision = decider.canRemain(shard, routingNode, allocation); + decision = decider.canRemain(indexMetadata, shard, routingNode, allocation); assertThat(decision.type(), equalTo(Decision.Type.YES)); assertThat(decision.getExplanation(), equalTo("node [" + NODE_C.getId() + "] is not being replaced")); } diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/NodeShutdownAllocationDeciderTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/NodeShutdownAllocationDeciderTests.java index f018ef21ac225..ca92761dba516 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/NodeShutdownAllocationDeciderTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/NodeShutdownAllocationDeciderTests.java @@ -117,7 +117,7 @@ public void testShardsCanRemainOnRestartingNode() { RoutingNode routingNode = RoutingNodesHelper.routingNode(DATA_NODE.getId(), DATA_NODE, shard); allocation.debugDecision(true); - Decision decision = decider.canRemain(shard, routingNode, allocation); + Decision decision = decider.canRemain(null, shard, routingNode, allocation); assertThat(decision.type(), equalTo(Decision.Type.YES)); assertThat( decision.getExplanation(), @@ -134,7 +134,7 @@ public void testShardsCannotRemainOnRemovingNode() { RoutingNode routingNode = RoutingNodesHelper.routingNode(DATA_NODE.getId(), DATA_NODE, shard); allocation.debugDecision(true); - Decision decision = decider.canRemain(shard, routingNode, allocation); + Decision decision = decider.canRemain(null, shard, routingNode, allocation); assertThat(decision.type(), equalTo(Decision.Type.NO)); assertThat(decision.getExplanation(), equalTo("node [" + DATA_NODE.getId() + "] is preparing to be removed from the cluster")); } diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/RestoreInProgressAllocationDeciderTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/RestoreInProgressAllocationDeciderTests.java index 6fda78125b8ed..2562d367cb6f1 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/RestoreInProgressAllocationDeciderTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/RestoreInProgressAllocationDeciderTests.java @@ -150,6 +150,7 @@ public void testCanAllocatePrimaryExistingInRestoreInProgress() { recoverySource.restoreUUID(), snapshot, restoreState, + false, singletonList("test"), shards ); diff --git a/server/src/test/java/org/elasticsearch/cluster/serialization/ClusterSerializationTests.java b/server/src/test/java/org/elasticsearch/cluster/serialization/ClusterSerializationTests.java index 1bffc1931d840..f57627649706d 100644 --- a/server/src/test/java/org/elasticsearch/cluster/serialization/ClusterSerializationTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/serialization/ClusterSerializationTests.java @@ -137,6 +137,7 @@ public void testSnapshotDeletionsInProgressSerialization() throws Exception { UUIDs.randomBase64UUID(), new Snapshot("repo2", new SnapshotId("snap2", UUIDs.randomBase64UUID())), RestoreInProgress.State.STARTED, + false, Collections.singletonList("index_name"), Map.of() ) diff --git a/server/src/test/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilderTests.java index 233cd44765214..c0875b88207da 100644 --- a/server/src/test/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilderTests.java @@ -15,6 +15,9 @@ import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.geo.GeoUtils; import org.elasticsearch.geo.GeometryTestUtils; @@ -23,9 +26,15 @@ import org.elasticsearch.index.mapper.GeoPointFieldMapper; import org.elasticsearch.index.mapper.GeoShapeFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.AbstractQueryTestCase; +import org.elasticsearch.test.TestGeoShapeFieldMapperPlugin; +import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; +import java.util.Collection; +import java.util.Collections; import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.instanceOf; @@ -35,10 +44,28 @@ public class GeoBoundingBoxQueryBuilderTests extends AbstractQueryTestCase { /** Randomly generate either NaN or one of the two infinity values. */ private static final Double[] brokenDoubles = { Double.NaN, Double.POSITIVE_INFINITY, Double.NEGATIVE_INFINITY }; + private static final String GEO_SHAPE_FIELD_NAME = "mapped_geo_shape"; + protected static final String GEO_SHAPE_ALIAS_FIELD_NAME = "mapped_geo_shape_alias"; + + @Override + protected void initializeAdditionalMappings(MapperService mapperService) throws IOException { + final XContentBuilder builder = PutMappingRequest.simpleMapping( + GEO_SHAPE_FIELD_NAME, + "type=geo_shape", + GEO_SHAPE_ALIAS_FIELD_NAME, + "type=alias,path=" + GEO_SHAPE_FIELD_NAME + ); + mapperService.merge("_doc", new CompressedXContent(Strings.toString(builder)), MapperService.MergeReason.MAPPING_UPDATE); + } + + @SuppressWarnings("deprecation") // dependencies in server for geo_shape field should be decoupled + protected Collection> getPlugins() { + return Collections.singletonList(TestGeoShapeFieldMapperPlugin.class); + } @Override protected GeoBoundingBoxQueryBuilder doCreateTestQueryBuilder() { - String fieldName = randomFrom(GEO_POINT_FIELD_NAME, GEO_POINT_ALIAS_FIELD_NAME, GEO_SHAPE_FIELD_NAME); + String fieldName = randomFrom(GEO_POINT_FIELD_NAME, GEO_POINT_ALIAS_FIELD_NAME, GEO_SHAPE_FIELD_NAME, GEO_SHAPE_ALIAS_FIELD_NAME); GeoBoundingBoxQueryBuilder builder = new GeoBoundingBoxQueryBuilder(fieldName); // make sure that minX != maxX and minY != maxY after geohash encoding Rectangle box = randomValueOtherThanMany( diff --git a/server/src/test/java/org/elasticsearch/index/query/GeoDistanceQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/GeoDistanceQueryBuilderTests.java index f3c9090012d44..a564b04122537 100644 --- a/server/src/test/java/org/elasticsearch/index/query/GeoDistanceQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/GeoDistanceQueryBuilderTests.java @@ -14,7 +14,10 @@ import org.apache.lucene.search.IndexOrDocValuesQuery; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; +import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; import org.elasticsearch.common.ParsingException; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.geo.GeoDistance; import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.unit.DistanceUnit; @@ -22,9 +25,15 @@ import org.elasticsearch.index.mapper.GeoPointFieldMapper; import org.elasticsearch.index.mapper.GeoShapeFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.AbstractQueryTestCase; +import org.elasticsearch.test.TestGeoShapeFieldMapperPlugin; +import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; +import java.util.Collection; +import java.util.Collections; import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.instanceOf; @@ -33,9 +42,28 @@ @SuppressWarnings("checkstyle:MissingJavadocMethod") public class GeoDistanceQueryBuilderTests extends AbstractQueryTestCase { + private static final String GEO_SHAPE_FIELD_NAME = "mapped_geo_shape"; + protected static final String GEO_SHAPE_ALIAS_FIELD_NAME = "mapped_geo_shape_alias"; + + @Override + protected void initializeAdditionalMappings(MapperService mapperService) throws IOException { + final XContentBuilder builder = PutMappingRequest.simpleMapping( + GEO_SHAPE_FIELD_NAME, + "type=geo_shape", + GEO_SHAPE_ALIAS_FIELD_NAME, + "type=alias,path=" + GEO_SHAPE_FIELD_NAME + ); + mapperService.merge("_doc", new CompressedXContent(Strings.toString(builder)), MapperService.MergeReason.MAPPING_UPDATE); + } + + @SuppressWarnings("deprecation") // dependencies in server for geo_shape field should be decoupled + protected Collection> getPlugins() { + return Collections.singletonList(TestGeoShapeFieldMapperPlugin.class); + } + @Override protected GeoDistanceQueryBuilder doCreateTestQueryBuilder() { - String fieldName = randomFrom(GEO_POINT_FIELD_NAME, GEO_POINT_ALIAS_FIELD_NAME, GEO_SHAPE_FIELD_NAME); + String fieldName = randomFrom(GEO_POINT_FIELD_NAME, GEO_POINT_ALIAS_FIELD_NAME, GEO_SHAPE_FIELD_NAME, GEO_SHAPE_ALIAS_FIELD_NAME); GeoDistanceQueryBuilder qb = new GeoDistanceQueryBuilder(fieldName); String distance = "" + randomDouble(); if (randomBoolean()) { diff --git a/server/src/test/java/org/elasticsearch/index/query/GeoShapeQueryBuilderGeoShapeTests.java b/server/src/test/java/org/elasticsearch/index/query/GeoShapeQueryBuilderGeoShapeTests.java index 430906c6602e1..f359f4600faaf 100644 --- a/server/src/test/java/org/elasticsearch/index/query/GeoShapeQueryBuilderGeoShapeTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/GeoShapeQueryBuilderGeoShapeTests.java @@ -8,13 +8,37 @@ package org.elasticsearch.index.query; import org.elasticsearch.Version; +import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.geo.ShapeRelation; import org.elasticsearch.geo.GeometryTestUtils; import org.elasticsearch.geometry.Geometry; import org.elasticsearch.geometry.ShapeType; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.test.TestGeoShapeFieldMapperPlugin; +import org.elasticsearch.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.Collection; +import java.util.Collections; public class GeoShapeQueryBuilderGeoShapeTests extends GeoShapeQueryBuilderTests { + private static final String GEO_SHAPE_FIELD_NAME = "mapped_geo_shape"; + + @Override + protected void initializeAdditionalMappings(MapperService mapperService) throws IOException { + XContentBuilder builder = PutMappingRequest.simpleMapping(GEO_SHAPE_FIELD_NAME, "type=geo_shape"); + mapperService.merge("_doc", new CompressedXContent(Strings.toString(builder)), MapperService.MergeReason.MAPPING_UPDATE); + } + + @SuppressWarnings("deprecation") // dependencies in server for geo_shape field should be decoupled + protected Collection> getPlugins() { + return Collections.singletonList(TestGeoShapeFieldMapperPlugin.class); + } + protected String fieldName() { return GEO_SHAPE_FIELD_NAME; } diff --git a/server/src/test/java/org/elasticsearch/index/query/TermsQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/TermsQueryBuilderTests.java index 5b553a46aed6c..9cc03612ccbe4 100644 --- a/server/src/test/java/org/elasticsearch/index/query/TermsQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/TermsQueryBuilderTests.java @@ -68,7 +68,6 @@ protected TermsQueryBuilder doCreateTestQueryBuilder() { String fieldName = randomValueOtherThanMany( choice -> choice.equals(GEO_POINT_FIELD_NAME) || choice.equals(GEO_POINT_ALIAS_FIELD_NAME) - || choice.equals(GEO_SHAPE_FIELD_NAME) || choice.equals(INT_RANGE_FIELD_NAME) || choice.equals(DATE_RANGE_FIELD_NAME) || choice.equals(DATE_NANOS_FIELD_NAME), // TODO: needs testing for date_nanos type diff --git a/server/src/test/java/org/elasticsearch/index/query/TermsSetQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/TermsSetQueryBuilderTests.java index 52aafbff72d9a..a9c9ae7f422cf 100644 --- a/server/src/test/java/org/elasticsearch/index/query/TermsSetQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/TermsSetQueryBuilderTests.java @@ -38,7 +38,6 @@ import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptType; import org.elasticsearch.test.AbstractQueryTestCase; -import org.elasticsearch.test.TestGeoShapeFieldMapperPlugin; import org.elasticsearch.test.rest.ObjectPath; import java.io.IOException; @@ -61,7 +60,7 @@ public class TermsSetQueryBuilderTests extends AbstractQueryTestCase> getPlugins() { - return Arrays.asList(CustomScriptPlugin.class, TestGeoShapeFieldMapperPlugin.class); + return Arrays.asList(CustomScriptPlugin.class); } @Override @@ -76,10 +75,7 @@ protected void initializeAdditionalMappings(MapperService mapperService) throws @Override protected TermsSetQueryBuilder doCreateTestQueryBuilder() { - String fieldName = randomValueOtherThanMany( - value -> value.equals(GEO_POINT_FIELD_NAME) || value.equals(GEO_SHAPE_FIELD_NAME), - () -> randomFrom(MAPPED_FIELD_NAMES) - ); + String fieldName = randomValueOtherThanMany(value -> value.equals(GEO_POINT_FIELD_NAME), () -> randomFrom(MAPPED_FIELD_NAMES)); List randomTerms = randomValues(fieldName); TermsSetQueryBuilder queryBuilder = new TermsSetQueryBuilder(TEXT_FIELD_NAME, randomTerms); if (randomBoolean()) { @@ -151,9 +147,7 @@ public TermsSetQueryBuilder mutateInstance(final TermsSetQueryBuilder instance) switch (randomIntBetween(0, 3)) { case 0 -> { - Predicate predicate = s -> s.equals(instance.getFieldName()) == false - && s.equals(GEO_POINT_FIELD_NAME) == false - && s.equals(GEO_SHAPE_FIELD_NAME) == false; + Predicate predicate = s -> s.equals(instance.getFieldName()) == false && s.equals(GEO_POINT_FIELD_NAME) == false; fieldName = randomValueOtherThanMany(predicate, () -> randomFrom(MAPPED_FIELD_NAMES)); values = randomValues(fieldName); } diff --git a/server/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreQueryBuilderTests.java index b779d7c747b19..75f28e8cdbf3c 100644 --- a/server/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreQueryBuilderTests.java @@ -47,7 +47,6 @@ import org.elasticsearch.script.ScriptType; import org.elasticsearch.search.MultiValueMode; import org.elasticsearch.test.AbstractQueryTestCase; -import org.elasticsearch.test.TestGeoShapeFieldMapperPlugin; import org.elasticsearch.xcontent.XContentParseException; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xcontent.XContentType; @@ -90,7 +89,7 @@ public class FunctionScoreQueryBuilderTests extends AbstractQueryTestCase> getPlugins() { - return Arrays.asList(TestPlugin.class, TestGeoShapeFieldMapperPlugin.class); + return Arrays.asList(TestPlugin.class); } @Override diff --git a/server/src/test/java/org/elasticsearch/ingest/IngestSourceAndMetadataTests.java b/server/src/test/java/org/elasticsearch/ingest/IngestCtxMapTests.java similarity index 85% rename from server/src/test/java/org/elasticsearch/ingest/IngestSourceAndMetadataTests.java rename to server/src/test/java/org/elasticsearch/ingest/IngestCtxMapTests.java index fac543fa05864..e5575910ad881 100644 --- a/server/src/test/java/org/elasticsearch/ingest/IngestSourceAndMetadataTests.java +++ b/server/src/test/java/org/elasticsearch/ingest/IngestCtxMapTests.java @@ -10,7 +10,6 @@ import org.elasticsearch.index.VersionType; import org.elasticsearch.script.Metadata; -import org.elasticsearch.script.TestMetadata; import org.elasticsearch.test.ESTestCase; import java.util.HashMap; @@ -21,10 +20,12 @@ import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasEntry; +import static org.hamcrest.Matchers.notNullValue; -public class IngestSourceAndMetadataTests extends ESTestCase { +public class IngestCtxMapTests extends ESTestCase { - IngestSourceAndMetadata map; + IngestCtxMap map; Metadata md; public void testSettersAndGetters() { @@ -37,7 +38,7 @@ public void testSettersAndGetters() { metadata.put("_if_primary_term", 10000); metadata.put("_version_type", "internal"); metadata.put("_dynamic_templates", Map.of("foo", "bar")); - map = new IngestSourceAndMetadata(new HashMap<>(), new Metadata(metadata, null)); + map = new IngestCtxMap(new HashMap<>(), new Metadata(metadata, null)); md = map.getMetadata(); assertEquals("myIndex", md.getIndex()); md.setIndex("myIndex2"); @@ -76,7 +77,7 @@ public String toString() { }); metadata.put("c", null); metadata.put("d", 1234); - map = new IngestSourceAndMetadata(new HashMap<>(), new TestMetadata(metadata, allowAllValidators("a", "b", "c", "d"))); + map = new IngestCtxMap(new HashMap<>(), new TestIngestCtxMetadata(metadata, allowAllValidators("a", "b", "c", "d"))); md = map.getMetadata(); assertNull(md.getString("c")); assertNull(md.getString("no key")); @@ -91,7 +92,7 @@ public void testGetNumber() { metadata.put("b", Double.MAX_VALUE); metadata.put("c", "NaN"); metadata.put("d", null); - map = new IngestSourceAndMetadata(new HashMap<>(), new TestMetadata(metadata, allowAllValidators("a", "b", "c", "d"))); + map = new IngestCtxMap(new HashMap<>(), new TestIngestCtxMetadata(metadata, allowAllValidators("a", "b", "c", "d"))); md = map.getMetadata(); assertEquals(Long.MAX_VALUE, md.getNumber("a")); assertEquals(Double.MAX_VALUE, md.getNumber("b")); @@ -106,7 +107,7 @@ public void testInvalidMetadata() { metadata.put("_version", Double.MAX_VALUE); IllegalArgumentException err = expectThrows( IllegalArgumentException.class, - () -> new IngestSourceAndMetadata(new HashMap<>(), new Metadata(metadata, null)) + () -> new IngestCtxMap(new HashMap<>(), new Metadata(metadata, null)) ); assertThat(err.getMessage(), containsString("_version may only be set to an int or a long but was [")); assertThat(err.getMessage(), containsString("] with type [java.lang.Double]")); @@ -117,7 +118,7 @@ public void testSourceInMetadata() { source.put("_version", 25); IllegalArgumentException err = expectThrows( IllegalArgumentException.class, - () -> new IngestSourceAndMetadata(source, new Metadata(source, null)) + () -> new IngestCtxMap(source, new Metadata(source, null)) ); assertEquals("unexpected metadata [_version:25] in source", err.getMessage()); } @@ -129,7 +130,7 @@ public void testExtraMetadata() { metadata.put("routing", "myRouting"); IllegalArgumentException err = expectThrows( IllegalArgumentException.class, - () -> new IngestSourceAndMetadata(new HashMap<>(), new Metadata(metadata, null)) + () -> new IngestCtxMap(new HashMap<>(), new Metadata(metadata, null)) ); assertEquals("Unexpected metadata keys [routing:myRouting, version:567]", err.getMessage()); } @@ -138,7 +139,7 @@ public void testPutSource() { Map metadata = new HashMap<>(); metadata.put("_version", 123); Map source = new HashMap<>(); - map = new IngestSourceAndMetadata(source, new Metadata(metadata, null)); + map = new IngestCtxMap(source, new Metadata(metadata, null)); } public void testRemove() { @@ -146,11 +147,18 @@ public void testRemove() { String canRemove = "canRemove"; Map metadata = new HashMap<>(); metadata.put(cannotRemove, "value"); - map = new IngestSourceAndMetadata(new HashMap<>(), new TestMetadata(metadata, Map.of(cannotRemove, (o, k, v) -> { - if (v == null) { - throw new IllegalArgumentException(k + " cannot be null or removed"); - } - }, canRemove, (o, k, v) -> {}))); + map = new IngestCtxMap( + new HashMap<>(), + new TestIngestCtxMetadata( + metadata, + Map.of( + cannotRemove, + new Metadata.FieldProperty<>(String.class, false, true, null), + canRemove, + new Metadata.FieldProperty<>(String.class, true, true, null) + ) + ) + ); String msg = "cannotRemove cannot be null or removed"; IllegalArgumentException err = expectThrows(IllegalArgumentException.class, () -> map.remove(cannotRemove)); assertEquals(msg, err.getMessage()); @@ -211,7 +219,7 @@ public void testEntryAndIterator() { source.put("foo", "bar"); source.put("baz", "qux"); source.put("noz", "zon"); - map = new IngestSourceAndMetadata(source, TestMetadata.withNullableVersion(metadata)); + map = new IngestCtxMap(source, TestIngestCtxMetadata.withNullableVersion(metadata)); md = map.getMetadata(); for (Map.Entry entry : map.entrySet()) { @@ -251,7 +259,7 @@ public void testEntryAndIterator() { } public void testContainsValue() { - map = new IngestSourceAndMetadata(Map.of("myField", "fieldValue"), new Metadata(Map.of("_version", 5678), null)); + map = new IngestCtxMap(Map.of("myField", "fieldValue"), new Metadata(Map.of("_version", 5678), null)); assertTrue(map.containsValue(5678)); assertFalse(map.containsValue(5679)); assertTrue(map.containsValue("fieldValue")); @@ -259,7 +267,7 @@ public void testContainsValue() { } public void testValidators() { - map = new IngestSourceAndMetadata("myIndex", "myId", 1234, "myRouting", VersionType.EXTERNAL, null, new HashMap<>()); + map = new IngestCtxMap("myIndex", "myId", 1234, "myRouting", VersionType.EXTERNAL, null, new HashMap<>()); md = map.getMetadata(); IllegalArgumentException err = expectThrows(IllegalArgumentException.class, () -> map.put("_index", 555)); assertEquals("_index must be null or a String but was [555] with type [java.lang.Integer]", err.getMessage()); @@ -322,7 +330,7 @@ public void testValidators() { public void testHandlesAllVersionTypes() { Map mdRawMap = new HashMap<>(); mdRawMap.put("_version", 1234); - map = new IngestSourceAndMetadata(new HashMap<>(), new Metadata(mdRawMap, null)); + map = new IngestCtxMap(new HashMap<>(), new Metadata(mdRawMap, null)); md = map.getMetadata(); assertNull(md.getVersionType()); for (VersionType vt : VersionType.values()) { @@ -364,11 +372,18 @@ public Object setValue(Object value) { } } - private static Map allowAllValidators(String... keys) { - Map validators = new HashMap<>(); + private static Map> allowAllValidators(String... keys) { + Map> validators = new HashMap<>(); for (String key : keys) { - validators.put(key, (o, k, v) -> {}); + validators.put(key, Metadata.FieldProperty.ALLOW_ALL); } return validators; } + + public void testDefaultFieldPropertiesForAllMetadata() { + for (IngestDocument.Metadata m : IngestDocument.Metadata.values()) { + assertThat(IngestCtxMap.IngestMetadata.PROPERTIES, hasEntry(equalTo(m.getFieldName()), notNullValue())); + } + assertEquals(IngestDocument.Metadata.values().length, IngestCtxMap.IngestMetadata.PROPERTIES.size()); + } } diff --git a/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java b/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java index 39d93a0691856..6182e4709a6b8 100644 --- a/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java +++ b/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java @@ -2228,7 +2228,7 @@ private class IngestDocumentMatcher implements ArgumentMatcher { @Override public boolean matches(IngestDocument other) { - // ingest metadata and IngestSourceAndMetadata will not be the same (timestamp differs every time) + // ingest metadata and IngestCtxMap will not be the same (timestamp differs every time) return Objects.equals(ingestDocument.getSource(), other.getSource()) && Objects.equals(ingestDocument.getMetadata().getMap(), other.getMetadata().getMap()); } diff --git a/server/src/test/java/org/elasticsearch/plugins/PluginsServiceTests.java b/server/src/test/java/org/elasticsearch/plugins/PluginsServiceTests.java index ad334a2d42f8a..ac0e967face4f 100644 --- a/server/src/test/java/org/elasticsearch/plugins/PluginsServiceTests.java +++ b/server/src/test/java/org/elasticsearch/plugins/PluginsServiceTests.java @@ -531,6 +531,10 @@ public TestExtension() {} public TestExtension(TestPlugin plugin) { } + + public TestExtension(TestPlugin plugin, String anotherArg) { + + } } IllegalStateException e = expectThrows( IllegalStateException.class, diff --git a/server/src/test/java/org/elasticsearch/script/MetadataTests.java b/server/src/test/java/org/elasticsearch/script/MetadataTests.java deleted file mode 100644 index c241d14d93b1b..0000000000000 --- a/server/src/test/java/org/elasticsearch/script/MetadataTests.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.script; - -import org.elasticsearch.ingest.IngestDocument; -import org.elasticsearch.test.ESTestCase; - -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.hasEntry; -import static org.hamcrest.Matchers.notNullValue; - -public class MetadataTests extends ESTestCase { - public void testDefaultValidatorForAllMetadata() { - for (IngestDocument.Metadata m : IngestDocument.Metadata.values()) { - assertThat(Metadata.VALIDATORS, hasEntry(equalTo(m.getFieldName()), notNullValue())); - } - assertEquals(IngestDocument.Metadata.values().length, Metadata.VALIDATORS.size()); - } -} diff --git a/server/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java b/server/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java index ffe2d3c1bafbf..9c0c11a179115 100644 --- a/server/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java @@ -500,10 +500,12 @@ public void testNegativeSizeErrors() throws IOException { expected = expectThrows(IllegalArgumentException.class, () -> new SearchSourceBuilder().size(-1)); assertEquals("[size] parameter cannot be negative, found [-1]", expected.getMessage()); - String restContent = "{\"size\" : " + randomSize + "}"; + // SearchSourceBuilder.fromXContent treats -1 as not-set + int boundedRandomSize = randomIntBetween(-100000, -2); + String restContent = "{\"size\" : " + boundedRandomSize + "}"; try (XContentParser parser = createParser(JsonXContent.jsonXContent, restContent)) { IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, () -> SearchSourceBuilder.fromXContent(parser)); - assertThat(ex.getMessage(), containsString(Integer.toString(randomSize))); + assertThat(ex.getMessage(), containsString(Integer.toString(boundedRandomSize))); } restContent = "{\"size\" : -1}"; diff --git a/server/src/test/java/org/elasticsearch/search/geo/GeoPointShapeQueryTests.java b/server/src/test/java/org/elasticsearch/search/geo/GeoPointShapeQueryTests.java index 63c651107735b..b468e963a1f83 100644 --- a/server/src/test/java/org/elasticsearch/search/geo/GeoPointShapeQueryTests.java +++ b/server/src/test/java/org/elasticsearch/search/geo/GeoPointShapeQueryTests.java @@ -15,10 +15,14 @@ import org.elasticsearch.geo.GeometryTestUtils; import org.elasticsearch.geometry.Point; import org.elasticsearch.geometry.utils.WellKnownText; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.test.TestGeoShapeFieldMapperPlugin; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentFactory; import java.io.IOException; +import java.util.Collection; +import java.util.Collections; import java.util.Map; import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE; @@ -27,6 +31,12 @@ public class GeoPointShapeQueryTests extends GeoPointShapeQueryTestCase { + @SuppressWarnings("deprecation") + @Override + protected Collection> getPlugins() { + return Collections.singleton(TestGeoShapeFieldMapperPlugin.class); + } + @Override protected void createMapping(String indexName, String fieldName, Settings settings) throws Exception { XContentBuilder xcb = XContentFactory.jsonBuilder() diff --git a/server/src/test/java/org/elasticsearch/search/geo/GeoShapeQueryTests.java b/server/src/test/java/org/elasticsearch/search/geo/GeoShapeQueryTests.java index f2e9bfcc28f8a..50f209b67790c 100644 --- a/server/src/test/java/org/elasticsearch/search/geo/GeoShapeQueryTests.java +++ b/server/src/test/java/org/elasticsearch/search/geo/GeoShapeQueryTests.java @@ -14,10 +14,14 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.geo.GeometryTestUtils; import org.elasticsearch.geometry.MultiPoint; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.test.TestGeoShapeFieldMapperPlugin; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentFactory; import java.io.IOException; +import java.util.Collection; +import java.util.Collections; import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE; import static org.elasticsearch.index.query.QueryBuilders.geoShapeQuery; @@ -25,6 +29,12 @@ public class GeoShapeQueryTests extends GeoShapeQueryTestCase { + @SuppressWarnings("deprecation") + @Override + protected Collection> getPlugins() { + return Collections.singleton(TestGeoShapeFieldMapperPlugin.class); + } + @Override protected void createMapping(String indexName, String fieldName, Settings settings) throws Exception { XContentBuilder xcb = XContentFactory.jsonBuilder() diff --git a/server/src/test/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilderTests.java index 9e87d2009d626..d52cafc8e6857 100644 --- a/server/src/test/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilderTests.java @@ -24,18 +24,14 @@ import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.index.query.TermQueryBuilder; -import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.AbstractBuilderTestCase; import org.elasticsearch.test.AbstractQueryTestCase; -import org.elasticsearch.test.TestGeoShapeFieldMapperPlugin; import org.elasticsearch.test.VersionUtils; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentFactory; import java.io.IOException; import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; import java.util.List; import static org.hamcrest.Matchers.containsString; @@ -46,11 +42,6 @@ public class KnnVectorQueryBuilderTests extends AbstractQueryTestCase> getPlugins() { - return Arrays.asList(TestGeoShapeFieldMapperPlugin.class); - } - @Override protected void initializeAdditionalMappings(MapperService mapperService) throws IOException { XContentBuilder builder = XContentFactory.jsonBuilder() diff --git a/server/src/test/java/org/elasticsearch/snapshots/InternalSnapshotsInfoServiceTests.java b/server/src/test/java/org/elasticsearch/snapshots/InternalSnapshotsInfoServiceTests.java index 45d9f37bd054b..72fb1f17a4b1e 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/InternalSnapshotsInfoServiceTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/InternalSnapshotsInfoServiceTests.java @@ -451,6 +451,7 @@ private ClusterState addUnassignedShards(final ClusterState currentState, String recoverySource.restoreUUID(), recoverySource.snapshot(), RestoreInProgress.State.INIT, + false, Collections.singletonList(indexName), shards ) diff --git a/test/external-modules/delayed-aggs/src/test/java/org/elasticsearch/test/delayedshard/DelayedShardAggregationBuilderTests.java b/test/external-modules/delayed-aggs/src/test/java/org/elasticsearch/test/delayedshard/DelayedShardAggregationBuilderTests.java index 7a442920c0f00..e289bec89e8c7 100644 --- a/test/external-modules/delayed-aggs/src/test/java/org/elasticsearch/test/delayedshard/DelayedShardAggregationBuilderTests.java +++ b/test/external-modules/delayed-aggs/src/test/java/org/elasticsearch/test/delayedshard/DelayedShardAggregationBuilderTests.java @@ -10,7 +10,6 @@ import org.elasticsearch.core.TimeValue; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.search.aggregations.BaseAggregationTestCase; -import org.elasticsearch.test.TestGeoShapeFieldMapperPlugin; import java.util.Arrays; import java.util.Collection; @@ -18,7 +17,7 @@ public class DelayedShardAggregationBuilderTests extends BaseAggregationTestCase { @Override protected Collection> getPlugins() { - return Arrays.asList(DelayedShardAggregationPlugin.class, TestGeoShapeFieldMapperPlugin.class); + return Arrays.asList(DelayedShardAggregationPlugin.class); } @Override diff --git a/test/external-modules/error-query/src/test/java/org/elasticsearch/test/errorquery/ErrorQueryBuilderTests.java b/test/external-modules/error-query/src/test/java/org/elasticsearch/test/errorquery/ErrorQueryBuilderTests.java index c92c2a5f89362..058e66caad4b9 100644 --- a/test/external-modules/error-query/src/test/java/org/elasticsearch/test/errorquery/ErrorQueryBuilderTests.java +++ b/test/external-modules/error-query/src/test/java/org/elasticsearch/test/errorquery/ErrorQueryBuilderTests.java @@ -13,7 +13,6 @@ import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.AbstractQueryTestCase; -import org.elasticsearch.test.TestGeoShapeFieldMapperPlugin; import java.io.IOException; import java.util.ArrayList; @@ -24,7 +23,7 @@ public class ErrorQueryBuilderTests extends AbstractQueryTestCase { @Override protected Collection> getPlugins() { - return Arrays.asList(ErrorQueryPlugin.class, TestGeoShapeFieldMapperPlugin.class); + return Arrays.asList(ErrorQueryPlugin.class); } @Override diff --git a/test/framework/src/main/java/org/elasticsearch/ingest/TestIngestCtxMetadata.java b/test/framework/src/main/java/org/elasticsearch/ingest/TestIngestCtxMetadata.java new file mode 100644 index 0000000000000..2040fa5ee9322 --- /dev/null +++ b/test/framework/src/main/java/org/elasticsearch/ingest/TestIngestCtxMetadata.java @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.ingest; + +import org.elasticsearch.script.Metadata; + +import java.util.HashMap; +import java.util.Map; + +public class TestIngestCtxMetadata extends Metadata { + public TestIngestCtxMetadata(Map map, Map> properties) { + super(map, properties); + } + + public static TestIngestCtxMetadata withNullableVersion(Map map) { + Map> updatedProperties = new HashMap<>(IngestCtxMap.IngestMetadata.PROPERTIES); + updatedProperties.replace(VERSION, new Metadata.FieldProperty<>(Number.class, true, true, FieldProperty.LONGABLE_NUMBER)); + return new TestIngestCtxMetadata(map, updatedProperties); + } +} diff --git a/test/framework/src/main/java/org/elasticsearch/ingest/TestIngestDocument.java b/test/framework/src/main/java/org/elasticsearch/ingest/TestIngestDocument.java index ffd9ca324f63b..a9135b80ae790 100644 --- a/test/framework/src/main/java/org/elasticsearch/ingest/TestIngestDocument.java +++ b/test/framework/src/main/java/org/elasticsearch/ingest/TestIngestDocument.java @@ -12,11 +12,13 @@ import org.elasticsearch.core.Tuple; import org.elasticsearch.index.VersionType; import org.elasticsearch.script.Metadata; -import org.elasticsearch.script.TestMetadata; import org.elasticsearch.test.ESTestCase; +import java.util.Arrays; import java.util.HashMap; import java.util.Map; +import java.util.function.BiConsumer; +import java.util.stream.Collectors; /** * Construct ingest documents for testing purposes @@ -37,8 +39,11 @@ public static IngestDocument withNullableVersion(Map sourceAndMe * _versions. Normally null _version is not allowed, but many tests don't care about that invariant. */ public static IngestDocument ofIngestWithNullableVersion(Map sourceAndMetadata, Map ingestMetadata) { - Tuple, Map> sm = IngestSourceAndMetadata.splitSourceAndMetadata(sourceAndMetadata); - return new IngestDocument(new IngestSourceAndMetadata(sm.v1(), TestMetadata.withNullableVersion(sm.v2())), ingestMetadata); + Tuple, Map> sm = IngestCtxMap.splitSourceAndMetadata( + sourceAndMetadata, + Arrays.stream(IngestDocument.Metadata.values()).map(IngestDocument.Metadata::getFieldName).collect(Collectors.toSet()) + ); + return new IngestDocument(new IngestCtxMap(sm.v1(), TestIngestCtxMetadata.withNullableVersion(sm.v2())), ingestMetadata); } /** @@ -56,8 +61,10 @@ public static IngestDocument withDefaultVersion(Map sourceAndMet * Create an IngestDocument with a metadata map and validators. The metadata map is passed by reference, not copied, so callers * can observe changes to the map directly. */ - public static IngestDocument ofMetadataWithValidator(Map metadata, Map validators) { - return new IngestDocument(new IngestSourceAndMetadata(new HashMap<>(), new TestMetadata(metadata, validators)), new HashMap<>()); + public static IngestDocument ofMetadataWithValidator(Map metadata, Map> validators) { + Map> properties = new HashMap<>(); + validators.forEach((k, v) -> properties.put(k, new Metadata.FieldProperty<>(Object.class, true, true, v))); + return new IngestDocument(new IngestCtxMap(new HashMap<>(), new TestIngestCtxMetadata(metadata, properties)), new HashMap<>()); } /** diff --git a/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java b/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java index 0b9f7f8972620..0e4b950214a46 100644 --- a/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java +++ b/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java @@ -166,7 +166,7 @@ public boolean execute(Map ctx) { }; return context.factoryClazz.cast(factory); } else if (context.instanceClazz.equals(UpdateScript.class)) { - UpdateScript.Factory factory = (parameters, ctx) -> new UpdateScript(parameters, ctx) { + UpdateScript.Factory factory = (parameters, ctx, md) -> new UpdateScript(parameters, ctx, md) { @Override public void execute() { final Map vars = new HashMap<>(); diff --git a/test/framework/src/main/java/org/elasticsearch/script/TestMetadata.java b/test/framework/src/main/java/org/elasticsearch/script/TestMetadata.java deleted file mode 100644 index 87a45ea857549..0000000000000 --- a/test/framework/src/main/java/org/elasticsearch/script/TestMetadata.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.script; - -import java.util.HashMap; -import java.util.Map; - -/** - * An implementation of {@link Metadata} with customizable {@link org.elasticsearch.script.Metadata.Validator}s for use in testing. - */ -public class TestMetadata extends Metadata { - public TestMetadata(Map map, Map validators) { - super(map, null, validators); - } - - public static TestMetadata withNullableVersion(Map map) { - Map updatedValidators = new HashMap<>(VALIDATORS); - updatedValidators.replace(VERSION, Metadata::longValidator); - return new TestMetadata(map, updatedValidators); - } -} diff --git a/test/framework/src/main/java/org/elasticsearch/search/geo/GeoPointShapeQueryTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/geo/GeoPointShapeQueryTestCase.java index fe1b4c6e80602..d21a9f710c3d7 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/geo/GeoPointShapeQueryTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/search/geo/GeoPointShapeQueryTestCase.java @@ -35,15 +35,11 @@ import org.elasticsearch.geometry.utils.WellKnownText; import org.elasticsearch.index.query.GeoShapeQueryBuilder; import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.plugins.Plugin; import org.elasticsearch.search.SearchHits; import org.elasticsearch.test.ESSingleNodeTestCase; -import org.elasticsearch.test.TestGeoShapeFieldMapperPlugin; import org.elasticsearch.xcontent.XContentFactory; import org.elasticsearch.xcontent.XContentType; -import java.util.Collection; -import java.util.Collections; import java.util.List; import java.util.Map; @@ -57,11 +53,6 @@ public abstract class GeoPointShapeQueryTestCase extends ESSingleNodeTestCase { - @Override - protected Collection> getPlugins() { - return Collections.singleton(TestGeoShapeFieldMapperPlugin.class); - } - protected abstract void createMapping(String indexName, String fieldName, Settings settings) throws Exception; protected void createMapping(String indexName, String fieldName) throws Exception { diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java index 660554efc6d2c..1a49a898a69f3 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java @@ -110,7 +110,6 @@ public abstract class AbstractBuilderTestCase extends ESTestCase { protected static final String OBJECT_FIELD_NAME = "mapped_object"; protected static final String GEO_POINT_FIELD_NAME = "mapped_geo_point"; protected static final String GEO_POINT_ALIAS_FIELD_NAME = "mapped_geo_point_alias"; - protected static final String GEO_SHAPE_FIELD_NAME = "mapped_geo_shape"; // we don't include the binary field in the arrays below as it is not searchable protected static final String BINARY_FIELD_NAME = "mapped_binary"; protected static final String[] MAPPED_FIELD_NAMES = new String[] { @@ -125,8 +124,7 @@ public abstract class AbstractBuilderTestCase extends ESTestCase { DATE_RANGE_FIELD_NAME, OBJECT_FIELD_NAME, GEO_POINT_FIELD_NAME, - GEO_POINT_ALIAS_FIELD_NAME, - GEO_SHAPE_FIELD_NAME }; + GEO_POINT_ALIAS_FIELD_NAME }; protected static final String[] MAPPED_LEAF_FIELD_NAMES = new String[] { TEXT_FIELD_NAME, TEXT_ALIAS_FIELD_NAME, @@ -159,9 +157,8 @@ protected static Index getIndex() { return index; } - @SuppressWarnings("deprecation") // dependencies in server for geo_shape field should be decoupled protected Collection> getPlugins() { - return Collections.singletonList(TestGeoShapeFieldMapperPlugin.class); + return Collections.emptyList(); } /** @@ -455,8 +452,6 @@ public void onRemoval(ShardId shardId, Accountable accountable) { "type=geo_point", GEO_POINT_ALIAS_FIELD_NAME, "type=alias,path=" + GEO_POINT_FIELD_NAME, - GEO_SHAPE_FIELD_NAME, - "type=geo_shape", BINARY_FIELD_NAME, "type=binary" ) diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java index 3e45394e70e24..7d52f28d32d6c 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java @@ -2049,11 +2049,6 @@ protected boolean addMockInternalEngine() { return true; } - /** Returns {@code true} iff this test cluster should use a dummy geo_shape field mapper */ - protected boolean addMockGeoShapeFieldMapper() { - return true; - } - /** * Returns a function that allows to wrap / filter all clients that are exposed by the test cluster. This is useful * for debugging or request / response pre and post processing. It also allows to intercept all calls done by the test @@ -2095,9 +2090,6 @@ protected Collection> getMockPlugins() { mocks.add(TestSeedPlugin.class); mocks.add(AssertActionNamePlugin.class); mocks.add(MockScriptService.TestPlugin.class); - if (addMockGeoShapeFieldMapper()) { - mocks.add(TestGeoShapeFieldMapperPlugin.class); - } return Collections.unmodifiableList(mocks); } diff --git a/x-pack/docs/en/security/auditing/event-types.asciidoc b/x-pack/docs/en/security/auditing/event-types.asciidoc index 7cc041ff4ccf8..db4209ec60e9d 100644 --- a/x-pack/docs/en/security/auditing/event-types.asciidoc +++ b/x-pack/docs/en/security/auditing/event-types.asciidoc @@ -231,6 +231,29 @@ event action. ["index-b*"],"privileges":["all"]}],"applications":[],"run_as":[]}]}}} ==== +[[event-change-apikey]] +`change_apikey`:: +Logged when the <> API is +invoked to update the attributes of an existing API key. ++ +You must include the `security_config_change` event type to audit the related +event action. ++ +.Example +[%collapsible%open] +==== +[source,js] +{"type":"audit", "timestamp":"2020-12-31T00:33:52,521+0200", "node.id": +"9clhpgjJRR-iKzOw20xBNQ", "event.type":"security_config_change", "event.action": +"change_apikey", "request.id":"9FteCmovTzWHVI-9Gpa_vQ", "change":{"apikey": +{"id":"zcwN3YEBBmnjw-K-hW5_","role_descriptors":[{"cluster": +["monitor","manage_ilm"],"indices":[{"names":["index-a*"],"privileges": +["read","maintenance"]},{"names":["in*","alias*"],"privileges":["read"], +"field_security":{"grant":["field1*","@timestamp"],"except":["field11"]}}], +"applications":[],"run_as":[]},{"cluster":["all"],"indices":[{"names": +["index-b*"],"privileges":["all"]}],"applications":[],"run_as":[]}]}}} +==== + [[event-delete-privileges]] `delete_privileges`:: Logged when the @@ -535,8 +558,8 @@ In addition, if `event.type` equals <>. + [source,js] ---- -`{"name": , "expiration": , "role_descriptors" []}` +`{"id": , "name": , "expiration": , "role_descriptors" []}` ---- // NOTCONSOLE + The `role_descriptors` objects have the same schema as the `role_descriptor` object that is part of the above `role` config object. +The object for an API key update will differ in that it will not include +a `name` or `expiration`. + `grant` :: An object like: + [source,js] diff --git a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderService.java b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderService.java index 5d0c1b426a3f7..476c6b21df034 100644 --- a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderService.java +++ b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderService.java @@ -225,7 +225,7 @@ static long nodeSizeForDataBelowLowWatermark(long bytes, DiskThresholdSettings t } else { double percentThreshold = thresholdSettings.getFreeDiskThresholdLow(); if (percentThreshold >= 0.0 && percentThreshold < 100.0) { - return (long) (bytes / ((100.0 - percentThreshold) / 100)); + return (long) (100 * bytes / (100 - percentThreshold)); } else { return bytes; } @@ -793,7 +793,14 @@ private static class ExtendedClusterInfo extends ClusterInfo { private final ClusterInfo delegate; private ExtendedClusterInfo(Map extraShardSizes, ClusterInfo info) { - super(info.getNodeLeastAvailableDiskUsages(), info.getNodeMostAvailableDiskUsages(), extraShardSizes, Map.of(), null, null); + super( + info.getNodeLeastAvailableDiskUsages(), + info.getNodeMostAvailableDiskUsages(), + extraShardSizes, + Map.of(), + Map.of(), + Map.of() + ); this.delegate = info; } diff --git a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCalculateCapacityServiceTests.java b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCalculateCapacityServiceTests.java index 0ea41e147cec7..d7d715334a2dc 100644 --- a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCalculateCapacityServiceTests.java +++ b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCalculateCapacityServiceTests.java @@ -260,7 +260,7 @@ public void testContext() { } } state = ClusterState.builder(ClusterName.DEFAULT).nodes(nodes).build(); - info = new ClusterInfo(leastUsages, mostUsages, null, null, null, null); + info = new ClusterInfo(leastUsages, mostUsages, Map.of(), Map.of(), Map.of(), Map.of()); context = new AutoscalingCalculateCapacityService.DefaultAutoscalingDeciderContext( roleNames, state, @@ -306,7 +306,7 @@ public void testContext() { ) ); - info = new ClusterInfo(leastUsages, mostUsages, null, null, null, null); + info = new ClusterInfo(leastUsages, mostUsages, Map.of(), Map.of(), Map.of(), Map.of()); context = new AutoscalingCalculateCapacityService.DefaultAutoscalingDeciderContext( roleNames, state, diff --git a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/storage/FrozenStorageDeciderServiceTests.java b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/storage/FrozenStorageDeciderServiceTests.java index f1a52024dcca2..d88e067777185 100644 --- a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/storage/FrozenStorageDeciderServiceTests.java +++ b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/storage/FrozenStorageDeciderServiceTests.java @@ -109,7 +109,7 @@ public Tuple sizeAndClusterInfo(IndexMetadata indexMetadata) // add irrelevant shards noise for completeness (should not happen IRL). sizes.put(new ShardId(index, i), randomLongBetween(0, Integer.MAX_VALUE)); } - ClusterInfo info = new ClusterInfo(null, null, null, sizes, null, null); + ClusterInfo info = new ClusterInfo(Map.of(), Map.of(), Map.of(), sizes, Map.of(), Map.of()); return Tuple.tuple(totalSize, info); } } diff --git a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderDecisionTests.java b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderDecisionTests.java index c7d81944cdee1..4d13727cd268b 100644 --- a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderDecisionTests.java +++ b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderDecisionTests.java @@ -88,7 +88,7 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, Routing }; private static final AllocationDecider CAN_REMAIN_NO_DECIDER = new AllocationDecider() { @Override - public Decision canRemain(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { + public Decision canRemain(IndexMetadata indexMetadata, ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { return Decision.NO; } }; @@ -116,13 +116,13 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, Routing // say NO with disk label for subject shards private final AllocationDecider mockCanRemainDiskDecider = new AllocationDecider() { @Override - public Decision canRemain(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { + public Decision canRemain(IndexMetadata indexMetadata, ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { if (subjectShards.contains(shardRouting.shardId()) && node.node().getName().startsWith("hot")) return allocation.decision( Decision.NO, DiskThresholdDecider.NAME, "test" ); - return super.canRemain(shardRouting, node, allocation); + return super.canRemain(indexMetadata, shardRouting, node, allocation); } }; diff --git a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderServiceTests.java b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderServiceTests.java index 4483a1bbe9261..353fa8d789c84 100644 --- a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderServiceTests.java +++ b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderServiceTests.java @@ -403,7 +403,7 @@ public void validateSizeOf(ClusterState clusterState, ShardRouting subjectShard, } private ReactiveStorageDeciderService.AllocationState createAllocationState(Map shardSize, ClusterState clusterState) { - ClusterInfo info = new ClusterInfo(null, null, shardSize, null, null, null); + ClusterInfo info = new ClusterInfo(Map.of(), Map.of(), shardSize, Map.of(), Map.of(), Map.of()); ReactiveStorageDeciderService.AllocationState allocationState = new ReactiveStorageDeciderService.AllocationState( clusterState, null, @@ -567,7 +567,7 @@ public void testUnmovableSize() { if (shardsWithSizes.isEmpty() == false) { shardSize.put(shardIdentifier(randomFrom(shardsWithSizes)), ByteSizeUnit.KB.toBytes(minShardSize)); } - ClusterInfo info = new ClusterInfo(diskUsages, diskUsages, shardSize, null, null, null); + ClusterInfo info = new ClusterInfo(diskUsages, diskUsages, shardSize, Map.of(), Map.of(), Map.of()); ReactiveStorageDeciderService.AllocationState allocationState = new ReactiveStorageDeciderService.AllocationState( clusterState, @@ -615,7 +615,12 @@ public void testCanRemainOnlyHighestTierPreference() { AllocationDecider no = new AllocationDecider() { @Override - public Decision canRemain(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { + public Decision canRemain( + IndexMetadata indexMetadata, + ShardRouting shardRouting, + RoutingNode node, + RoutingAllocation allocation + ) { return Decision.NO; } }; diff --git a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CcrRepositoryIT.java b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CcrRepositoryIT.java index 55e0e5a614962..65192908c28ed 100644 --- a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CcrRepositoryIT.java +++ b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CcrRepositoryIT.java @@ -158,7 +158,8 @@ public void testThatRepositoryRecoversEmptyIndexBasedOnLeaderSettings() throws I .renamePattern("^(.*)$") .renameReplacement(followerIndex) .masterNodeTimeout(TimeValue.MAX_VALUE) - .indexSettings(settingsBuilder); + .indexSettings(settingsBuilder) + .quiet(true); PlainActionFuture future = PlainActionFuture.newFuture(); restoreService.restoreSnapshot(restoreRequest, waitForRestore(clusterService, future)); @@ -230,7 +231,8 @@ public void testDocsAreRecovered() throws Exception { .renamePattern("^(.*)$") .renameReplacement(followerIndex) .masterNodeTimeout(new TimeValue(1L, TimeUnit.HOURS)) - .indexSettings(settingsBuilder); + .indexSettings(settingsBuilder) + .quiet(true); PlainActionFuture future = PlainActionFuture.newFuture(); restoreService.restoreSnapshot(restoreRequest, waitForRestore(clusterService, future)); @@ -298,7 +300,8 @@ public void testRateLimitingIsEmployed() throws Exception { .renamePattern("^(.*)$") .renameReplacement(followerIndex) .masterNodeTimeout(TimeValue.MAX_VALUE) - .indexSettings(settingsBuilder); + .indexSettings(settingsBuilder) + .quiet(true); PlainActionFuture future = PlainActionFuture.newFuture(); restoreService.restoreSnapshot(restoreRequest, waitForRestore(clusterService, future)); @@ -364,7 +367,8 @@ public void testIndividualActionsTimeout() throws Exception { .renamePattern("^(.*)$") .renameReplacement(followerIndex) .masterNodeTimeout(new TimeValue(1L, TimeUnit.HOURS)) - .indexSettings(settingsBuilder); + .indexSettings(settingsBuilder) + .quiet(true); try { final RestoreService restoreService = getFollowerCluster().getCurrentMasterNodeInstance(RestoreService.class); @@ -427,7 +431,8 @@ public void testFollowerMappingIsUpdated() throws IOException { .renamePattern("^(.*)$") .renameReplacement(followerIndex) .masterNodeTimeout(new TimeValue(1L, TimeUnit.HOURS)) - .indexSettings(settingsBuilder); + .indexSettings(settingsBuilder) + .quiet(true); List transportServices = new ArrayList<>(); CountDownLatch latch = new CountDownLatch(1); @@ -586,7 +591,8 @@ public void testCcrRepositoryFetchesSnapshotShardSizeFromIndexShardStoreStats() Settings.builder() .put(IndexMetadata.SETTING_INDEX_PROVIDED_NAME, followerIndex) .put(CcrSettings.CCR_FOLLOWING_INDEX_SETTING.getKey(), true) - ); + ) + .quiet(true); restoreService.restoreSnapshot(restoreRequest, PlainActionFuture.newFuture()); waitForRestoreInProgress.get(30L, TimeUnit.SECONDS); diff --git a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CcrRetentionLeaseIT.java b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CcrRetentionLeaseIT.java index a3cd5750f0d39..8d8fc8120c36b 100644 --- a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CcrRetentionLeaseIT.java +++ b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CcrRetentionLeaseIT.java @@ -157,7 +157,8 @@ private RestoreSnapshotRequest setUpRestoreSnapshotRequest( .indicesOptions(indicesOptions) .renamePattern("^(.*)$") .renameReplacement(followerIndex) - .masterNodeTimeout(TimeValue.MAX_VALUE); + .masterNodeTimeout(TimeValue.MAX_VALUE) + .quiet(true); } public void testRetentionLeaseIsTakenAtTheStartOfRecovery() throws Exception { diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutFollowAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutFollowAction.java index a22ac0abc9e03..88301c49c2101 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutFollowAction.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutFollowAction.java @@ -194,7 +194,8 @@ private void createFollowerIndex( .renamePattern("^(.*)$") .renameReplacement(request.getFollowerIndex()) .masterNodeTimeout(request.masterNodeTimeout()) - .indexSettings(overrideSettings); + .indexSettings(overrideSettings) + .quiet(true); final Client clientWithHeaders = CcrLicenseChecker.wrapClient( this.client, diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierAllocationDecider.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierAllocationDecider.java index 0dc1f22a85c46..cf2f17903960e 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierAllocationDecider.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierAllocationDecider.java @@ -41,7 +41,7 @@ private DataTierAllocationDecider() {} @Override public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { - return shouldFilter(shardRouting, node.node(), allocation); + return shouldFilter(allocation.metadata().getIndexSafe(shardRouting.index()), node.node(), allocation); } @Override @@ -50,8 +50,8 @@ public Decision canAllocate(IndexMetadata indexMetadata, RoutingNode node, Routi } @Override - public Decision canRemain(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { - return shouldFilter(shardRouting, node.node(), allocation); + public Decision canRemain(IndexMetadata indexMetadata, ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { + return shouldFilter(indexMetadata, node.node(), allocation); } @Override @@ -59,8 +59,8 @@ public Decision shouldAutoExpandToNode(IndexMetadata indexMetadata, DiscoveryNod return shouldFilter(indexMetadata, node.getRoles(), allocation); } - private Decision shouldFilter(ShardRouting shardRouting, DiscoveryNode node, RoutingAllocation allocation) { - return shouldFilter(allocation.metadata().getIndexSafe(shardRouting.index()), node.getRoles(), allocation); + private Decision shouldFilter(IndexMetadata indexMetadata, DiscoveryNode node, RoutingAllocation allocation) { + return shouldFilter(indexMetadata, node.getRoles(), allocation); } private static Decision shouldFilter(IndexMetadata indexMd, Set roles, RoutingAllocation allocation) { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/RealmDomain.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/RealmDomain.java index 8863953dc844d..53de14b5b68bb 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/RealmDomain.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/RealmDomain.java @@ -14,6 +14,7 @@ import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.xcontent.ToXContentObject; import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentParser; import java.io.IOException; import java.util.List; @@ -48,6 +49,10 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws return builder; } + public static RealmDomain fromXContent(final XContentParser parser) { + return REALM_DOMAIN_PARSER.apply(parser, null); + } + @Override public String toString() { return "RealmDomain{" + "name='" + name + '\'' + ", realms=" + realms + '}'; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierAllocationDeciderTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierAllocationDeciderTests.java index 712053287c683..bdbdfc4f77f56 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierAllocationDeciderTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierAllocationDeciderTests.java @@ -646,7 +646,12 @@ private void assertAllocationDecision(ClusterState state, DiscoveryNode node, De } { - final var decision = DataTierAllocationDecider.INSTANCE.canRemain(shard, routingNode, allocation); + final var decision = DataTierAllocationDecider.INSTANCE.canRemain( + allocation.metadata().getIndexSafe(shard.index()), + shard, + routingNode, + allocation + ); assertThat(routingNode.toString(), decision.type(), equalTo(decisionType)); assertThat(routingNode.toString(), decision.getExplanation(), containsString(explanationMessage)); } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/AuthenticationTestHelper.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/AuthenticationTestHelper.java index 8cb8b684a64ab..1bbcec5d92ce8 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/AuthenticationTestHelper.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/AuthenticationTestHelper.java @@ -90,6 +90,36 @@ public static User randomUser() { ); } + public static User userWithRandomMetadataAndDetails(final String username, final String... roles) { + return new User( + username, + roles, + ESTestCase.randomFrom(ESTestCase.randomAlphaOfLengthBetween(1, 10), null), + // Not a very realistic email address, but we don't validate this nor rely on correct format, so keeping it simple + ESTestCase.randomFrom(ESTestCase.randomAlphaOfLengthBetween(1, 10), null), + randomUserMetadata(), + true + ); + } + + public static Map randomUserMetadata() { + return ESTestCase.randomFrom( + Map.of( + "employee_id", + ESTestCase.randomAlphaOfLength(5), + "number", + 1, + "numbers", + List.of(1, 3, 5), + "extra", + Map.of("favorite pizza", "hawaii", "age", 42) + ), + Map.of(ESTestCase.randomAlphaOfLengthBetween(3, 8), ESTestCase.randomAlphaOfLengthBetween(3, 8)), + Map.of(), + null + ); + } + public static RealmDomain randomDomain(boolean includeInternal) { final Supplier randomRealmTypeSupplier = randomRealmTypeSupplier(includeInternal); final Set domainRealms = new HashSet<>( diff --git a/x-pack/plugin/enrich/build.gradle b/x-pack/plugin/enrich/build.gradle index 0fb22fe48ced8..ea59ebe2a2a92 100644 --- a/x-pack/plugin/enrich/build.gradle +++ b/x-pack/plugin/enrich/build.gradle @@ -14,6 +14,7 @@ dependencies { testImplementation(testArtifact(project(xpackModule('core')))) testImplementation project(path: ':modules:ingest-common') testImplementation project(path: ':modules:lang-mustache') + testImplementation project(path: ':modules:legacy-geo') testImplementation project(xpackModule('spatial')) testImplementation(testArtifact(project(xpackModule('monitoring')))) } diff --git a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/BasicEnrichTests.java b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/BasicEnrichTests.java index 48d97fe6d7b35..43e982cec08f1 100644 --- a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/BasicEnrichTests.java +++ b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/BasicEnrichTests.java @@ -25,7 +25,6 @@ import org.elasticsearch.reindex.ReindexPlugin; import org.elasticsearch.script.mustache.MustachePlugin; import org.elasticsearch.test.ESSingleNodeTestCase; -import org.elasticsearch.test.TestGeoShapeFieldMapperPlugin; import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.core.XPackSettings; import org.elasticsearch.xpack.core.enrich.EnrichPolicy; @@ -33,6 +32,7 @@ import org.elasticsearch.xpack.core.enrich.action.ExecuteEnrichPolicyAction; import org.elasticsearch.xpack.core.enrich.action.ExecuteEnrichPolicyStatus; import org.elasticsearch.xpack.core.enrich.action.PutEnrichPolicyAction; +import org.elasticsearch.xpack.spatial.SpatialPlugin; import java.util.Arrays; import java.util.Collection; @@ -61,7 +61,7 @@ protected Collection> getPlugins() { ReindexPlugin.class, IngestCommonPlugin.class, MustachePlugin.class, - TestGeoShapeFieldMapperPlugin.class + SpatialPlugin.class ); } diff --git a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichPolicyRunnerTests.java b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichPolicyRunnerTests.java index 051ce60e87639..000864758d984 100644 --- a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichPolicyRunnerTests.java +++ b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichPolicyRunnerTests.java @@ -52,7 +52,6 @@ import org.elasticsearch.tasks.TaskId; import org.elasticsearch.tasks.TaskManager; import org.elasticsearch.test.ESSingleNodeTestCase; -import org.elasticsearch.test.TestGeoShapeFieldMapperPlugin; import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xcontent.XContentBuilder; @@ -63,6 +62,7 @@ import org.elasticsearch.xpack.core.enrich.EnrichPolicy; import org.elasticsearch.xpack.core.enrich.action.ExecuteEnrichPolicyStatus; import org.elasticsearch.xpack.enrich.action.EnrichReindexAction; +import org.elasticsearch.xpack.spatial.SpatialPlugin; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -87,7 +87,7 @@ public class EnrichPolicyRunnerTests extends ESSingleNodeTestCase { @Override protected Collection> getPlugins() { - return Arrays.asList(ReindexPlugin.class, IngestCommonPlugin.class, TestGeoShapeFieldMapperPlugin.class, LocalStateEnrich.class); + return Arrays.asList(ReindexPlugin.class, IngestCommonPlugin.class, SpatialPlugin.class, LocalStateEnrich.class); } private static ThreadPool testThreadPool; diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/ILMImmutableStateHandlerProvider.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/ILMImmutableStateHandlerProvider.java index afd2397b8fb6f..fb69802fbf09f 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/ILMImmutableStateHandlerProvider.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/ILMImmutableStateHandlerProvider.java @@ -10,23 +10,24 @@ import org.elasticsearch.immutablestate.ImmutableClusterStateHandler; import org.elasticsearch.immutablestate.ImmutableClusterStateHandlerProvider; -import java.util.Arrays; import java.util.Collection; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; /** * ILM Provider implementation for the {@link ImmutableClusterStateHandlerProvider} service interface */ public class ILMImmutableStateHandlerProvider implements ImmutableClusterStateHandlerProvider { - private static final Set> handlers = ConcurrentHashMap.newKeySet(); + private final IndexLifecycle plugin; - @Override - public Collection> handlers() { - return handlers; + public ILMImmutableStateHandlerProvider() { + throw new IllegalStateException("Provider must be constructed using PluginsService"); + } + + public ILMImmutableStateHandlerProvider(IndexLifecycle plugin) { + this.plugin = plugin; } - public static void registerHandlers(ImmutableClusterStateHandler... stateHandlers) { - handlers.addAll(Arrays.asList(stateHandlers)); + @Override + public Collection> handlers() { + return plugin.immutableClusterStateHandlers(); } } diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycle.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycle.java index b7f824b4c2845..e477442bcc2e4 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycle.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycle.java @@ -27,6 +27,7 @@ import org.elasticsearch.env.Environment; import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.health.HealthIndicatorService; +import org.elasticsearch.immutablestate.ImmutableClusterStateHandler; import org.elasticsearch.index.IndexModule; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.license.XPackLicenseState; @@ -159,6 +160,7 @@ public class IndexLifecycle extends Plugin implements ActionPlugin, HealthPlugin private final SetOnce snapshotHistoryStore = new SetOnce<>(); private final SetOnce ilmHealthIndicatorService = new SetOnce<>(); private final SetOnce slmHealthIndicatorService = new SetOnce<>(); + private final SetOnce immutableLifecycleAction = new SetOnce<>(); private final Settings settings; public IndexLifecycle(Settings settings) { @@ -268,10 +270,8 @@ public Collection createComponents( components.addAll(Arrays.asList(snapshotLifecycleService.get(), snapshotHistoryStore.get(), snapshotRetentionService.get())); ilmHealthIndicatorService.set(new IlmHealthIndicatorService(clusterService)); slmHealthIndicatorService.set(new SlmHealthIndicatorService(clusterService)); + immutableLifecycleAction.set(new ImmutableLifecycleAction(xContentRegistry, client, XPackPlugin.getSharedLicenseState())); - ILMImmutableStateHandlerProvider.registerHandlers( - new ImmutableLifecycleAction(xContentRegistry, client, XPackPlugin.getSharedLicenseState()) - ); return components; } @@ -422,6 +422,10 @@ public List getRestHandlers( return actions; } + List> immutableClusterStateHandlers() { + return List.of(immutableLifecycleAction.get()); + } + @Override public Collection getHealthIndicatorServices() { return List.of(ilmHealthIndicatorService.get(), slmHealthIndicatorService.get()); diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/v2/RollupShardIndexer.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/v2/RollupShardIndexer.java index 68e09e470cdce..a9498c785971f 100644 --- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/v2/RollupShardIndexer.java +++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/v2/RollupShardIndexer.java @@ -127,6 +127,7 @@ class RollupShardIndexer { } public RollupIndexerAction.ShardRollupResponse execute() throws IOException { + long startTime = System.currentTimeMillis(); BulkProcessor bulkProcessor = createBulkProcessor(); try (searcher; bulkProcessor) { // TODO: add cancellations @@ -138,11 +139,12 @@ public RollupIndexerAction.ShardRollupResponse execute() throws IOException { } logger.info( - "Shard {} successfully sent [{}], indexed [{}], failed [{}]", + "Shard [{}] successfully sent [{}], indexed [{}], failed [{}], took [{}]", indexShard.shardId(), numSent.get(), numIndexed.get(), - numFailed.get() + numFailed.get(), + TimeValue.timeValueMillis(System.currentTimeMillis() - startTime) ); if (numIndexed.get() != numSent.get()) { @@ -209,6 +211,7 @@ private class TimeSeriesBucketCollector extends BucketCollector { private long bucketsCreated; private final RollupBucketBuilder rollupBucketBuilder = new RollupBucketBuilder(); long lastTimestamp = Long.MAX_VALUE; + long lastHistoTimestamp = Long.MAX_VALUE; BytesRef lastTsid = null; TimeSeriesBucketCollector(BulkProcessor bulkProcessor) { @@ -232,15 +235,21 @@ public void collect(int docId, long owningBucketOrd) throws IOException { final BytesRef tsid = aggCtx.getTsid(); assert tsid != null : "Document without [" + TimeSeriesIdFieldMapper.NAME + "] field was found."; final long timestamp = aggCtx.getTimestamp(); - final long histoTimestamp = rounding.round(timestamp); - logger.trace( - "Doc: [{}] - _tsid: [{}], @timestamp: [{}}] -> rollup bucket ts: [{}]", - docId, - DocValueFormat.TIME_SERIES_ID.format(tsid), - timestampFormat.format(timestamp), - timestampFormat.format(histoTimestamp) - ); + boolean tsidChanged = tsid.equals(rollupBucketBuilder.tsid()) == false; + if (tsidChanged || timestamp < lastHistoTimestamp) { + lastHistoTimestamp = rounding.round(timestamp); + } + + if (logger.isTraceEnabled()) { + logger.trace( + "Doc: [{}] - _tsid: [{}], @timestamp: [{}}] -> rollup bucket ts: [{}]", + docId, + DocValueFormat.TIME_SERIES_ID.format(tsid), + timestampFormat.format(timestamp), + timestampFormat.format(lastHistoTimestamp) + ); + } /* * Sanity checks to ensure that we receive documents in the correct order @@ -262,7 +271,7 @@ public void collect(int docId, long owningBucketOrd) throws IOException { lastTsid = BytesRef.deepCopyOf(tsid); lastTimestamp = timestamp; - if (tsid.equals(rollupBucketBuilder.tsid()) == false || rollupBucketBuilder.timestamp() != histoTimestamp) { + if (tsidChanged || rollupBucketBuilder.timestamp() != lastHistoTimestamp) { // Flush rollup doc if not empty if (rollupBucketBuilder.isEmpty() == false) { Map doc = rollupBucketBuilder.buildRollupDocument(); @@ -270,7 +279,7 @@ public void collect(int docId, long owningBucketOrd) throws IOException { } // Create new rollup bucket - rollupBucketBuilder.init(tsid, histoTimestamp); + rollupBucketBuilder.init(tsid, lastHistoTimestamp); bucketsCreated++; } @@ -344,11 +353,14 @@ public RollupBucketBuilder init(BytesRef tsid, long timestamp) { this.timestamp = timestamp; this.docCount = 0; this.metricFieldProducers.values().stream().forEach(p -> p.reset()); - logger.trace( - "New bucket for _tsid: [{}], @timestamp: [{}]", - DocValueFormat.TIME_SERIES_ID.format(tsid), - timestampFormat.format(timestamp) - ); + if (logger.isTraceEnabled()) { + logger.trace( + "New bucket for _tsid: [{}], @timestamp: [{}]", + DocValueFormat.TIME_SERIES_ID.format(tsid), + timestampFormat.format(timestamp) + ); + } + return this; } diff --git a/x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/PinnedQueryBuilderTests.java b/x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/PinnedQueryBuilderTests.java index b8880330d3d30..3931af7d770a5 100644 --- a/x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/PinnedQueryBuilderTests.java +++ b/x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/PinnedQueryBuilderTests.java @@ -16,7 +16,6 @@ import org.elasticsearch.index.query.TermQueryBuilder; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.AbstractQueryTestCase; -import org.elasticsearch.test.TestGeoShapeFieldMapperPlugin; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentFactory; import org.elasticsearch.xcontent.XContentParser; @@ -110,7 +109,6 @@ protected void doAssertLuceneQuery(PinnedQueryBuilder queryBuilder, Query query, protected Collection> getPlugins() { List> classpathPlugins = new ArrayList<>(); classpathPlugins.add(SearchBusinessRules.class); - classpathPlugins.add(TestGeoShapeFieldMapperPlugin.class); return classpathPlugins; } diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/TransportMountSearchableSnapshotAction.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/TransportMountSearchableSnapshotAction.java index 1b8bf8c991efc..ae476d6f8539c 100644 --- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/TransportMountSearchableSnapshotAction.java +++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/TransportMountSearchableSnapshotAction.java @@ -272,7 +272,9 @@ protected void masterOperation( // Pass through the master-node timeout .masterNodeTimeout(request.masterNodeTimeout()) // Fail the restore if the snapshot found above is swapped out from under us before the restore happens - .snapshotUuid(snapshotId.getUUID()), + .snapshotUuid(snapshotId.getUUID()) + // Log snapshot restore at the DEBUG log level + .quiet(true), listener ); }, listener::onFailure), threadPool.executor(ThreadPool.Names.SNAPSHOT_META), null); diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/allocation/decider/DedicatedFrozenNodeAllocationDecider.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/allocation/decider/DedicatedFrozenNodeAllocationDecider.java index 58f2af3a0c944..cf9159d3f5af5 100644 --- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/allocation/decider/DedicatedFrozenNodeAllocationDecider.java +++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/allocation/decider/DedicatedFrozenNodeAllocationDecider.java @@ -48,8 +48,8 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, Routing } @Override - public Decision canRemain(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { - return canAllocateToNode(allocation.metadata().getIndexSafe(shardRouting.index()), node.node()); + public Decision canRemain(IndexMetadata indexMetadata, ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { + return canAllocateToNode(indexMetadata, node.node()); } @Override diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/allocation/decider/HasFrozenCacheAllocationDecider.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/allocation/decider/HasFrozenCacheAllocationDecider.java index 666b33512f351..d16f5483aab09 100644 --- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/allocation/decider/HasFrozenCacheAllocationDecider.java +++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/allocation/decider/HasFrozenCacheAllocationDecider.java @@ -60,8 +60,8 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, Routing } @Override - public Decision canRemain(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { - return canAllocateToNode(allocation.metadata().getIndexSafe(shardRouting.index()), node.node()); + public Decision canRemain(IndexMetadata indexMetadata, ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { + return canAllocateToNode(indexMetadata, node.node()); } @Override diff --git a/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java b/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java index c7f89106338cd..516d3b3d7a3a2 100644 --- a/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java +++ b/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java @@ -198,7 +198,7 @@ public void testGrantApiKeyWithOnlyManageOwnApiKeyPrivilegeFails() throws IOExce public void testUpdateApiKey() throws IOException { final var apiKeyName = "my-api-key-name"; - final Map apiKeyMetadata = Map.of("not", "returned"); + final Map apiKeyMetadata = Map.of("not", "returned"); final Map createApiKeyRequestBody = Map.of("name", apiKeyName, "metadata", apiKeyMetadata); final Request createApiKeyRequest = new Request("POST", "_security/api_key"); @@ -215,7 +215,7 @@ public void testUpdateApiKey() throws IOException { assertThat(apiKeyId, not(emptyString())); assertThat(apiKeyEncoded, not(emptyString())); - doTestUpdateApiKey(apiKeyName, apiKeyId, apiKeyEncoded); + doTestUpdateApiKey(apiKeyName, apiKeyId, apiKeyEncoded, apiKeyMetadata); } public void testGrantTargetCanUpdateApiKey() throws IOException { @@ -240,7 +240,7 @@ public void testGrantTargetCanUpdateApiKey() throws IOException { assertThat(apiKeyId, not(emptyString())); assertThat(apiKeyEncoded, not(emptyString())); - doTestUpdateApiKey(apiKeyName, apiKeyId, apiKeyEncoded); + doTestUpdateApiKey(apiKeyName, apiKeyId, apiKeyEncoded, null); } public void testGrantorCannotUpdateApiKeyOfGrantTarget() throws IOException { @@ -283,18 +283,26 @@ private void doTestAuthenticationWithApiKey(final String apiKeyName, final Strin assertThat(authenticate, hasEntry("api_key", Map.of("id", apiKeyId, "name", apiKeyName))); } - private void doTestUpdateApiKey(String apiKeyName, String apiKeyId, String apiKeyEncoded) throws IOException { + private void doTestUpdateApiKey( + final String apiKeyName, + final String apiKeyId, + final String apiKeyEncoded, + final Map oldMetadata + ) throws IOException { final var updateApiKeyRequest = new Request("PUT", "_security/api_key/" + apiKeyId); - final Map expectedApiKeyMetadata = Map.of("not", "returned (changed)", "foo", "bar"); - final Map updateApiKeyRequestBody = Map.of("metadata", expectedApiKeyMetadata); + final boolean updated = randomBoolean(); + final Map expectedApiKeyMetadata = updated ? Map.of("not", "returned (changed)", "foo", "bar") : oldMetadata; + final Map updateApiKeyRequestBody = expectedApiKeyMetadata == null + ? Map.of() + : Map.of("metadata", expectedApiKeyMetadata); updateApiKeyRequest.setJsonEntity(XContentTestUtils.convertToXContent(updateApiKeyRequestBody, XContentType.JSON).utf8ToString()); final Response updateApiKeyResponse = doUpdateUsingRandomAuthMethod(updateApiKeyRequest); assertOK(updateApiKeyResponse); final Map updateApiKeyResponseMap = responseAsMap(updateApiKeyResponse); - assertTrue((Boolean) updateApiKeyResponseMap.get("updated")); - expectMetadata(apiKeyId, expectedApiKeyMetadata); + assertEquals(updated, updateApiKeyResponseMap.get("updated")); + expectMetadata(apiKeyId, expectedApiKeyMetadata == null ? Map.of() : expectedApiKeyMetadata); // validate authentication still works after update doTestAuthenticationWithApiKey(apiKeyName, apiKeyId, apiKeyEncoded); } diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DocumentLevelSecurityTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DocumentLevelSecurityTests.java index 7ddded23b73b1..5beffd910382f 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DocumentLevelSecurityTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DocumentLevelSecurityTests.java @@ -119,12 +119,6 @@ protected Collection> nodePlugins() { ); } - @Override - protected boolean addMockGeoShapeFieldMapper() { - // a test requires the real SpatialPlugin because it utilizes the shape query - return false; - } - @Override protected String configUsers() { final String usersPasswdHashed = new String(getFastStoredHashAlgoForTests().hash(USERS_PASSWD)); diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/FieldLevelSecurityTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/FieldLevelSecurityTests.java index 8311ce48df312..0feb8f4b03add 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/FieldLevelSecurityTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/FieldLevelSecurityTests.java @@ -104,12 +104,6 @@ protected Collection> nodePlugins() { ); } - @Override - protected boolean addMockGeoShapeFieldMapper() { - // a test requires the real SpatialPlugin because it utilizes the shape query - return false; - } - @Override protected String configUsers() { final String usersPasswHashed = new String(getFastStoredHashAlgoForTests().hash(USERS_PASSWD)); diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java index 36d60d6e25fd3..a2624cc1fdca1 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java @@ -28,11 +28,13 @@ import org.elasticsearch.client.RestClient; import org.elasticsearch.client.internal.Client; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; import org.elasticsearch.common.util.set.Sets; +import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.core.TimeValue; import org.elasticsearch.core.Tuple; import org.elasticsearch.rest.RestStatus; @@ -43,6 +45,8 @@ import org.elasticsearch.test.XContentTestUtils; import org.elasticsearch.test.rest.ObjectPath; import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentFactory; import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.core.XPackSettings; import org.elasticsearch.xpack.core.security.action.ClearSecurityCacheAction; @@ -137,6 +141,13 @@ public class ApiKeyIntegTests extends SecurityIntegTestCase { private static final long DELETE_INTERVAL_MILLIS = 100L; private static final int CRYPTO_THREAD_POOL_QUEUE_SIZE = 10; + private static final RoleDescriptor DEFAULT_API_KEY_ROLE_DESCRIPTOR = new RoleDescriptor( + "role", + new String[] { "monitor" }, + null, + null + ); + @Override public Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { return Settings.builder() @@ -1435,6 +1446,7 @@ public void testSecurityIndexStateChangeWillInvalidateApiKeyCaches() throws Exce public void testUpdateApiKey() throws ExecutionException, InterruptedException, IOException { final Tuple> createdApiKey = createApiKey(TEST_USER_NAME, null); final var apiKeyId = createdApiKey.v1().getId(); + final Map oldMetadata = createdApiKey.v2(); final var newRoleDescriptors = randomRoleDescriptors(); final boolean nullRoleDescriptors = newRoleDescriptors == null; // Role descriptor corresponding to SecuritySettingsSource.TEST_ROLE_YML @@ -1449,11 +1461,14 @@ public void testUpdateApiKey() throws ExecutionException, InterruptedException, ); final var request = new UpdateApiKeyRequest(apiKeyId, newRoleDescriptors, ApiKeyTests.randomMetadata()); - final PlainActionFuture listener = new PlainActionFuture<>(); - final UpdateApiKeyResponse response = executeUpdateApiKey(TEST_USER_NAME, request, listener); + final UpdateApiKeyResponse response = executeUpdateApiKey(TEST_USER_NAME, request); assertNotNull(response); - assertTrue(response.isUpdated()); + // In this test, non-null roleDescriptors always result in an update since they either update the role name, or associated + // privileges. As such null descriptors (plus matching or null metadata) is the only way we can get a noop here + final boolean metadataChanged = request.getMetadata() != null && false == request.getMetadata().equals(oldMetadata); + final boolean isUpdated = nullRoleDescriptors == false || metadataChanged; + assertEquals(isUpdated, response.isUpdated()); final PlainActionFuture getListener = new PlainActionFuture<>(); client().filterWithHeader( @@ -1475,31 +1490,37 @@ public void testUpdateApiKey() throws ExecutionException, InterruptedException, final var updatedApiKeyDoc = getApiKeyDocument(apiKeyId); expectMetadataForApiKey(expectedMetadata, updatedApiKeyDoc); expectRoleDescriptorsForApiKey("limited_by_role_descriptors", expectedLimitedByRoleDescriptors, updatedApiKeyDoc); - if (nullRoleDescriptors) { - // Default role descriptor assigned to api key in `createApiKey` - final var expectedRoleDescriptor = new RoleDescriptor("role", new String[] { "monitor" }, null, null); - expectRoleDescriptorsForApiKey("role_descriptors", List.of(expectedRoleDescriptor), updatedApiKeyDoc); - - // Create user action unauthorized because we did not update key role; it only has `monitor` cluster priv - final Map authorizationHeaders = Collections.singletonMap( - "Authorization", - "ApiKey " + getBase64EncodedApiKeyValue(createdApiKey.v1().getId(), createdApiKey.v1().getKey()) - ); + final var expectedRoleDescriptors = nullRoleDescriptors ? List.of(DEFAULT_API_KEY_ROLE_DESCRIPTOR) : newRoleDescriptors; + expectRoleDescriptorsForApiKey("role_descriptors", expectedRoleDescriptors, updatedApiKeyDoc); + final Map expectedCreator = new HashMap<>(); + expectedCreator.put("principal", TEST_USER_NAME); + expectedCreator.put("full_name", null); + expectedCreator.put("email", null); + expectedCreator.put("metadata", Map.of()); + expectedCreator.put("realm_type", "file"); + expectedCreator.put("realm", "file"); + expectCreatorForApiKey(expectedCreator, updatedApiKeyDoc); + + // Check if update resulted in API key role going from `monitor` to `all` cluster privilege and assert that action that requires + // `all` is authorized or denied accordingly + final boolean hasAllClusterPrivilege = expectedRoleDescriptors.stream() + .filter(rd -> Arrays.asList(rd.getClusterPrivileges()).contains("all")) + .toList() + .isEmpty() == false; + final var authorizationHeaders = Collections.singletonMap( + "Authorization", + "ApiKey " + getBase64EncodedApiKeyValue(createdApiKey.v1().getId(), createdApiKey.v1().getKey()) + ); + if (hasAllClusterPrivilege) { + createUserWithRunAsRole(authorizationHeaders); + } else { ExecutionException e = expectThrows(ExecutionException.class, () -> createUserWithRunAsRole(authorizationHeaders)); assertThat(e.getMessage(), containsString("unauthorized")); assertThat(e.getCause(), instanceOf(ElasticsearchSecurityException.class)); - } else { - expectRoleDescriptorsForApiKey("role_descriptors", newRoleDescriptors, updatedApiKeyDoc); - // Create user action authorized because we updated key role to `all` cluster priv - final var authorizationHeaders = Collections.singletonMap( - "Authorization", - "ApiKey " + getBase64EncodedApiKeyValue(createdApiKey.v1().getId(), createdApiKey.v1().getKey()) - ); - createUserWithRunAsRole(authorizationHeaders); } } - public void testUpdateApiKeyAutoUpdatesUserRoles() throws IOException, ExecutionException, InterruptedException { + public void testUpdateApiKeyAutoUpdatesUserFields() throws IOException, ExecutionException, InterruptedException { // Create separate native realm user and role for user role change test final var nativeRealmUser = randomAlphaOfLengthBetween(5, 10); final var nativeRealmRole = randomAlphaOfLengthBetween(5, 10); @@ -1536,13 +1557,39 @@ public void testUpdateApiKeyAutoUpdatesUserRoles() throws IOException, Execution newClusterPrivileges.toArray(new String[0]) ); - // Update API key - final PlainActionFuture listener = new PlainActionFuture<>(); - final UpdateApiKeyResponse response = executeUpdateApiKey(nativeRealmUser, UpdateApiKeyRequest.usingApiKeyId(apiKeyId), listener); + UpdateApiKeyResponse response = executeUpdateApiKey(nativeRealmUser, UpdateApiKeyRequest.usingApiKeyId(apiKeyId)); assertNotNull(response); assertTrue(response.isUpdated()); expectRoleDescriptorsForApiKey("limited_by_role_descriptors", Set.of(roleDescriptorAfterUpdate), getApiKeyDocument(apiKeyId)); + + // Update user role name only + final RoleDescriptor roleDescriptorWithNewName = putRoleWithClusterPrivileges( + randomValueOtherThan(nativeRealmRole, () -> randomAlphaOfLength(10)), + // Keep old privileges + newClusterPrivileges.toArray(new String[0]) + ); + final User updatedUser = AuthenticationTestHelper.userWithRandomMetadataAndDetails( + nativeRealmUser, + roleDescriptorWithNewName.getName() + ); + updateUser(updatedUser); + + // Update API key + response = executeUpdateApiKey(nativeRealmUser, UpdateApiKeyRequest.usingApiKeyId(apiKeyId)); + + assertNotNull(response); + assertTrue(response.isUpdated()); + final Map updatedApiKeyDoc = getApiKeyDocument(apiKeyId); + expectRoleDescriptorsForApiKey("limited_by_role_descriptors", Set.of(roleDescriptorWithNewName), updatedApiKeyDoc); + final Map expectedCreator = new HashMap<>(); + expectedCreator.put("principal", updatedUser.principal()); + expectedCreator.put("full_name", updatedUser.fullName()); + expectedCreator.put("email", updatedUser.email()); + expectedCreator.put("metadata", updatedUser.metadata()); + expectedCreator.put("realm_type", "native"); + expectedCreator.put("realm", "index"); + expectCreatorForApiKey(expectedCreator, updatedApiKeyDoc); } public void testUpdateApiKeyNotFoundScenarios() throws ExecutionException, InterruptedException { @@ -1552,8 +1599,7 @@ public void testUpdateApiKeyNotFoundScenarios() throws ExecutionException, Inter final var request = new UpdateApiKeyRequest(apiKeyId, List.of(expectedRoleDescriptor), ApiKeyTests.randomMetadata()); // Validate can update own API key - final PlainActionFuture listener = new PlainActionFuture<>(); - final UpdateApiKeyResponse response = executeUpdateApiKey(TEST_USER_NAME, request, listener); + final UpdateApiKeyResponse response = executeUpdateApiKey(TEST_USER_NAME, request); assertNotNull(response); assertTrue(response.isUpdated()); @@ -1646,7 +1692,7 @@ public void testInvalidUpdateApiKeyScenarios() throws ExecutionException, Interr } } - public void testUpdateApiKeyAccountsForSecurityDomains() throws ExecutionException, InterruptedException { + public void testUpdateApiKeyAccountsForSecurityDomains() throws ExecutionException, InterruptedException, IOException { final Tuple> createdApiKey = createApiKey(TEST_USER_NAME, null); final var apiKeyId = createdApiKey.v1().getId(); @@ -1679,6 +1725,147 @@ public void testUpdateApiKeyAccountsForSecurityDomains() throws ExecutionExcepti assertNotNull(response); assertTrue(response.isUpdated()); + final Map expectedCreator = new HashMap<>(); + expectedCreator.put("principal", TEST_USER_NAME); + expectedCreator.put("full_name", null); + expectedCreator.put("email", null); + expectedCreator.put("metadata", Map.of()); + expectedCreator.put("realm_type", authenticatingRealm.getType()); + expectedCreator.put("realm", authenticatingRealm.getName()); + final XContentBuilder builder = realmDomain.toXContent(XContentFactory.jsonBuilder(), null); + expectedCreator.put("realm_domain", XContentHelper.convertToMap(BytesReference.bytes(builder), false, XContentType.JSON).v2()); + expectCreatorForApiKey(expectedCreator, getApiKeyDocument(apiKeyId)); + } + + public void testNoopUpdateApiKey() throws ExecutionException, InterruptedException, IOException { + final Tuple> createdApiKey = createApiKey(TEST_USER_NAME, null); + final var apiKeyId = createdApiKey.v1().getId(); + + final var initialRequest = new UpdateApiKeyRequest( + apiKeyId, + List.of(new RoleDescriptor(randomAlphaOfLength(10), new String[] { "all" }, null, null)), + ApiKeyTests.randomMetadata() + ); + UpdateApiKeyResponse response = executeUpdateApiKey(TEST_USER_NAME, initialRequest); + assertNotNull(response); + // First update is not noop, because role descriptors changed and possibly metadata + assertTrue(response.isUpdated()); + + // Update with same request is a noop and does not clear cache + authenticateWithApiKey(apiKeyId, createdApiKey.v1().getKey()); + final var serviceWithNameForDoc1 = Arrays.stream(internalCluster().getNodeNames()) + .map(n -> internalCluster().getInstance(ApiKeyService.class, n)) + .filter(s -> s.getDocCache().get(apiKeyId) != null) + .findFirst() + .orElseThrow(); + final int count = serviceWithNameForDoc1.getDocCache().count(); + response = executeUpdateApiKey(TEST_USER_NAME, initialRequest); + assertNotNull(response); + assertFalse(response.isUpdated()); + assertEquals(count, serviceWithNameForDoc1.getDocCache().count()); + + // Update with empty request is a noop + response = executeUpdateApiKey(TEST_USER_NAME, UpdateApiKeyRequest.usingApiKeyId(apiKeyId)); + assertNotNull(response); + assertFalse(response.isUpdated()); + + // Update with different role descriptors is not a noop + final List newRoleDescriptors = List.of( + randomValueOtherThanMany( + rd -> (RoleDescriptorRequestValidator.validate(rd) != null) && initialRequest.getRoleDescriptors().contains(rd) == false, + () -> RoleDescriptorTests.randomRoleDescriptor(false) + ), + randomValueOtherThanMany( + rd -> (RoleDescriptorRequestValidator.validate(rd) != null) && initialRequest.getRoleDescriptors().contains(rd) == false, + () -> RoleDescriptorTests.randomRoleDescriptor(false) + ) + ); + response = executeUpdateApiKey(TEST_USER_NAME, new UpdateApiKeyRequest(apiKeyId, newRoleDescriptors, null)); + assertNotNull(response); + assertTrue(response.isUpdated()); + + // Update with re-ordered role descriptors is a noop + response = executeUpdateApiKey( + TEST_USER_NAME, + new UpdateApiKeyRequest(apiKeyId, List.of(newRoleDescriptors.get(1), newRoleDescriptors.get(0)), null) + ); + assertNotNull(response); + assertFalse(response.isUpdated()); + + // Update with different metadata is not a noop + response = executeUpdateApiKey( + TEST_USER_NAME, + new UpdateApiKeyRequest( + apiKeyId, + null, + randomValueOtherThanMany(md -> md == null || md.equals(initialRequest.getMetadata()), ApiKeyTests::randomMetadata) + ) + ); + assertNotNull(response); + assertTrue(response.isUpdated()); + + // Update with different creator info is not a noop + // First, ensure that the user role descriptors alone do *not* cause an update, so we can test that we correctly perform the noop + // check when we update creator info + final ServiceWithNodeName serviceWithNodeName = getServiceWithNodeName(); + PlainActionFuture listener = new PlainActionFuture<>(); + // Role descriptor corresponding to SecuritySettingsSource.TEST_ROLE_YML, i.e., should not result in update + final Set oldUserRoleDescriptors = Set.of( + new RoleDescriptor( + TEST_ROLE, + new String[] { "ALL" }, + new RoleDescriptor.IndicesPrivileges[] { + RoleDescriptor.IndicesPrivileges.builder().indices("*").allowRestrictedIndices(true).privileges("ALL").build() }, + null + ) + ); + serviceWithNodeName.service() + .updateApiKey( + Authentication.newRealmAuthentication( + new User(TEST_USER_NAME, TEST_ROLE), + new Authentication.RealmRef("file", "file", serviceWithNodeName.nodeName()) + ), + UpdateApiKeyRequest.usingApiKeyId(apiKeyId), + oldUserRoleDescriptors, + listener + ); + response = listener.get(); + assertNotNull(response); + assertFalse(response.isUpdated()); + final User updatedUser = AuthenticationTestHelper.userWithRandomMetadataAndDetails(TEST_USER_NAME, TEST_ROLE); + final RealmConfig.RealmIdentifier creatorRealmOnCreatedApiKey = new RealmConfig.RealmIdentifier(FileRealmSettings.TYPE, "file"); + final boolean noUserChanges = updatedUser.equals(new User(TEST_USER_NAME, TEST_ROLE)); + final Authentication.RealmRef realmRef; + if (randomBoolean() || noUserChanges) { + final RealmConfig.RealmIdentifier otherRealmInDomain = AuthenticationTestHelper.randomRealmIdentifier(true); + final var realmDomain = new RealmDomain( + ESTestCase.randomAlphaOfLengthBetween(3, 8), + Set.of(creatorRealmOnCreatedApiKey, otherRealmInDomain) + ); + // Using other realm from domain should result in update + realmRef = new Authentication.RealmRef( + otherRealmInDomain.getName(), + otherRealmInDomain.getType(), + serviceWithNodeName.nodeName(), + realmDomain + ); + } else { + realmRef = new Authentication.RealmRef( + creatorRealmOnCreatedApiKey.getName(), + creatorRealmOnCreatedApiKey.getType(), + serviceWithNodeName.nodeName() + ); + } + final var authentication = randomValueOtherThanMany( + Authentication::isApiKey, + () -> AuthenticationTestHelper.builder().user(updatedUser).realmRef(realmRef).build() + ); + listener = new PlainActionFuture<>(); + serviceWithNodeName.service() + .updateApiKey(authentication, UpdateApiKeyRequest.usingApiKeyId(apiKeyId), oldUserRoleDescriptors, listener); + response = listener.get(); + assertNotNull(response); + assertTrue(response.isUpdated()); } public void testUpdateApiKeyClearsApiKeyDocCache() throws IOException, ExecutionException, InterruptedException { @@ -1720,7 +1907,12 @@ public void testUpdateApiKeyClearsApiKeyDocCache() throws IOException, Execution final Client client = client().filterWithHeader( Collections.singletonMap("Authorization", basicAuthHeaderValue(ES_TEST_ROOT_USER, TEST_PASSWORD_SECURE_STRING)) ); - client.execute(UpdateApiKeyAction.INSTANCE, new UpdateApiKeyRequest(apiKey1.v1(), List.of(), null), listener); + client.execute( + UpdateApiKeyAction.INSTANCE, + // Set metadata to ensure update + new UpdateApiKeyRequest(apiKey1.v1(), List.of(), Map.of(randomAlphaOfLength(5), randomAlphaOfLength(10))), + listener + ); final var response = listener.get(); assertNotNull(response); assertTrue(response.isUpdated()); @@ -1741,7 +1933,7 @@ public void testUpdateApiKeyClearsApiKeyDocCache() throws IOException, Execution } private List randomRoleDescriptors() { - int caseNo = randomIntBetween(0, 2); + int caseNo = randomIntBetween(0, 3); return switch (caseNo) { case 0 -> List.of(new RoleDescriptor(randomAlphaOfLength(10), new String[] { "all" }, null, null)); case 1 -> List.of( @@ -1752,6 +1944,15 @@ private List randomRoleDescriptors() { ) ); case 2 -> null; + // vary default role descriptor assigned to created API keys by name only + case 3 -> List.of( + new RoleDescriptor( + randomValueOtherThan(DEFAULT_API_KEY_ROLE_DESCRIPTOR.getName(), () -> randomAlphaOfLength(10)), + DEFAULT_API_KEY_ROLE_DESCRIPTOR.getClusterPrivileges(), + DEFAULT_API_KEY_ROLE_DESCRIPTOR.getIndicesPrivileges(), + DEFAULT_API_KEY_ROLE_DESCRIPTOR.getRunAs() + ) + ); default -> throw new IllegalStateException("unexpected case no"); }; } @@ -1774,6 +1975,13 @@ private void expectMetadataForApiKey(final Map expectedMetadata, assertThat("for api key doc " + actualRawApiKeyDoc, actualMetadata, equalTo(expectedMetadata)); } + private void expectCreatorForApiKey(final Map expectedCreator, final Map actualRawApiKeyDoc) { + assertNotNull(actualRawApiKeyDoc); + @SuppressWarnings("unchecked") + final var actualCreator = (Map) actualRawApiKeyDoc.get("creator"); + assertThat("for api key doc " + actualRawApiKeyDoc, actualCreator, equalTo(expectedCreator)); + } + @SuppressWarnings("unchecked") private void expectRoleDescriptorsForApiKey( final String roleDescriptorType, @@ -1940,12 +2148,17 @@ private void verifyGetResponse( } private Tuple> createApiKey(String user, TimeValue expiration) { - final Tuple, List>> res = createApiKeys(user, 1, expiration, "monitor"); + final Tuple, List>> res = createApiKeys( + user, + 1, + expiration, + DEFAULT_API_KEY_ROLE_DESCRIPTOR.getClusterPrivileges() + ); return new Tuple<>(res.v1().get(0), res.v2().get(0)); } private Tuple, List>> createApiKeys(int noOfApiKeys, TimeValue expiration) { - return createApiKeys(ES_TEST_ROOT_USER, noOfApiKeys, expiration, "monitor"); + return createApiKeys(ES_TEST_ROOT_USER, noOfApiKeys, expiration, DEFAULT_API_KEY_ROLE_DESCRIPTOR.getClusterPrivileges()); } private Tuple, List>> createApiKeys( @@ -1996,7 +2209,7 @@ private Tuple, List>> createApiKe List> metadatas = new ArrayList<>(noOfApiKeys); List responses = new ArrayList<>(); for (int i = 0; i < noOfApiKeys; i++) { - final RoleDescriptor descriptor = new RoleDescriptor("role", clusterPrivileges, null, null); + final RoleDescriptor descriptor = new RoleDescriptor(DEFAULT_API_KEY_ROLE_DESCRIPTOR.getName(), clusterPrivileges, null, null); Client client = client().filterWithHeader(headers); final Map metadata = ApiKeyTests.randomMetadata(); metadatas.add(metadata); @@ -2041,13 +2254,27 @@ private void createNativeRealmUser( assertTrue(putUserResponse.created()); } + private void updateUser(User user) throws ExecutionException, InterruptedException { + final PutUserRequest putUserRequest = new PutUserRequest(); + putUserRequest.username(user.principal()); + putUserRequest.roles(user.roles()); + putUserRequest.metadata(user.metadata()); + putUserRequest.fullName(user.fullName()); + putUserRequest.email(user.email()); + final PlainActionFuture listener = new PlainActionFuture<>(); + final Client client = client().filterWithHeader( + Map.of("Authorization", basicAuthHeaderValue(ES_TEST_ROOT_USER, TEST_PASSWORD_SECURE_STRING)) + ); + client.execute(PutUserAction.INSTANCE, putUserRequest, listener); + final PutUserResponse putUserResponse = listener.get(); + assertFalse(putUserResponse.created()); + } + private RoleDescriptor putRoleWithClusterPrivileges(final String nativeRealmRoleName, String... clusterPrivileges) throws InterruptedException, ExecutionException { final PutRoleRequest putRoleRequest = new PutRoleRequest(); putRoleRequest.name(nativeRealmRoleName); - for (final String clusterPrivilege : clusterPrivileges) { - putRoleRequest.cluster(clusterPrivilege); - } + putRoleRequest.cluster(clusterPrivileges); final PlainActionFuture roleListener = new PlainActionFuture<>(); client().filterWithHeader(Map.of("Authorization", basicAuthHeaderValue(ES_TEST_ROOT_USER, TEST_PASSWORD_SECURE_STRING))) .execute(PutRoleAction.INSTANCE, putRoleRequest, roleListener); @@ -2066,11 +2293,9 @@ private Client getClientForRunAsUser() { ); } - private UpdateApiKeyResponse executeUpdateApiKey( - final String username, - final UpdateApiKeyRequest request, - final PlainActionFuture listener - ) throws InterruptedException, ExecutionException { + private UpdateApiKeyResponse executeUpdateApiKey(final String username, final UpdateApiKeyRequest request) throws InterruptedException, + ExecutionException { + final var listener = new PlainActionFuture(); final Client client = client().filterWithHeader( Collections.singletonMap("Authorization", basicAuthHeaderValue(username, TEST_PASSWORD_SECURE_STRING)) ); diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java index 33f48b65fe9d1..867dd495e82a0 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java @@ -49,6 +49,8 @@ import org.elasticsearch.xpack.core.security.action.apikey.GrantApiKeyRequest; import org.elasticsearch.xpack.core.security.action.apikey.InvalidateApiKeyAction; import org.elasticsearch.xpack.core.security.action.apikey.InvalidateApiKeyRequest; +import org.elasticsearch.xpack.core.security.action.apikey.UpdateApiKeyAction; +import org.elasticsearch.xpack.core.security.action.apikey.UpdateApiKeyRequest; import org.elasticsearch.xpack.core.security.action.privilege.DeletePrivilegesAction; import org.elasticsearch.xpack.core.security.action.privilege.DeletePrivilegesRequest; import org.elasticsearch.xpack.core.security.action.privilege.PutPrivilegesAction; @@ -287,7 +289,8 @@ public class LoggingAuditTrail implements AuditTrail, ClusterStateListener { DeleteServiceAccountTokenAction.NAME, ActivateProfileAction.NAME, UpdateProfileDataAction.NAME, - SetProfileEnabledAction.NAME + SetProfileEnabledAction.NAME, + UpdateApiKeyAction.NAME ); private static final String FILTER_POLICY_PREFIX = setting("audit.logfile.events.ignore_filters."); // because of the default wildcard value (*) for the field filter, a policy with @@ -747,6 +750,9 @@ public void accessGranted( } else if (msg instanceof final SetProfileEnabledRequest setProfileEnabledRequest) { assert SetProfileEnabledAction.NAME.equals(action); securityChangeLogEntryBuilder(requestId).withRequestBody(setProfileEnabledRequest).build(); + } else if (msg instanceof final UpdateApiKeyRequest updateApiKeyRequest) { + assert UpdateApiKeyAction.NAME.equals(action); + securityChangeLogEntryBuilder(requestId).withRequestBody(updateApiKeyRequest).build(); } else { throw new IllegalStateException( "Unknown message class type [" @@ -1215,9 +1221,20 @@ LogEntryBuilder withRequestBody(GrantApiKeyRequest grantApiKeyRequest) throws IO return this; } + LogEntryBuilder withRequestBody(final UpdateApiKeyRequest updateApiKeyRequest) throws IOException { + logEntry.with(EVENT_ACTION_FIELD_NAME, "change_apikey"); + XContentBuilder builder = JsonXContent.contentBuilder().humanReadable(true); + builder.startObject(); + withRequestBody(builder, updateApiKeyRequest); + builder.endObject(); + logEntry.with(CHANGE_CONFIG_FIELD_NAME, Strings.toString(builder)); + return this; + } + private void withRequestBody(XContentBuilder builder, CreateApiKeyRequest createApiKeyRequest) throws IOException { TimeValue expiration = createApiKeyRequest.getExpiration(); builder.startObject("apikey") + .field("id", createApiKeyRequest.getId()) .field("name", createApiKeyRequest.getName()) .field("expiration", expiration != null ? expiration.toString() : null) .startArray("role_descriptors"); @@ -1228,6 +1245,18 @@ private void withRequestBody(XContentBuilder builder, CreateApiKeyRequest create .endObject(); // apikey } + private void withRequestBody(final XContentBuilder builder, final UpdateApiKeyRequest updateApiKeyRequest) throws IOException { + builder.startObject("apikey").field("id", updateApiKeyRequest.getId()); + if (updateApiKeyRequest.getRoleDescriptors() != null) { + builder.startArray("role_descriptors"); + for (RoleDescriptor roleDescriptor : updateApiKeyRequest.getRoleDescriptors()) { + withRoleDescriptor(builder, roleDescriptor); + } + builder.endArray(); + } + builder.endObject(); + } + private void withRoleDescriptor(XContentBuilder builder, RoleDescriptor roleDescriptor) throws IOException { builder.startObject().array(RoleDescriptor.Fields.CLUSTER.getPreferredName(), roleDescriptor.getClusterPrivileges()); if (roleDescriptor.getConditionalClusterPrivileges() != null && roleDescriptor.getConditionalClusterPrivileges().length > 0) { diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java index fe30462e9b8d6..883ad3ca98c19 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java @@ -119,6 +119,7 @@ import java.util.Collection; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; @@ -279,27 +280,27 @@ public void invalidateAll() { * Asynchronously creates a new API key based off of the request and authentication * @param authentication the authentication that this api key should be based off of * @param request the request to create the api key included any permission restrictions - * @param userRoles the user's actual roles that we always enforce + * @param userRoleDescriptors the user's actual roles that we always enforce * @param listener the listener that will be used to notify of completion */ public void createApiKey( Authentication authentication, CreateApiKeyRequest request, - Set userRoles, + Set userRoleDescriptors, ActionListener listener ) { ensureEnabled(); if (authentication == null) { listener.onFailure(new IllegalArgumentException("authentication must be provided")); } else { - createApiKeyAndIndexIt(authentication, request, userRoles, listener); + createApiKeyAndIndexIt(authentication, request, userRoleDescriptors, listener); } } private void createApiKeyAndIndexIt( Authentication authentication, CreateApiKeyRequest request, - Set roleDescriptorSet, + Set userRoleDescriptors, ActionListener listener ) { final Instant created = clock.instant(); @@ -313,7 +314,7 @@ private void createApiKeyAndIndexIt( apiKeyHashChars, request.getName(), authentication, - roleDescriptorSet, + userRoleDescriptors, created, expiration, request.getRoleDescriptors(), @@ -358,7 +359,7 @@ private void createApiKeyAndIndexIt( public void updateApiKey( final Authentication authentication, final UpdateApiKeyRequest request, - final Set userRoles, + final Set userRoleDescriptors, final ActionListener listener ) { ensureEnabled(); @@ -386,10 +387,7 @@ public void updateApiKey( validateCurrentApiKeyDocForUpdate(apiKeyId, authentication, versionedDoc.doc()); - executeBulkRequest( - buildBulkRequestForUpdate(versionedDoc, authentication, request, userRoles), - ActionListener.wrap(bulkResponse -> translateResponseAndClearCache(apiKeyId, bulkResponse, listener), listener::onFailure) - ); + doUpdateApiKey(authentication, request, userRoleDescriptors, versionedDoc, listener); }, listener::onFailure)); } @@ -418,10 +416,10 @@ static XContentBuilder newDocument( char[] apiKeyHashChars, String name, Authentication authentication, - Set userRoles, + Set userRoleDescriptors, Instant created, Instant expiration, - List keyRoles, + List keyRoleDescriptors, Version version, @Nullable Map metadata ) throws IOException { @@ -433,8 +431,8 @@ static XContentBuilder newDocument( .field("api_key_invalidated", false); addApiKeyHash(builder, apiKeyHashChars); - addRoleDescriptors(builder, keyRoles); - addLimitedByRoleDescriptors(builder, userRoles); + addRoleDescriptors(builder, keyRoleDescriptors); + addLimitedByRoleDescriptors(builder, userRoleDescriptors); builder.field("name", name).field("version", version.id).field("metadata_flattened", metadata); addCreator(builder, authentication); @@ -442,14 +440,22 @@ static XContentBuilder newDocument( return builder.endObject(); } - static XContentBuilder buildUpdatedDocument( + // package private for testing + + /** + * @return `null` if the update is a noop, i.e., if no changes to `currentApiKeyDoc` are required + */ + XContentBuilder maybeBuildUpdatedDocument( final ApiKeyDoc currentApiKeyDoc, + final Version targetDocVersion, final Authentication authentication, - final Set userRoles, - final List keyRoles, - final Version version, - final Map metadata + final UpdateApiKeyRequest request, + final Set userRoleDescriptors ) throws IOException { + if (isNoop(currentApiKeyDoc, targetDocVersion, authentication, request, userRoleDescriptors)) { + return null; + } + final XContentBuilder builder = XContentFactory.jsonBuilder(); builder.startObject() .field("doc_type", "api_key") @@ -459,6 +465,7 @@ static XContentBuilder buildUpdatedDocument( addApiKeyHash(builder, currentApiKeyDoc.hash.toCharArray()); + final List keyRoles = request.getRoleDescriptors(); if (keyRoles != null) { logger.trace(() -> format("Building API key doc with updated role descriptors [{}]", keyRoles)); addRoleDescriptors(builder, keyRoles); @@ -467,14 +474,15 @@ static XContentBuilder buildUpdatedDocument( builder.rawField("role_descriptors", currentApiKeyDoc.roleDescriptorsBytes.streamInput(), XContentType.JSON); } - addLimitedByRoleDescriptors(builder, userRoles); + addLimitedByRoleDescriptors(builder, userRoleDescriptors); - builder.field("name", currentApiKeyDoc.name).field("version", version.id); + builder.field("name", currentApiKeyDoc.name).field("version", targetDocVersion.id); assert currentApiKeyDoc.metadataFlattened == null || MetadataUtils.containsReservedMetadata( XContentHelper.convertToMap(currentApiKeyDoc.metadataFlattened, false, XContentType.JSON).v2() ) == false : "API key doc to be updated contains reserved metadata"; + final Map metadata = request.getMetadata(); if (metadata != null) { logger.trace(() -> format("Building API key doc with updated metadata [{}]", metadata)); builder.field("metadata_flattened", metadata); @@ -493,6 +501,90 @@ static XContentBuilder buildUpdatedDocument( return builder.endObject(); } + private boolean isNoop( + final ApiKeyDoc apiKeyDoc, + final Version targetDocVersion, + final Authentication authentication, + final UpdateApiKeyRequest request, + final Set userRoleDescriptors + ) { + if (apiKeyDoc.version != targetDocVersion.id) { + return false; + } + + final Map currentCreator = apiKeyDoc.creator; + final var user = authentication.getEffectiveSubject().getUser(); + final var sourceRealm = authentication.getEffectiveSubject().getRealm(); + if (false == (Objects.equals(user.principal(), currentCreator.get("principal")) + && Objects.equals(user.fullName(), currentCreator.get("full_name")) + && Objects.equals(user.email(), currentCreator.get("email")) + && Objects.equals(user.metadata(), currentCreator.get("metadata")) + && Objects.equals(sourceRealm.getName(), currentCreator.get("realm")) + && Objects.equals(sourceRealm.getType(), currentCreator.get("realm_type")))) { + return false; + } + if (sourceRealm.getDomain() != null) { + if (currentCreator.get("realm_domain") == null) { + return false; + } + @SuppressWarnings("unchecked") + final var currentRealmDomain = RealmDomain.fromXContent( + XContentHelper.mapToXContentParser( + XContentParserConfiguration.EMPTY, + (Map) currentCreator.get("realm_domain") + ) + ); + if (sourceRealm.getDomain().equals(currentRealmDomain) == false) { + return false; + } + } else { + if (currentCreator.get("realm_domain") != null) { + return false; + } + } + + final Map newMetadata = request.getMetadata(); + if (newMetadata != null) { + if (apiKeyDoc.metadataFlattened == null) { + return false; + } + final Map currentMetadata = XContentHelper.convertToMap(apiKeyDoc.metadataFlattened, false, XContentType.JSON) + .v2(); + if (newMetadata.equals(currentMetadata) == false) { + return false; + } + } + + final List newRoleDescriptors = request.getRoleDescriptors(); + if (newRoleDescriptors != null) { + final List currentRoleDescriptors = parseRoleDescriptorsBytes( + request.getId(), + apiKeyDoc.roleDescriptorsBytes, + RoleReference.ApiKeyRoleType.ASSIGNED + ); + if (false == (newRoleDescriptors.size() == currentRoleDescriptors.size() + && Set.copyOf(newRoleDescriptors).containsAll(new HashSet<>(currentRoleDescriptors)))) { + return false; + } + } + + assert userRoleDescriptors != null; + // There is an edge case here when we update an 7.x API key that has a `LEGACY_SUPERUSER_ROLE_DESCRIPTOR` role descriptor: + // `parseRoleDescriptorsBytes` automatically transforms it to `ReservedRolesStore.SUPERUSER_ROLE_DESCRIPTOR`. As such, when we + // perform the noop check on `ReservedRolesStore.SUPERUSER_ROLE_DESCRIPTOR` we will treat it as a noop even though the actual + // role descriptor bytes on the API key are different, and correspond to `LEGACY_SUPERUSER_ROLE_DESCRIPTOR`. + // + // This does *not* present a functional issue, since whenever a `LEGACY_SUPERUSER_ROLE_DESCRIPTOR` is loaded at authentication time, + // it is likewise automatically transformed to `ReservedRolesStore.SUPERUSER_ROLE_DESCRIPTOR`. + final List currentLimitedByRoleDescriptors = parseRoleDescriptorsBytes( + request.getId(), + apiKeyDoc.limitedByRoleDescriptorsBytes, + RoleReference.ApiKeyRoleType.LIMITED_BY + ); + return (userRoleDescriptors.size() == currentLimitedByRoleDescriptors.size() + && userRoleDescriptors.containsAll(currentLimitedByRoleDescriptors)); + } + void tryAuthenticate(ThreadContext ctx, ApiKeyCredentials credentials, ActionListener> listener) { if (false == isEnabled()) { listener.onResponse(AuthenticationResult.notHandled()); @@ -971,6 +1063,68 @@ public void logRemovedField(String parserName, Supplier locati } } + private void doUpdateApiKey( + final Authentication authentication, + final UpdateApiKeyRequest request, + final Set userRoleDescriptors, + final VersionedApiKeyDoc currentVersionedDoc, + final ActionListener listener + ) throws IOException { + logger.trace( + "Building update request for API key doc [{}] with seqNo [{}] and primaryTerm [{}]", + request.getId(), + currentVersionedDoc.seqNo(), + currentVersionedDoc.primaryTerm() + ); + final var targetDocVersion = clusterService.state().nodes().getMinNodeVersion(); + final var currentDocVersion = Version.fromId(currentVersionedDoc.doc().version); + assert currentDocVersion.onOrBefore(targetDocVersion) : "current API key doc version must be on or before target version"; + if (currentDocVersion.before(targetDocVersion)) { + logger.debug( + "API key update for [{}] will update version from [{}] to [{}]", + request.getId(), + currentDocVersion, + targetDocVersion + ); + } + + final XContentBuilder builder = maybeBuildUpdatedDocument( + currentVersionedDoc.doc(), + targetDocVersion, + authentication, + request, + userRoleDescriptors + ); + final boolean isNoop = builder == null; + if (isNoop) { + logger.debug("Detected noop update request for API key [{}]. Skipping index request.", request.getId()); + listener.onResponse(new UpdateApiKeyResponse(false)); + return; + } + + final IndexRequest indexRequest = client.prepareIndex(SECURITY_MAIN_ALIAS) + .setId(request.getId()) + .setSource(builder) + .setIfSeqNo(currentVersionedDoc.seqNo()) + .setIfPrimaryTerm(currentVersionedDoc.primaryTerm()) + .setOpType(DocWriteRequest.OpType.INDEX) + .request(); + logger.trace("Executing index request to update API key [{}]", request.getId()); + securityIndex.prepareIndexIfNeededThenExecute( + listener::onFailure, + () -> executeAsyncWithOrigin( + client.threadPool().getThreadContext(), + SECURITY_ORIGIN, + client.prepareBulk().add(indexRequest).setRefreshPolicy(RefreshPolicy.WAIT_UNTIL).request(), + ActionListener.wrap( + bulkResponse -> translateResponseAndClearCache(request.getId(), bulkResponse, listener), + listener::onFailure + ), + client::bulk + ) + ); + } + /** * Invalidate API keys for given realm, user name, API key name and id. * @param realmNames realm names @@ -1235,63 +1389,11 @@ private static VersionedApiKeyDoc singleDoc(final String apiKeyId, final Collect return elements.iterator().next(); } - private BulkRequest buildBulkRequestForUpdate( - final VersionedApiKeyDoc versionedDoc, - final Authentication authentication, - final UpdateApiKeyRequest request, - final Set userRoles - ) throws IOException { - logger.trace( - "Building update request for API key doc [{}] with seqNo [{}] and primaryTerm [{}]", - request.getId(), - versionedDoc.seqNo(), - versionedDoc.primaryTerm() - ); - final var currentDocVersion = Version.fromId(versionedDoc.doc().version); - final var targetDocVersion = clusterService.state().nodes().getMinNodeVersion(); - assert currentDocVersion.onOrBefore(targetDocVersion) : "current API key doc version must be on or before target version"; - if (currentDocVersion.before(targetDocVersion)) { - logger.debug( - "API key update for [{}] will update version from [{}] to [{}]", - request.getId(), - currentDocVersion, - targetDocVersion - ); - } - final var bulkRequestBuilder = client.prepareBulk(); - bulkRequestBuilder.add( - client.prepareIndex(SECURITY_MAIN_ALIAS) - .setId(request.getId()) - .setSource( - buildUpdatedDocument( - versionedDoc.doc(), - authentication, - userRoles, - request.getRoleDescriptors(), - targetDocVersion, - request.getMetadata() - ) - ) - .setIfSeqNo(versionedDoc.seqNo()) - .setIfPrimaryTerm(versionedDoc.primaryTerm()) - .setOpType(DocWriteRequest.OpType.INDEX) - .request() - ); - bulkRequestBuilder.setRefreshPolicy(RefreshPolicy.WAIT_UNTIL); - return bulkRequestBuilder.request(); - } - - private void executeBulkRequest(final BulkRequest bulkRequest, final ActionListener listener) { - securityIndex.prepareIndexIfNeededThenExecute( - listener::onFailure, - () -> executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, bulkRequest, listener, client::bulk) - ); - } - - private static void addLimitedByRoleDescriptors(final XContentBuilder builder, final Set userRoles) throws IOException { - assert userRoles != null; + private static void addLimitedByRoleDescriptors(final XContentBuilder builder, final Set limitedByRoleDescriptors) + throws IOException { + assert limitedByRoleDescriptors != null; builder.startObject("limited_by_role_descriptors"); - for (RoleDescriptor descriptor : userRoles) { + for (RoleDescriptor descriptor : limitedByRoleDescriptors) { builder.field(descriptor.getName(), (contentBuilder, params) -> descriptor.toXContent(contentBuilder, params, true)); } builder.endObject(); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailTests.java index 4ea4ce42a19eb..f85b28bcbba98 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailTests.java @@ -46,12 +46,15 @@ import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.core.XPackSettings; +import org.elasticsearch.xpack.core.security.action.apikey.ApiKeyTests; import org.elasticsearch.xpack.core.security.action.apikey.CreateApiKeyAction; import org.elasticsearch.xpack.core.security.action.apikey.CreateApiKeyRequest; import org.elasticsearch.xpack.core.security.action.apikey.GrantApiKeyAction; import org.elasticsearch.xpack.core.security.action.apikey.GrantApiKeyRequest; import org.elasticsearch.xpack.core.security.action.apikey.InvalidateApiKeyAction; import org.elasticsearch.xpack.core.security.action.apikey.InvalidateApiKeyRequest; +import org.elasticsearch.xpack.core.security.action.apikey.UpdateApiKeyAction; +import org.elasticsearch.xpack.core.security.action.apikey.UpdateApiKeyRequest; import org.elasticsearch.xpack.core.security.action.privilege.DeletePrivilegesAction; import org.elasticsearch.xpack.core.security.action.privilege.DeletePrivilegesRequest; import org.elasticsearch.xpack.core.security.action.privilege.PutPrivilegesAction; @@ -587,8 +590,13 @@ public void testSecurityConfigChangeEventFormattingForRoles() throws IOException createApiKeyRequest.setRefreshPolicy(randomFrom(WriteRequest.RefreshPolicy.values())); auditTrail.accessGranted(requestId, authentication, CreateApiKeyAction.NAME, createApiKeyRequest, authorizationInfo); String expectedCreateKeyAuditEventString = """ - "create":{"apikey":{"name":"%s","expiration":%s,%s}}\ - """.formatted(keyName, expiration != null ? "\"" + expiration + "\"" : "null", roleDescriptorsStringBuilder); + "create":{"apikey":{"id":"%s","name":"%s","expiration":%s,%s}}\ + """.formatted( + createApiKeyRequest.getId(), + keyName, + expiration != null ? "\"" + expiration + "\"" : "null", + roleDescriptorsStringBuilder + ); List output = CapturingLogger.output(logger.getName(), Level.INFO); assertThat(output.size(), is(2)); String generatedCreateKeyAuditEventString = output.get(1); @@ -605,6 +613,32 @@ public void testSecurityConfigChangeEventFormattingForRoles() throws IOException // clear log CapturingLogger.output(logger.getName(), Level.INFO).clear(); + final String keyId = randomAlphaOfLength(10); + final var updateApiKeyRequest = new UpdateApiKeyRequest( + keyId, + randomBoolean() ? null : keyRoleDescriptors, + ApiKeyTests.randomMetadata() + ); + auditTrail.accessGranted(requestId, authentication, UpdateApiKeyAction.NAME, updateApiKeyRequest, authorizationInfo); + final var expectedUpdateKeyAuditEventString = """ + "change":{"apikey":{"id":"%s"%s}}\ + """.formatted(keyId, updateApiKeyRequest.getRoleDescriptors() == null ? "" : "," + roleDescriptorsStringBuilder); + output = CapturingLogger.output(logger.getName(), Level.INFO); + assertThat(output.size(), is(2)); + String generatedUpdateKeyAuditEventString = output.get(1); + assertThat(generatedUpdateKeyAuditEventString, containsString(expectedUpdateKeyAuditEventString)); + generatedUpdateKeyAuditEventString = generatedUpdateKeyAuditEventString.replace(", " + expectedUpdateKeyAuditEventString, ""); + checkedFields = new MapBuilder<>(commonFields); + checkedFields.remove(LoggingAuditTrail.ORIGIN_ADDRESS_FIELD_NAME); + checkedFields.remove(LoggingAuditTrail.ORIGIN_TYPE_FIELD_NAME); + checkedFields.put("type", "audit") + .put(LoggingAuditTrail.EVENT_TYPE_FIELD_NAME, "security_config_change") + .put(LoggingAuditTrail.EVENT_ACTION_FIELD_NAME, "change_apikey") + .put(LoggingAuditTrail.REQUEST_ID_FIELD_NAME, requestId); + assertMsg(generatedUpdateKeyAuditEventString, checkedFields.map()); + // clear log + CapturingLogger.output(logger.getName(), Level.INFO).clear(); + GrantApiKeyRequest grantApiKeyRequest = new GrantApiKeyRequest(); grantApiKeyRequest.setRefreshPolicy(randomFrom(WriteRequest.RefreshPolicy.values())); grantApiKeyRequest.getGrant().setType(randomFrom(randomAlphaOfLength(8), null)); @@ -617,7 +651,9 @@ public void testSecurityConfigChangeEventFormattingForRoles() throws IOException output = CapturingLogger.output(logger.getName(), Level.INFO); assertThat(output.size(), is(2)); String generatedGrantKeyAuditEventString = output.get(1); - StringBuilder grantKeyAuditEventStringBuilder = new StringBuilder().append("\"create\":{\"apikey\":{\"name\":\"") + StringBuilder grantKeyAuditEventStringBuilder = new StringBuilder().append("\"create\":{\"apikey\":{\"id\":\"") + .append(grantApiKeyRequest.getApiKeyRequest().getId()) + .append("\",\"name\":\"") .append(keyName) .append("\",\"expiration\":") .append(expiration != null ? "\"" + expiration + "\"" : "null") @@ -1800,7 +1836,8 @@ public void testSecurityConfigChangedEventSelection() { new Tuple<>( SetProfileEnabledAction.NAME, new SetProfileEnabledRequest(randomAlphaOfLength(20), randomBoolean(), WriteRequest.RefreshPolicy.WAIT_UNTIL) - ) + ), + new Tuple<>(UpdateApiKeyAction.NAME, UpdateApiKeyRequest.usingApiKeyId(randomAlphaOfLength(10))) ); auditTrail.accessGranted(requestId, authentication, actionAndRequest.v1(), actionAndRequest.v2(), authorizationInfo); List output = CapturingLogger.output(logger.getName(), Level.INFO); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java index 181dcf8211283..ae4fa08bc0806 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java @@ -51,6 +51,7 @@ import org.elasticsearch.test.ClusterServiceUtils; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.MockLogAppender; +import org.elasticsearch.test.VersionUtils; import org.elasticsearch.test.XContentTestUtils; import org.elasticsearch.threadpool.FixedExecutorBuilder; import org.elasticsearch.threadpool.TestThreadPool; @@ -68,12 +69,14 @@ import org.elasticsearch.xpack.core.security.action.apikey.CreateApiKeyResponse; import org.elasticsearch.xpack.core.security.action.apikey.GetApiKeyResponse; import org.elasticsearch.xpack.core.security.action.apikey.InvalidateApiKeyResponse; +import org.elasticsearch.xpack.core.security.action.apikey.UpdateApiKeyRequest; import org.elasticsearch.xpack.core.security.authc.Authentication; import org.elasticsearch.xpack.core.security.authc.Authentication.RealmRef; import org.elasticsearch.xpack.core.security.authc.AuthenticationField; import org.elasticsearch.xpack.core.security.authc.AuthenticationResult; import org.elasticsearch.xpack.core.security.authc.AuthenticationTestHelper; import org.elasticsearch.xpack.core.security.authc.AuthenticationTests; +import org.elasticsearch.xpack.core.security.authc.RealmDomain; import org.elasticsearch.xpack.core.security.authc.support.AuthenticationContextSerializer; import org.elasticsearch.xpack.core.security.authc.support.Hasher; import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; @@ -1671,91 +1674,135 @@ public void testValidateApiKeyDocBeforeUpdate() throws IOException { assertThat(ex.getMessage(), containsString("cannot update legacy API key [" + apiKeyId + "] without name")); } - public void testBuildUpdatedDocument() throws IOException { + public void testMaybeBuildUpdatedDocument() throws IOException { final var apiKey = randomAlphaOfLength(16); final var hasher = getFastStoredHashAlgoForTests(); final char[] hash = hasher.hash(new SecureString(apiKey.toCharArray())); - - final var oldApiKeyDoc = buildApiKeyDoc(hash, randomBoolean() ? -1 : Instant.now().toEpochMilli(), false); - - final Set newUserRoles = randomBoolean() ? Set.of() : Set.of(RoleDescriptorTests.randomRoleDescriptor()); - - final boolean nullKeyRoles = randomBoolean(); - final List newKeyRoles; - if (nullKeyRoles) { - newKeyRoles = null; - } else { - newKeyRoles = List.of(RoleDescriptorTests.randomRoleDescriptor()); - } - - final var metadata = ApiKeyTests.randomMetadata(); - final var version = Version.CURRENT; - final var authentication = randomValueOtherThanMany( + final var oldAuthentication = randomValueOtherThanMany( Authentication::isApiKey, - () -> AuthenticationTestHelper.builder().user(new User("user", "role")).build(false) - ); - - final var keyDocSource = ApiKeyService.buildUpdatedDocument( - oldApiKeyDoc, - authentication, - newUserRoles, - newKeyRoles, - version, - metadata + () -> AuthenticationTestHelper.builder() + .user(AuthenticationTestHelper.userWithRandomMetadataAndDetails("user", "role")) + .build(false) ); - final var updatedApiKeyDoc = ApiKeyDoc.fromXContent( - XContentHelper.createParser(XContentParserConfiguration.EMPTY, BytesReference.bytes(keyDocSource), XContentType.JSON) + final Set oldUserRoles = randomSet(0, 3, RoleDescriptorTests::randomRoleDescriptor); + final List oldKeyRoles = randomList(3, RoleDescriptorTests::randomRoleDescriptor); + final Map oldMetadata = ApiKeyTests.randomMetadata(); + final Version oldVersion = VersionUtils.randomVersion(random()); + final ApiKeyDoc oldApiKeyDoc = ApiKeyDoc.fromXContent( + XContentHelper.createParser( + XContentParserConfiguration.EMPTY, + BytesReference.bytes( + ApiKeyService.newDocument( + hash, + randomAlphaOfLength(10), + oldAuthentication, + oldUserRoles, + Instant.now(), + randomBoolean() ? null : Instant.now(), + oldKeyRoles, + oldVersion, + oldMetadata + ) + ), + XContentType.JSON + ) ); - assertEquals(oldApiKeyDoc.docType, updatedApiKeyDoc.docType); - assertEquals(oldApiKeyDoc.name, updatedApiKeyDoc.name); - assertEquals(oldApiKeyDoc.hash, updatedApiKeyDoc.hash); - assertEquals(oldApiKeyDoc.expirationTime, updatedApiKeyDoc.expirationTime); - assertEquals(oldApiKeyDoc.creationTime, updatedApiKeyDoc.creationTime); - assertEquals(oldApiKeyDoc.invalidated, updatedApiKeyDoc.invalidated); - - final var service = createApiKeyService(Settings.EMPTY); - final var actualUserRoles = service.parseRoleDescriptorsBytes( - "", - updatedApiKeyDoc.limitedByRoleDescriptorsBytes, - RoleReference.ApiKeyRoleType.LIMITED_BY - ); - assertEquals(newUserRoles.size(), actualUserRoles.size()); - assertEquals(new HashSet<>(newUserRoles), new HashSet<>(actualUserRoles)); + final boolean changeUserRoles = randomBoolean(); + final boolean changeKeyRoles = randomBoolean(); + final boolean changeMetadata = randomBoolean(); + final boolean changeVersion = randomBoolean(); + final boolean changeCreator = randomBoolean(); + final Set newUserRoles = changeUserRoles + ? randomValueOtherThan(oldUserRoles, () -> randomSet(0, 3, RoleDescriptorTests::randomRoleDescriptor)) + : oldUserRoles; + final List newKeyRoles = changeKeyRoles + ? randomValueOtherThan(oldKeyRoles, () -> randomList(0, 3, RoleDescriptorTests::randomRoleDescriptor)) + : (randomBoolean() ? oldKeyRoles : null); + final Map newMetadata = changeMetadata + ? randomValueOtherThanMany(md -> md == null || md.equals(oldMetadata), ApiKeyTests::randomMetadata) + : (randomBoolean() ? oldMetadata : null); + final Version newVersion = changeVersion + ? randomValueOtherThan(oldVersion, () -> VersionUtils.randomVersion(random())) + : oldVersion; + final Authentication newAuthentication = changeCreator + ? randomValueOtherThanMany( + (auth -> auth.isApiKey() || auth.getEffectiveSubject().getUser().equals(oldAuthentication.getEffectiveSubject().getUser())), + () -> AuthenticationTestHelper.builder() + .user(AuthenticationTestHelper.userWithRandomMetadataAndDetails("user", "role")) + .build(false) + ) + : oldAuthentication; + final var request = new UpdateApiKeyRequest(randomAlphaOfLength(10), newKeyRoles, newMetadata); + final var service = createApiKeyService(); - final var actualKeyRoles = service.parseRoleDescriptorsBytes( - "", - updatedApiKeyDoc.roleDescriptorsBytes, - RoleReference.ApiKeyRoleType.ASSIGNED + final XContentBuilder builder = service.maybeBuildUpdatedDocument( + oldApiKeyDoc, + newVersion, + newAuthentication, + request, + newUserRoles ); - if (nullKeyRoles) { - assertEquals( - service.parseRoleDescriptorsBytes("", oldApiKeyDoc.roleDescriptorsBytes, RoleReference.ApiKeyRoleType.ASSIGNED), - actualKeyRoles - ); - } else { - assertEquals(newKeyRoles.size(), actualKeyRoles.size()); - assertEquals(new HashSet<>(newKeyRoles), new HashSet<>(actualKeyRoles)); - } - if (metadata == null) { - assertEquals(oldApiKeyDoc.metadataFlattened, updatedApiKeyDoc.metadataFlattened); - } else { - assertEquals(metadata, XContentHelper.convertToMap(updatedApiKeyDoc.metadataFlattened, true, XContentType.JSON).v2()); - } - assertEquals(authentication.getEffectiveSubject().getUser().principal(), updatedApiKeyDoc.creator.getOrDefault("principal", null)); - assertEquals(authentication.getEffectiveSubject().getUser().fullName(), updatedApiKeyDoc.creator.getOrDefault("fullName", null)); - assertEquals(authentication.getEffectiveSubject().getUser().email(), updatedApiKeyDoc.creator.getOrDefault("email", null)); - assertEquals(authentication.getEffectiveSubject().getUser().metadata(), updatedApiKeyDoc.creator.getOrDefault("metadata", null)); - RealmRef realm = authentication.getEffectiveSubject().getRealm(); - assertEquals(realm.getName(), updatedApiKeyDoc.creator.getOrDefault("realm", null)); - assertEquals(realm.getType(), updatedApiKeyDoc.creator.getOrDefault("realm_type", null)); - if (realm.getDomain() != null) { - @SuppressWarnings("unchecked") - final var actualDomain = (Map) updatedApiKeyDoc.creator.getOrDefault("realm_domain", null); - assertEquals(realm.getDomain().name(), actualDomain.get("name")); + final boolean noop = (changeCreator || changeMetadata || changeKeyRoles || changeUserRoles || changeVersion) == false; + if (noop) { + assertNull(builder); } else { - assertFalse(updatedApiKeyDoc.creator.containsKey("realm_domain")); + final ApiKeyDoc updatedApiKeyDoc = ApiKeyDoc.fromXContent( + XContentHelper.createParser(XContentParserConfiguration.EMPTY, BytesReference.bytes(builder), XContentType.JSON) + ); + assertEquals(oldApiKeyDoc.docType, updatedApiKeyDoc.docType); + assertEquals(oldApiKeyDoc.name, updatedApiKeyDoc.name); + assertEquals(oldApiKeyDoc.hash, updatedApiKeyDoc.hash); + assertEquals(oldApiKeyDoc.expirationTime, updatedApiKeyDoc.expirationTime); + assertEquals(oldApiKeyDoc.creationTime, updatedApiKeyDoc.creationTime); + assertEquals(oldApiKeyDoc.invalidated, updatedApiKeyDoc.invalidated); + assertEquals(newVersion.id, updatedApiKeyDoc.version); + final var actualUserRoles = service.parseRoleDescriptorsBytes( + "", + updatedApiKeyDoc.limitedByRoleDescriptorsBytes, + RoleReference.ApiKeyRoleType.LIMITED_BY + ); + assertEquals(newUserRoles.size(), actualUserRoles.size()); + assertEquals(new HashSet<>(newUserRoles), new HashSet<>(actualUserRoles)); + final var actualKeyRoles = service.parseRoleDescriptorsBytes( + "", + updatedApiKeyDoc.roleDescriptorsBytes, + RoleReference.ApiKeyRoleType.ASSIGNED + ); + if (changeKeyRoles == false) { + assertEquals( + service.parseRoleDescriptorsBytes("", oldApiKeyDoc.roleDescriptorsBytes, RoleReference.ApiKeyRoleType.ASSIGNED), + actualKeyRoles + ); + } else { + assertEquals(newKeyRoles.size(), actualKeyRoles.size()); + assertEquals(new HashSet<>(newKeyRoles), new HashSet<>(actualKeyRoles)); + } + if (changeMetadata == false) { + assertEquals(oldApiKeyDoc.metadataFlattened, updatedApiKeyDoc.metadataFlattened); + } else { + assertEquals(newMetadata, XContentHelper.convertToMap(updatedApiKeyDoc.metadataFlattened, true, XContentType.JSON).v2()); + } + assertEquals(newAuthentication.getEffectiveSubject().getUser().principal(), updatedApiKeyDoc.creator.get("principal")); + assertEquals(newAuthentication.getEffectiveSubject().getUser().fullName(), updatedApiKeyDoc.creator.get("full_name")); + assertEquals(newAuthentication.getEffectiveSubject().getUser().email(), updatedApiKeyDoc.creator.get("email")); + assertEquals(newAuthentication.getEffectiveSubject().getUser().metadata(), updatedApiKeyDoc.creator.get("metadata")); + final RealmRef realm = newAuthentication.getEffectiveSubject().getRealm(); + assertEquals(realm.getName(), updatedApiKeyDoc.creator.get("realm")); + assertEquals(realm.getType(), updatedApiKeyDoc.creator.get("realm_type")); + if (realm.getDomain() != null) { + @SuppressWarnings("unchecked") + final var actualRealmDomain = RealmDomain.fromXContent( + XContentHelper.mapToXContentParser( + XContentParserConfiguration.EMPTY, + (Map) updatedApiKeyDoc.creator.get("realm_domain") + ) + ); + assertEquals(realm.getDomain(), actualRealmDomain); + } else { + assertFalse(updatedApiKeyDoc.creator.containsKey("realm_domain")); + } } } diff --git a/x-pack/plugin/shutdown/src/test/java/org/elasticsearch/xpack/shutdown/TransportGetShutdownStatusActionTests.java b/x-pack/plugin/shutdown/src/test/java/org/elasticsearch/xpack/shutdown/TransportGetShutdownStatusActionTests.java index a12fe6bd57a15..f1ccfdd77f19b 100644 --- a/x-pack/plugin/shutdown/src/test/java/org/elasticsearch/xpack/shutdown/TransportGetShutdownStatusActionTests.java +++ b/x-pack/plugin/shutdown/src/test/java/org/elasticsearch/xpack/shutdown/TransportGetShutdownStatusActionTests.java @@ -93,7 +93,12 @@ public Decision canRebalance(ShardRouting shardRouting, RoutingAllocation alloca } @Override - public Decision canRemain(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { + public Decision canRemain( + IndexMetadata indexMetadata, + ShardRouting shardRouting, + RoutingNode node, + RoutingAllocation allocation + ) { return canRemain.get().test(shardRouting, node, allocation); } diff --git a/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/search/GeoBoundingBoxQueryGeoShapeWithDocValuesIT.java b/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/search/GeoBoundingBoxQueryGeoShapeWithDocValuesIT.java index 8206454e926b7..2b4400bb5a3fd 100644 --- a/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/search/GeoBoundingBoxQueryGeoShapeWithDocValuesIT.java +++ b/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/search/GeoBoundingBoxQueryGeoShapeWithDocValuesIT.java @@ -21,11 +21,6 @@ public class GeoBoundingBoxQueryGeoShapeWithDocValuesIT extends GeoBoundingBoxQueryIntegTestCase { - @Override - protected boolean addMockGeoShapeFieldMapper() { - return false; - } - @Override protected Collection> nodePlugins() { return Collections.singleton(LocalStateSpatialPlugin.class); diff --git a/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/search/GeoBoundingBoxQueryLegacyGeoShapeWithDocValuesIT.java b/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/search/GeoBoundingBoxQueryLegacyGeoShapeWithDocValuesIT.java index 2073eac56218f..9b8814053c066 100644 --- a/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/search/GeoBoundingBoxQueryLegacyGeoShapeWithDocValuesIT.java +++ b/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/search/GeoBoundingBoxQueryLegacyGeoShapeWithDocValuesIT.java @@ -21,11 +21,6 @@ public class GeoBoundingBoxQueryLegacyGeoShapeWithDocValuesIT extends GeoBoundingBoxQueryIntegTestCase { - @Override - protected boolean addMockGeoShapeFieldMapper() { - return false; - } - @Override protected Collection> nodePlugins() { return Collections.singleton(LocalStateSpatialPlugin.class); diff --git a/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/search/GeoGridAggAndQueryConsistencyIT.java b/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/search/GeoGridAggAndQueryConsistencyIT.java index 67b009ed9bf86..52497a0cc9b7d 100644 --- a/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/search/GeoGridAggAndQueryConsistencyIT.java +++ b/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/search/GeoGridAggAndQueryConsistencyIT.java @@ -51,11 +51,6 @@ public class GeoGridAggAndQueryConsistencyIT extends ESIntegTestCase { - @Override - protected boolean addMockGeoShapeFieldMapper() { - return false; - } - @Override protected Collection> nodePlugins() { return Collections.singleton(LocalStateSpatialPlugin.class); diff --git a/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/search/GeoShapeWithDocValuesIT.java b/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/search/GeoShapeWithDocValuesIT.java index 16b4735cb55c1..faca74d83c6cc 100644 --- a/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/search/GeoShapeWithDocValuesIT.java +++ b/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/search/GeoShapeWithDocValuesIT.java @@ -25,11 +25,6 @@ public class GeoShapeWithDocValuesIT extends GeoShapeIntegTestCase { - @Override - protected boolean addMockGeoShapeFieldMapper() { - return false; - } - @Override protected Collection> nodePlugins() { return Collections.singleton(LocalStateSpatialPlugin.class); diff --git a/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/search/LegacyGeoShapeWithDocValuesIT.java b/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/search/LegacyGeoShapeWithDocValuesIT.java index 660fc7fd95bc1..f6fec0f64f75b 100644 --- a/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/search/LegacyGeoShapeWithDocValuesIT.java +++ b/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/search/LegacyGeoShapeWithDocValuesIT.java @@ -29,11 +29,6 @@ public class LegacyGeoShapeWithDocValuesIT extends GeoShapeIntegTestCase { - @Override - protected boolean addMockGeoShapeFieldMapper() { - return false; - } - @Override protected Collection> nodePlugins() { return Collections.singleton(LocalStateSpatialPlugin.class); diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/query/GeoGridQueryBuilderTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/query/GeoGridQueryBuilderTests.java index 3016fd92dc33f..cd174d4c6fb0a 100644 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/query/GeoGridQueryBuilderTests.java +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/query/GeoGridQueryBuilderTests.java @@ -12,17 +12,22 @@ import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; import org.elasticsearch.common.ParsingException; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.geo.GeometryTestUtils; import org.elasticsearch.geometry.Rectangle; import org.elasticsearch.geometry.utils.Geohash; import org.elasticsearch.h3.H3; import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.query.QueryShardException; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils; import org.elasticsearch.test.AbstractQueryTestCase; +import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xpack.spatial.LocalStateSpatialPlugin; import java.io.IOException; @@ -38,6 +43,20 @@ public class GeoGridQueryBuilderTests extends AbstractQueryTestCase { + private static final String GEO_SHAPE_FIELD_NAME = "mapped_geo_shape"; + protected static final String GEO_SHAPE_ALIAS_FIELD_NAME = "mapped_geo_shape_alias"; + + @Override + protected void initializeAdditionalMappings(MapperService mapperService) throws IOException { + final XContentBuilder builder = PutMappingRequest.simpleMapping( + GEO_SHAPE_FIELD_NAME, + "type=geo_shape", + GEO_SHAPE_ALIAS_FIELD_NAME, + "type=alias,path=" + GEO_SHAPE_FIELD_NAME + ); + mapperService.merge("_doc", new CompressedXContent(Strings.toString(builder)), MapperService.MergeReason.MAPPING_UPDATE); + } + @Override protected Collection> getPlugins() { return Arrays.asList(LocalStateSpatialPlugin.class); @@ -45,11 +64,11 @@ protected Collection> getPlugins() { @Override protected GeoGridQueryBuilder doCreateTestQueryBuilder() { - String fieldName = randomFrom(GEO_POINT_FIELD_NAME, GEO_POINT_ALIAS_FIELD_NAME, GEO_SHAPE_FIELD_NAME); + String fieldName = randomFrom(GEO_POINT_FIELD_NAME, GEO_POINT_ALIAS_FIELD_NAME, GEO_SHAPE_FIELD_NAME, GEO_SHAPE_ALIAS_FIELD_NAME); GeoGridQueryBuilder builder = new GeoGridQueryBuilder(fieldName); // Only use geohex for points - int path = randomIntBetween(0, GEO_SHAPE_FIELD_NAME.equals(fieldName) ? 1 : 2); + int path = randomIntBetween(0, GEO_SHAPE_FIELD_NAME.equals(fieldName) || GEO_SHAPE_ALIAS_FIELD_NAME.equals(fieldName) ? 1 : 3); switch (path) { case 0 -> builder.setGridId(GeoGridQueryBuilder.Grid.GEOHASH, randomGeohash()); case 1 -> builder.setGridId(GeoGridQueryBuilder.Grid.GEOTILE, randomGeotile()); diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/api_key/30_update.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/api_key/30_update.yml index 013d28113521b..73ff3fba19b46 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/api_key/30_update.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/api_key/30_update.yml @@ -223,7 +223,7 @@ teardown: Authorization: "Basic YXBpX2tleV91c2VyXzE6eC1wYWNrLXRlc3QtcGFzc3dvcmQ=" # api_key_user_1 security.update_api_key: id: "$user1_key_id" - - match: { updated: true } + - match: { updated: false } # Check metadata did not change - do: diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/security/authz/14_cat_indices.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/security/authz/14_cat_indices.yml index 09d0d416e54da..31fb9e0810b90 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/security/authz/14_cat_indices.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/security/authz/14_cat_indices.yml @@ -120,8 +120,6 @@ teardown: "Test empty request while single authorized closed index": - skip: - version: "all" - reason: "AwaitsFix https://github.com/elastic/elasticsearch/issues/47875" features: ["allowed_warnings"] - do: